377 lines
13 KiB
Python
377 lines
13 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
|
||
|
|
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||
|
|
# Copyright 2012 California Institute of Technology. ALL RIGHTS RESERVED.
|
||
|
|
#
|
||
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
# you may not use this file except in compliance with the License.
|
||
|
|
# You may obtain a copy of the License at
|
||
|
|
#
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
#
|
||
|
|
# Unless required by applicable law or agreed to in writing, software
|
||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
# See the License for the specific language governing permissions and
|
||
|
|
# limitations under the License.
|
||
|
|
#
|
||
|
|
# United States Government Sponsorship acknowledged. This software is subject to
|
||
|
|
# U.S. export control laws and regulations and has been classified as 'EAR99 NLR'
|
||
|
|
# (No [Export] License Required except when exporting to an embargoed country,
|
||
|
|
# end user, or in support of a prohibited end use). By downloading this software,
|
||
|
|
# the user agrees to comply with all applicable U.S. export laws and regulations.
|
||
|
|
# The user has the responsibility to obtain export licenses, or other export
|
||
|
|
# authority as may be required before exporting this software to any 'EAR99'
|
||
|
|
# embargoed foreign country or citizen of those countries.
|
||
|
|
#
|
||
|
|
# Author: Giangi Sacco
|
||
|
|
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
from iscesys.Component.Component import Component
|
||
|
|
import numpy as np
|
||
|
|
import os
|
||
|
|
import abc
|
||
|
|
from iscesys.Stitcher.Stitcher import Stitcher as ST
|
||
|
|
from iscesys.DataRetriever.DataRetriever import DataRetriever as DR
|
||
|
|
DTYPE = Component.Parameter('_dtype',
|
||
|
|
public_name = 'dtype',
|
||
|
|
default = '',
|
||
|
|
type = str,
|
||
|
|
mandatory = True,
|
||
|
|
doc = 'Data type')
|
||
|
|
OUTPUT_FILE = Component.Parameter('_outputFile',
|
||
|
|
public_name='outputFile',
|
||
|
|
default = '',
|
||
|
|
type = str,
|
||
|
|
mandatory = True,
|
||
|
|
doc = 'Output file.')
|
||
|
|
TILE_SIZE = Component.Parameter('_tileSize',
|
||
|
|
public_name = 'tileSize',
|
||
|
|
default = [],
|
||
|
|
container=list,
|
||
|
|
type=int,
|
||
|
|
mandatory = True,
|
||
|
|
doc = 'Two element list with the number of row and columns of the tile.')
|
||
|
|
OVERLAP = Component.Parameter('_overlap',
|
||
|
|
public_name = 'overlap',
|
||
|
|
default = [1,1],
|
||
|
|
container=list,
|
||
|
|
type=int,
|
||
|
|
mandatory = False,
|
||
|
|
doc = 'Number of overlapping pixels between two tiles along the rows and columns.')
|
||
|
|
URL = Component.Parameter('_url',
|
||
|
|
public_name = 'URL',default = '',
|
||
|
|
type = str,
|
||
|
|
mandatory = False,
|
||
|
|
doc = "URL where to get the data from")
|
||
|
|
USERNAME = Component.Parameter('_un',
|
||
|
|
public_name='username',
|
||
|
|
default = None,
|
||
|
|
type = str,
|
||
|
|
mandatory = False,
|
||
|
|
doc = "Username in case the url is password protected")
|
||
|
|
PASSWORD = Component.Parameter('_pw',
|
||
|
|
public_name='password',
|
||
|
|
default = None,
|
||
|
|
type = str,
|
||
|
|
mandatory = False,
|
||
|
|
doc = "Password in case the url is password protected")
|
||
|
|
DIRECTORY = Component.Parameter('_downloadDir',
|
||
|
|
public_name='directory',
|
||
|
|
default = './',
|
||
|
|
type = str,
|
||
|
|
mandatory = False,
|
||
|
|
doc = "Location where the files are downloaded")
|
||
|
|
KEEP = Component.Parameter('_keep',
|
||
|
|
public_name='keep',
|
||
|
|
default = False,
|
||
|
|
type = bool,
|
||
|
|
mandatory = False,
|
||
|
|
doc = "Keep the files downloaded after stitching")
|
||
|
|
ENDIAN = Component.Parameter('_endian',
|
||
|
|
public_name = 'endian',
|
||
|
|
default = '>',
|
||
|
|
type = str,
|
||
|
|
mandatory = False,
|
||
|
|
doc = 'Data endianness. > big endian, < small endian')
|
||
|
|
USE_LOCAL = Component.Parameter('_useLocal',
|
||
|
|
public_name='useLocal',
|
||
|
|
default = False,
|
||
|
|
type = bool,
|
||
|
|
mandatory = False,
|
||
|
|
doc = "If the option is True then use the files that are in the location\n" + \
|
||
|
|
"specified by 'directory' for stitching. If not present 'directory' indicates\n" + \
|
||
|
|
"the directory where the files are downloaded.\n " + \
|
||
|
|
"When 'useLocal' is True then 'keep' is considered False\n " +\
|
||
|
|
"to avoid accidental removal of user files (default: False)")
|
||
|
|
FILLING_VALUE = Component.Parameter('_fillingValue',
|
||
|
|
public_name = 'fillingValue',
|
||
|
|
default = 0,
|
||
|
|
type=float,
|
||
|
|
mandatory = True,
|
||
|
|
doc = 'Value used for missing tiles.')
|
||
|
|
NO_FILLING = Component.Parameter('_noFilling',
|
||
|
|
public_name='noFilling',
|
||
|
|
default = True,
|
||
|
|
type = bool,
|
||
|
|
mandatory = False,
|
||
|
|
doc = "If the flag is False the missing tiles are filled with 'fillingValue' values" )
|
||
|
|
PROCEED_IF_NO_SERVER = Component.Parameter(
|
||
|
|
'_proceedIfNoServer',
|
||
|
|
public_name='proceed if no server',
|
||
|
|
default=False,
|
||
|
|
type=bool,
|
||
|
|
mandatory=False,
|
||
|
|
doc='Flag to continue even if server is down.'
|
||
|
|
)
|
||
|
|
|
||
|
|
class TileManager(Component,metaclass=abc.ABCMeta):
|
||
|
|
family = 'tilemanager'
|
||
|
|
parameter_list = (
|
||
|
|
URL,
|
||
|
|
USERNAME,
|
||
|
|
PASSWORD,
|
||
|
|
DIRECTORY,
|
||
|
|
DTYPE,
|
||
|
|
OUTPUT_FILE,
|
||
|
|
TILE_SIZE,
|
||
|
|
OVERLAP,
|
||
|
|
KEEP,
|
||
|
|
ENDIAN,
|
||
|
|
FILLING_VALUE,
|
||
|
|
USE_LOCAL,
|
||
|
|
NO_FILLING,
|
||
|
|
PROCEED_IF_NO_SERVER
|
||
|
|
)
|
||
|
|
##
|
||
|
|
# Abstract method to create a filename based on lat and lon
|
||
|
|
# Given a latitude and longitude in degrees it returns the expected filename.
|
||
|
|
# @param lat \c int latitude in the range (-90,90).
|
||
|
|
# @param lon \c int longitude in the range [-180,180)
|
||
|
|
# @return \c string the filename for that location
|
||
|
|
@abc.abstractmethod
|
||
|
|
def createFilename(self,lat,lon):
|
||
|
|
pass
|
||
|
|
##
|
||
|
|
#Abstract method to create an image instance
|
||
|
|
#@return \c Image instance
|
||
|
|
@abc.abstractmethod
|
||
|
|
def createImage(self,lats,lons):
|
||
|
|
pass
|
||
|
|
|
||
|
|
## Convenience method to create a list of file names from bounding box
|
||
|
|
# which can be generated by the lat and lon.
|
||
|
|
# Given a rectangle (latitude,longitude) defined by a maximum and minimum latitude and by a maximum and minimum longitude (in degrees) it returns
|
||
|
|
# an ordered list of the filenames defining the rectangle. The list is ordered first in ascending longitudes and then ascending latitudes.
|
||
|
|
# @param lats \c list \c int list containing the minimum and maximum latitudes in the range (-90,90).
|
||
|
|
# @param lons \c list \c int list containing the minimum and maximum longitudes in the range [-180,180).
|
||
|
|
# @return \c tuple (\list strings the list of filenames covering the specified area, \c int the number of frames found along the longitude direction,
|
||
|
|
# \c int the number of frames found along the latitude direction)
|
||
|
|
#NOTE: createFilename needs to be implemented
|
||
|
|
def createNameListFromBounds(self,lats,lons):
|
||
|
|
self._inputFileList = []
|
||
|
|
|
||
|
|
lons = sorted(lons)
|
||
|
|
lats = sorted(lats)
|
||
|
|
lons[1] = int(np.ceil(lons[1]))
|
||
|
|
lons[0] = int(np.floor(lons[0]))
|
||
|
|
lats[1] = int(np.ceil(lats[1]))
|
||
|
|
lats[0] = int(np.floor(lats[0]))
|
||
|
|
#lats are from larger to smaller
|
||
|
|
latList = np.arange(lats[0],lats[1])[::-1]
|
||
|
|
lonList = np.arange(lons[0],lons[1])
|
||
|
|
# give error if crossing 180 and -180.
|
||
|
|
|
||
|
|
if(lons[1] - lons[0] < 180):
|
||
|
|
lonList = np.arange(lons[0],lons[1])
|
||
|
|
else:
|
||
|
|
print("Error. The crossing of E180 and W180 is not handled.")
|
||
|
|
raise Exception
|
||
|
|
for lat in latList:
|
||
|
|
for lon in lonList:
|
||
|
|
name = self.createFilename(lat,lon)
|
||
|
|
self._inputFileList.append(name)
|
||
|
|
return self._inputFileList,len(latList),len(lonList)
|
||
|
|
## Convenience method to create a list of file names from two lists of lats and lons.
|
||
|
|
# @param lats \c list \c int list containing the minimum and maximum latitudes in the range (-90,90).
|
||
|
|
# @param lons \c list \c int list containing the minimum and maximum longitudes in the range [-180,180).
|
||
|
|
# @return \c tuple (\list strings the list of filenames covering the specified area, \c int the number of frames found along the longitude direction,
|
||
|
|
# \c int the number of frames found along the latitude direction)
|
||
|
|
#NOTE: createFilename needs to be implemented
|
||
|
|
def createNameList(self,lats,lons):
|
||
|
|
return [self.createFilename(lat, lon) for lat,lon in zip(lats,lons)]
|
||
|
|
|
||
|
|
def configureStitcher(self,names,arrangement):
|
||
|
|
self._stitcher.configure()
|
||
|
|
self._stitcher.arrangement = arrangement
|
||
|
|
self._stitcher.tileSize = self._tileSize
|
||
|
|
self._stitcher.fileList = names
|
||
|
|
self._stitcher.dtype = self._dtype
|
||
|
|
self._stitcher.outputFile = self._outputFile
|
||
|
|
self._stitcher.endian = self._endian
|
||
|
|
self._stitcher.directory = self._downloadDir
|
||
|
|
self._stitcher._fillingValue = self._fillingValue
|
||
|
|
|
||
|
|
|
||
|
|
def configureRetriever(self):
|
||
|
|
self._retriever.configure()
|
||
|
|
self._retriever.url = self._url
|
||
|
|
self._retriever.pw = self._pw
|
||
|
|
self._retriever.un = self._un
|
||
|
|
self._retriever.downloadDir = self._downloadDir
|
||
|
|
self._retriever.proceedIfNoServer = self._proceedIfNoServer
|
||
|
|
|
||
|
|
def getFileList(self,names,report,map):
|
||
|
|
ret = []
|
||
|
|
for name in names:
|
||
|
|
if name in report and report[name] == self._retriever._succeded:
|
||
|
|
#the map returns a list of file that normally should have only
|
||
|
|
#one element
|
||
|
|
ret.append(map[name][0])
|
||
|
|
else:
|
||
|
|
ret.append(self._stitcher._toSkipName)
|
||
|
|
return ret
|
||
|
|
|
||
|
|
def stitch(self,lats,lons):
|
||
|
|
result = True
|
||
|
|
names,nlats,nlons = self.createNameListFromBounds(lats, lons)
|
||
|
|
self.configureStitcher(names, [nlats,nlons])
|
||
|
|
if not self._useLocal:
|
||
|
|
self.configureRetriever()
|
||
|
|
self._retriever.getFiles(names)
|
||
|
|
self._stitcher.fileList = self.getFileList(names,self._retriever._downloadReport,
|
||
|
|
self._retriever._namesMapping)
|
||
|
|
|
||
|
|
#the second part checks that everything was downloaded
|
||
|
|
if self._noFilling and self._stitcher._toSkipName in self._stitcher.fileList:
|
||
|
|
result = False
|
||
|
|
self.clean()
|
||
|
|
else:
|
||
|
|
self._stitcher.fileList = names
|
||
|
|
if result:
|
||
|
|
self._stitcher.stitch()
|
||
|
|
self.createXml(lats,lons)
|
||
|
|
if (not self._keep) and (not self._useLocal):
|
||
|
|
self.clean()
|
||
|
|
return result
|
||
|
|
|
||
|
|
def clean(self):
|
||
|
|
for name in self._stitcher.fileList:
|
||
|
|
if not name == self._stitcher._toSkipName:
|
||
|
|
os.remove(name)
|
||
|
|
def createXml(self,lats,lons):
|
||
|
|
image = self.createImage(lats,lons,self.outputFile)
|
||
|
|
self._image = image
|
||
|
|
image.dump(self.outputFile + '.xml')
|
||
|
|
|
||
|
|
def download(self,lats,lons,fromBounds=True):
|
||
|
|
if fromBounds:
|
||
|
|
names,nlats,nlons = self.createNameListFromBounds(lats,lons)
|
||
|
|
else:
|
||
|
|
names = self.createNameList(lats,lons)
|
||
|
|
self.configureRetriever()
|
||
|
|
self._retriever.getFiles(names)
|
||
|
|
|
||
|
|
@property
|
||
|
|
def proceedIfNoServer(self):
|
||
|
|
return self._proceedIfNoServer
|
||
|
|
@proceedIfNoServer.setter
|
||
|
|
def proceedIfNoServer(self,proceedIfNoServer):
|
||
|
|
self._proceedIfNoServer = proceedIfNoServer
|
||
|
|
@property
|
||
|
|
def url(self):
|
||
|
|
return self._url
|
||
|
|
@url.setter
|
||
|
|
def url(self,url):
|
||
|
|
self._url = url
|
||
|
|
@property
|
||
|
|
def un(self):
|
||
|
|
return self._un
|
||
|
|
@un.setter
|
||
|
|
def un(self,un):
|
||
|
|
self._un = un
|
||
|
|
@property
|
||
|
|
def pw(self):
|
||
|
|
return self._pw
|
||
|
|
@pw.setter
|
||
|
|
def pw(self,pw):
|
||
|
|
self._pw = pw
|
||
|
|
@property
|
||
|
|
def dtype(self):
|
||
|
|
return self._dtype
|
||
|
|
@dtype.setter
|
||
|
|
def dtype(self,val):
|
||
|
|
self._dtype = val
|
||
|
|
@property
|
||
|
|
def outputFile(self):
|
||
|
|
return self._outputFile
|
||
|
|
@outputFile.setter
|
||
|
|
def outputFile(self,val):
|
||
|
|
self._outputFile = val
|
||
|
|
@property
|
||
|
|
def tileSize(self):
|
||
|
|
return self._tileSize
|
||
|
|
@tileSize.setter
|
||
|
|
def tileSize(self,val):
|
||
|
|
self._tileSize = val
|
||
|
|
@property
|
||
|
|
def overlap(self):
|
||
|
|
return self._overlap
|
||
|
|
@overlap.setter
|
||
|
|
def overlap(self,val):
|
||
|
|
self._overlap = val
|
||
|
|
@property
|
||
|
|
def keep(self):
|
||
|
|
return self._keep
|
||
|
|
@keep.setter
|
||
|
|
def keep(self,val):
|
||
|
|
self._keep = val
|
||
|
|
@property
|
||
|
|
def endian(self):
|
||
|
|
return self._endian
|
||
|
|
@endian.setter
|
||
|
|
def endian(self,val):
|
||
|
|
self._endian = val
|
||
|
|
@property
|
||
|
|
def fillValue(self):
|
||
|
|
return self._fillValue
|
||
|
|
@fillValue.setter
|
||
|
|
def fillValue(self,val):
|
||
|
|
self._fillValue = val
|
||
|
|
@property
|
||
|
|
def useLocal(self):
|
||
|
|
return self._useLocal
|
||
|
|
@useLocal.setter
|
||
|
|
def useLocal(self,val):
|
||
|
|
self._useLocal = val
|
||
|
|
@property
|
||
|
|
def noFilling(self):
|
||
|
|
return self._noFilling
|
||
|
|
@noFilling.setter
|
||
|
|
def noFilling(self,val):
|
||
|
|
self._noFilling = val
|
||
|
|
@property
|
||
|
|
def image(self):
|
||
|
|
return self._image
|
||
|
|
@image.setter
|
||
|
|
def image(self,val):
|
||
|
|
self._image = val
|
||
|
|
##
|
||
|
|
# Setter function for the download directory.
|
||
|
|
# @param ddir \c string directory where the data are downloaded.
|
||
|
|
@property
|
||
|
|
def downloadDir(self):
|
||
|
|
return self._downloadDir
|
||
|
|
@downloadDir.setter
|
||
|
|
def downloadDir(self,ddir):
|
||
|
|
self._downloadDir = ddir
|
||
|
|
def __init__(self,family = '', name = ''):
|
||
|
|
#the .configure() methods are called in configureStitcher/Retriever
|
||
|
|
self._retriever = DR()
|
||
|
|
self._stitcher = ST()
|
||
|
|
self._image = None
|
||
|
|
|
||
|
|
super(TileManager, self).__init__(family if family else self.__class__.family, name=name)
|