From 7b51af405ddc3722881fd2e86bea77e5dedcec47 Mon Sep 17 00:00:00 2001 From: Lijun Zhu Date: Thu, 15 Apr 2021 00:40:16 -0700 Subject: [PATCH 01/19] cuda Scons fix: remove unnecessary -lcuda which prevents compiling on cluster head node without cuda drivers --- components/zerodop/GPUgeo2rdr/src/SConscript | 6 +++--- components/zerodop/GPUresampslc/src/SConscript | 2 +- components/zerodop/GPUtopozero/src/SConscript | 4 ++-- contrib/PyCuAmpcor/SConscript | 2 +- scons_tools/cuda.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/zerodop/GPUgeo2rdr/src/SConscript b/components/zerodop/GPUgeo2rdr/src/SConscript index 623955a..2375941 100644 --- a/components/zerodop/GPUgeo2rdr/src/SConscript +++ b/components/zerodop/GPUgeo2rdr/src/SConscript @@ -3,7 +3,7 @@ import os Import('envGPUgeo2rdr') -package = envGPUgeo2rdr['PACKAGE'] +package = envGPUgeo2rdr['PACKAGE'] project = envGPUgeo2rdr['PROJECT'] install = envGPUgeo2rdr['PRJ_SCONS_INSTALL'] + '/' + package + '/' + project build = envGPUgeo2rdr['PRJ_SCONS_BUILD'] + '/' + package + '/' + project + '/src' @@ -11,7 +11,7 @@ build = envGPUgeo2rdr['PRJ_SCONS_BUILD'] + '/' + package + '/' + project + '/src if envGPUgeo2rdr['GPU_ACC_ENABLED']: envGPUgeo2rdr.AppendUnique(CPPPATH=envGPUgeo2rdr['CUDACPPPATH']) envGPUgeo2rdr.AppendUnique(LIBPATH=envGPUgeo2rdr['CUDALIBPATH']) - envGPUgeo2rdr.AppendUnique(LIBS=['cuda','cudart']) + envGPUgeo2rdr.AppendUnique(LIBS=['cudart']) ###Custom cython builder @@ -24,7 +24,7 @@ def cythonPseudoBuilder(env,source,bld,inst): cppCode = env.Pyx2Cpp(source) listFiles = [source+'.cpp', 'Ellipsoid.cpp', 'Geo2rdr.cpp', 'GeoController.cpp', 'LinAlg.cpp', 'Orbit.cpp', 'Poly1d.cpp'] env.MergeFlags('-fopenmp -O3 -std=c++11 -fPIC -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -Wall -Wstrict-prototypes') - + if (env['GPU_ACC_ENABLED']): listFiles.append('GPUgeo.cu') lib = env.LoadableModule(target = 'GPUgeo2rdr.abi3.so', source = listFiles, CPPDEFINES = 'GPU_ACC_ENABLED') diff --git a/components/zerodop/GPUresampslc/src/SConscript b/components/zerodop/GPUresampslc/src/SConscript index eba18fd..a5046e7 100644 --- a/components/zerodop/GPUresampslc/src/SConscript +++ b/components/zerodop/GPUresampslc/src/SConscript @@ -11,7 +11,7 @@ build = envGPUresampslc['PRJ_SCONS_BUILD'] + '/' + package + '/' + project + '/s if envGPUresampslc['GPU_ACC_ENABLED']: envGPUresampslc.AppendUnique(CPPPATH=envGPUresampslc['CUDACPPPATH']) envGPUresampslc.AppendUnique(LIBPATH=envGPUresampslc['CUDALIBPATH']) - envGPUresampslc.AppendUnique(LIBS=['cuda','cudart']) + envGPUresampslc.AppendUnique(LIBS=['cudart']) ###Custom cython builder diff --git a/components/zerodop/GPUtopozero/src/SConscript b/components/zerodop/GPUtopozero/src/SConscript index b123502..6d4c088 100644 --- a/components/zerodop/GPUtopozero/src/SConscript +++ b/components/zerodop/GPUtopozero/src/SConscript @@ -3,7 +3,7 @@ import os Import('envGPUtopozero') -package = envGPUtopozero['PACKAGE'] +package = envGPUtopozero['PACKAGE'] project = envGPUtopozero['PROJECT'] install = envGPUtopozero['PRJ_SCONS_INSTALL'] + '/' + package + '/' + project build = envGPUtopozero['PRJ_SCONS_BUILD'] + '/' + package + '/' + project + '/src' @@ -11,7 +11,7 @@ build = envGPUtopozero['PRJ_SCONS_BUILD'] + '/' + package + '/' + project + '/sr if envGPUtopozero['GPU_ACC_ENABLED']: envGPUtopozero.AppendUnique(CPPPATH=envGPUtopozero['CUDACPPPATH']) envGPUtopozero.AppendUnique(LIBPATH=envGPUtopozero['CUDALIBPATH']) - envGPUtopozero.AppendUnique(LIBS=['cuda','cudart']) + envGPUtopozero.AppendUnique(LIBS=['cudart']) ###Custom cython builder cythonBuilder = Builder(action = 'cython3 $SOURCE --cplus', diff --git a/contrib/PyCuAmpcor/SConscript b/contrib/PyCuAmpcor/SConscript index 227a34f..0b232b3 100644 --- a/contrib/PyCuAmpcor/SConscript +++ b/contrib/PyCuAmpcor/SConscript @@ -14,7 +14,7 @@ Export('envPyCuAmpcor') if envPyCuAmpcor['GPU_ACC_ENABLED']: envPyCuAmpcor.Append(CPPPATH=envPyCuAmpcor['CUDACPPPATH']) envPyCuAmpcor.Append(LIBPATH=envPyCuAmpcor['CUDALIBPATH']) - envPyCuAmpcor.Append(LIBS=['cuda','cudart','cufft','cublas']) + envPyCuAmpcor.Append(LIBS=['cudart','cufft','cublas']) build = envPyCuAmpcor['PRJ_SCONS_BUILD'] + '/' + package + '/' + project # includeScons = os.path.join('include','SConscript') diff --git a/scons_tools/cuda.py b/scons_tools/cuda.py index 4edaf12..19b0bba 100644 --- a/scons_tools/cuda.py +++ b/scons_tools/cuda.py @@ -52,7 +52,7 @@ def generate(env): # default flags for the NVCC compiler env['STATICNVCCFLAGS'] = '' env['SHAREDNVCCFLAGS'] = '' - env['ENABLESHAREDNVCCFLAG'] = '-std=c++11 -shared -Xcompiler -fPIC -I/opt/conda/include' + env['ENABLESHAREDNVCCFLAG'] = '-std=c++11 -shared -Xcompiler -fPIC' # default NVCC commands env['STATICNVCCCMD'] = '$NVCC -o $TARGET -c $NVCCFLAGS $STATICNVCCFLAGS $SOURCES' From e712ca358c42acddd2e9deb55f7d64869abab98e Mon Sep 17 00:00:00 2001 From: Zhang Yunjun Date: Wed, 17 Mar 2021 22:48:28 -0700 Subject: [PATCH 02/19] stackStripMap: indentation adjusts following PEP008 --- contrib/stack/stripmapStack/stackStripMap.py | 202 ++++++++++--------- 1 file changed, 102 insertions(+), 100 deletions(-) diff --git a/contrib/stack/stripmapStack/stackStripMap.py b/contrib/stack/stripmapStack/stackStripMap.py index 01a3417..62dac63 100755 --- a/contrib/stack/stripmapStack/stackStripMap.py +++ b/contrib/stack/stripmapStack/stackStripMap.py @@ -26,6 +26,7 @@ noMCF = 'False' defoMax = '2' maxNodes = 72 + def createParser(): parser = argparse.ArgumentParser( description='Preparing the directory structure and config files for stack processing of StripMap data') @@ -42,7 +43,7 @@ def createParser(): parser.add_argument('-m', '--reference_date', dest='referenceDate', type=str, default=None, help='Directory with reference acquisition') - + parser.add_argument('-t', '--time_threshold', dest='dtThr', type=float, default=10000.0, help='Time threshold (max temporal baseline in days)') @@ -57,7 +58,7 @@ def createParser(): parser.add_argument('-S', '--sensor', dest='sensor', type=str, required=False, help='SAR sensor used to define square multi-look pixels') - parser.add_argument('-u', '--unw_method', dest='unwMethod', type=str, default='snaphu', + parser.add_argument('-u', '--unw_method', dest='unwMethod', type=str, default='snaphu', help='unwrapping method (icu, snaphu, or snaphu2stage), no to skip phase unwrapping.') parser.add_argument('-f','--filter_strength', dest='filtStrength', type=str, default=filtStrength, @@ -72,7 +73,7 @@ def createParser(): iono.add_argument('-B', '--subband_bandwidth ', dest='bandWidth', type=str, default=None, help='sub-band band width') - iono.add_argument('--filter_sigma_x', dest='filterSigmaX', type=str, default='100', + iono.add_argument('--filter_sigma_x', dest='filterSigmaX', type=str, default='100', help='filter sigma for gaussian filtering the dispersive and nonDispersive phase') iono.add_argument('--filter_sigma_y', dest='filterSigmaY', type=str, default='100.0', @@ -87,14 +88,14 @@ def createParser(): iono.add_argument('--filter_kernel_rotation', dest='filterKernelRotation', type=str, default='0.0', help='rotation angle of the filter kernel in degrees (default = 0.0)') - parser.add_argument('-W', '--workflow', dest='workflow', type=str, default='slc', + parser.add_argument('-W', '--workflow', dest='workflow', type=str, default='slc', help='The InSAR processing workflow : (slc, interferogram, ionosphere)') - parser.add_argument('-z', '--zero', dest='zerodop', action='store_true', default=False, + parser.add_argument('-z', '--zero', dest='zerodop', action='store_true', default=False, help='Use zero doppler geometry for processing - Default : No') - parser.add_argument('--nofocus', dest='nofocus', action='store_true', default=False, + parser.add_argument('--nofocus', dest='nofocus', action='store_true', default=False, help='If input data is already focused to SLCs - Default : do focus') - parser.add_argument('-c', '--text_cmd', dest='text_cmd', type=str, default='', + parser.add_argument('-c', '--text_cmd', dest='text_cmd', type=str, default='', help='text command to be added to the beginning of each line of the run files. Example : source ~/.bash_profile;') parser.add_argument('-useGPU', '--useGPU', dest='useGPU',action='store_true', default=False, help='Allow App to use GPU when available') @@ -102,18 +103,19 @@ def createParser(): parser.add_argument('--summary', dest='summary', action='store_true', default=False, help='Show summary only') return parser + def cmdLineParse(iargs = None): parser = createParser() inps = parser.parse_args(args=iargs) inps.slcDir = os.path.abspath(inps.slcDir) inps.workDir = os.path.abspath(inps.workDir) inps.dem = os.path.abspath(inps.dem) - + return inps - + def get_dates(inps): - + dirs = glob.glob(inps.slcDir+'/*') acuisitionDates = [] for dirf in dirs: @@ -134,13 +136,13 @@ def get_dates(inps): inps.referenceDate = acuisitionDates[0] secondaryDates = acuisitionDates.copy() secondaryDates.remove(inps.referenceDate) - return acuisitionDates, inps.referenceDate, secondaryDates - + return acuisitionDates, inps.referenceDate, secondaryDates + + def slcStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs, splitFlag=False, rubberSheet=False): # A coregistered stack of SLCs i=0 - if inps.bbox: i+=1 runObj = run() @@ -149,7 +151,6 @@ def slcStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs, runObj.crop(acquisitionDates, config_prefix, native=not inps.zerodop, israw=not inps.nofocus) runObj.finalize() - i+=1 runObj = run() runObj.configure(inps, 'run_{:02d}_reference'.format(i)) @@ -190,40 +191,40 @@ def slcStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs, config_prefix = 'config_fineResamp_' runObj.secondarys_fine_resampleSlc(stackReferenceDate, secondaryDates, config_prefix, split=splitFlag) runObj.finalize() - + if rubberSheet: - i+=1 - runObj = run() - runObj.configure(inps, 'run_{:02d}_denseOffset'.format(i)) - config_prefix = 'config_denseOffset_' - runObj.denseOffsets_Network(pairs, stackReferenceDate, secondaryDates, config_prefix) - runObj.finalize() + i+=1 + runObj = run() + runObj.configure(inps, 'run_{:02d}_denseOffset'.format(i)) + config_prefix = 'config_denseOffset_' + runObj.denseOffsets_Network(pairs, stackReferenceDate, secondaryDates, config_prefix) + runObj.finalize() - i+=1 - runObj = run() - runObj.configure(inps, 'run_{:02d}_invertDenseOffsets'.format(i)) - runObj.invertDenseOffsets() - runObj.finalize() - - i+=1 - runObj = run() - runObj.configure(inps, 'run_{:02d}_resampleOffset'.format(i)) - config_prefix = 'config_resampOffsets_' - runObj.resampleOffset(secondaryDates, config_prefix) - runObj.finalize() + i+=1 + runObj = run() + runObj.configure(inps, 'run_{:02d}_invertDenseOffsets'.format(i)) + runObj.invertDenseOffsets() + runObj.finalize() - i+=1 - runObj = run() - runObj.configure(inps, 'run_{:02d}_replaceOffsets'.format(i)) - runObj.replaceOffsets(secondaryDates) - runObj.finalize() + i+=1 + runObj = run() + runObj.configure(inps, 'run_{:02d}_resampleOffset'.format(i)) + config_prefix = 'config_resampOffsets_' + runObj.resampleOffset(secondaryDates, config_prefix) + runObj.finalize() - i+=1 - runObj = run() - runObj.configure(inps, 'run_{:02d}_fineResamp'.format(i)) - config_prefix = 'config_fineResamp_' - runObj.secondarys_fine_resampleSlc(stackReferenceDate, secondaryDates, config_prefix, split=splitFlag) - runObj.finalize() + i+=1 + runObj = run() + runObj.configure(inps, 'run_{:02d}_replaceOffsets'.format(i)) + runObj.replaceOffsets(secondaryDates) + runObj.finalize() + + i+=1 + runObj = run() + runObj.configure(inps, 'run_{:02d}_fineResamp'.format(i)) + config_prefix = 'config_fineResamp_' + runObj.secondarys_fine_resampleSlc(stackReferenceDate, secondaryDates, config_prefix, split=splitFlag) + runObj.finalize() # adding the baseline grid generation i+=1 @@ -235,20 +236,22 @@ def slcStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs, return i + def interferogramStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs): - # an interferogram stack without ionosphere correction. + # an interferogram stack without ionosphere correction. # coregistration is with geometry + const offset - i = slcStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs, splitFlag=False, rubberSheet=False) - + i+=1 runObj = run() runObj.configure(inps, 'run_{:02d}_igram'.format(i)) config_prefix = 'config_igram_' low_or_high = "/" - runObj.igrams_network(pairs, acquisitionDates, stackReferenceDate, low_or_high, config_prefix) + runObj.igrams_network(pairs, acquisitionDates, stackReferenceDate, low_or_high, config_prefix) runObj.finalize() + return + def interferogramIonoStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs): @@ -273,7 +276,7 @@ def interferogramIonoStack(inps, acquisitionDates, stackReferenceDate, secondary config_prefix = 'config_igram_' low_or_high = "/" runObj.igrams_network(pairs, acquisitionDates, stackReferenceDate, low_or_high, config_prefix) - runObj.finalize() + runObj.finalize() i+=1 runObj = run() @@ -297,69 +300,68 @@ def interferogramIonoStack(inps, acquisitionDates, stackReferenceDate, secondary config_prefix = 'config_iono_' lowBand = '/LowBand/' highBand = '/HighBand/' - runObj.dispersive_nonDispersive(pairs, acquisitionDates, stackReferenceDate, - lowBand, highBand, config_prefix) + runObj.dispersive_nonDispersive(pairs, acquisitionDates, stackReferenceDate, lowBand, highBand, config_prefix) runObj.finalize() + return + def main(iargs=None): - inps = cmdLineParse(iargs) - # name of the folder of the coreg SLCs including baselines, SLC, geom_reference subfolders - inps.stack_folder = 'merged' - inps.dense_offsets_folder = 'dense_offsets' + inps = cmdLineParse(iargs) + # name of the folder of the coreg SLCs including baselines, SLC, geom_reference subfolders + inps.stack_folder = 'merged' + inps.dense_offsets_folder = 'dense_offsets' - # check if a sensor is defined and update if needed azimuth looks to give square pixels - ar=1 - if inps.sensor: - if inps.sensor.lower() == "alos": - ar=4 - print("Looks like " + inps.sensor.lower() + ", multi-look AR=" + str(ar)) - elif inps.sensor.lower() == "envisat" or inps.sensor.lower() == "ers": - ar=5 - print("Looks like " + inps.sensor.lower() + ", multi-look AR=" + str(ar)) - else: - print("Sensor is not hard-coded (ers, envisat, alos), will keep default alks") - # sensor is not recognised, report to user and state default - inps.alks = str(int(inps.alks)*int(ar)) - - # getting the acquisitions - acquisitionDates, stackReferenceDate, secondaryDates = get_dates(inps) - configDir = os.path.join(inps.workDir,'configs') - os.makedirs(configDir, exist_ok=True) - runDir = os.path.join(inps.workDir,'run_files') - os.makedirs(runDir, exist_ok=True) + # check if a sensor is defined and update if needed azimuth looks to give square pixels + ar=1 + if inps.sensor: + if inps.sensor.lower() == "alos": + ar=4 + print("Looks like " + inps.sensor.lower() + ", multi-look AR=" + str(ar)) + elif inps.sensor.lower() == "envisat" or inps.sensor.lower() == "ers": + ar=5 + print("Looks like " + inps.sensor.lower() + ", multi-look AR=" + str(ar)) + else: + print("Sensor is not hard-coded (ers, envisat, alos), will keep default alks") + # sensor is not recognised, report to user and state default + inps.alks = str(int(inps.alks)*int(ar)) - if inps.sensor.lower() == 'uavsar_stack': # don't try to calculate baselines for UAVSAR_STACK data - pairs = selectPairs(inps,stackReferenceDate, secondaryDates, acquisitionDates,doBaselines=False) - else: - pairs = selectPairs(inps,stackReferenceDate, secondaryDates, acquisitionDates,doBaselines=True) - print ('number of pairs: ', len(pairs)) + # getting the acquisitions + acquisitionDates, stackReferenceDate, secondaryDates = get_dates(inps) + configDir = os.path.join(inps.workDir,'configs') + os.makedirs(configDir, exist_ok=True) + runDir = os.path.join(inps.workDir,'run_files') + os.makedirs(runDir, exist_ok=True) - ###If only a summary is requested quit after this - if inps.summary: - return + if inps.sensor.lower() == 'uavsar_stack': # don't try to calculate baselines for UAVSAR_STACK data + pairs = selectPairs(inps,stackReferenceDate, secondaryDates, acquisitionDates,doBaselines=False) + else: + pairs = selectPairs(inps,stackReferenceDate, secondaryDates, acquisitionDates,doBaselines=True) + print ('number of pairs: ', len(pairs)) - #if cropping is requested, then change the slc directory: - inps.fullFrameSlcDir = inps.slcDir + ###If only a summary is requested quit after this + if inps.summary: + return - if inps.bbox: - inps.slcDir = inps.slcDir + "_crop" - ############################# + #if cropping is requested, then change the slc directory: + inps.fullFrameSlcDir = inps.slcDir - if inps.workflow == 'slc': - slcStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs, splitFlag=False, rubberSheet=False) + if inps.bbox: + inps.slcDir = inps.slcDir + "_crop" + ############################# - elif inps.workflow == 'interferogram': - interferogramStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs) + if inps.workflow == 'slc': + slcStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs, splitFlag=False, rubberSheet=False) + + elif inps.workflow == 'interferogram': + interferogramStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs) + + elif inps.workflow == 'ionosphere': + interferogramIonoStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs) + return - elif inps.workflow == 'ionosphere': - interferogramIonoStack(inps, acquisitionDates, stackReferenceDate, secondaryDates, pairs) - if __name__ == "__main__": - - # Main engine - main() - - + # Main engine + main(sys.argv[1:]) From ef88f247e2eb945f3eba6953bd8e68662269e286 Mon Sep 17 00:00:00 2001 From: yunjunz Date: Wed, 17 Mar 2021 22:53:51 -0700 Subject: [PATCH 03/19] stackStripMap: bugfix when --sensor is not specified --- contrib/stack/stripmapStack/stackStripMap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/stack/stripmapStack/stackStripMap.py b/contrib/stack/stripmapStack/stackStripMap.py index 62dac63..09749e4 100755 --- a/contrib/stack/stripmapStack/stackStripMap.py +++ b/contrib/stack/stripmapStack/stackStripMap.py @@ -334,7 +334,7 @@ def main(iargs=None): runDir = os.path.join(inps.workDir,'run_files') os.makedirs(runDir, exist_ok=True) - if inps.sensor.lower() == 'uavsar_stack': # don't try to calculate baselines for UAVSAR_STACK data + if inps.sensor and inps.sensor.lower() == 'uavsar_stack': # don't try to calculate baselines for UAVSAR_STACK data pairs = selectPairs(inps,stackReferenceDate, secondaryDates, acquisitionDates,doBaselines=False) else: pairs = selectPairs(inps,stackReferenceDate, secondaryDates, acquisitionDates,doBaselines=True) From a5891a8ac053979d594ec635eb1f408a769c6329 Mon Sep 17 00:00:00 2001 From: yunjunz Date: Fri, 2 Apr 2021 17:39:25 -0700 Subject: [PATCH 04/19] cuDenseOff*.prep*Geom*: supp. full-reso file w/o .full in the name + support full resolution geometry files without ".full" in the filename, such as provided by stripmapStack + add --fix-xml option to update the image file path in the xml file, so that IML.loadImage is not called all the time, which could lead to file access issue when multiple processes use the same SLC file --- contrib/PyCuAmpcor/examples/cuDenseOffsets.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/contrib/PyCuAmpcor/examples/cuDenseOffsets.py b/contrib/PyCuAmpcor/examples/cuDenseOffsets.py index 56b41d4..c1d374d 100755 --- a/contrib/PyCuAmpcor/examples/cuDenseOffsets.py +++ b/contrib/PyCuAmpcor/examples/cuDenseOffsets.py @@ -40,6 +40,8 @@ def createParser(): help='Reference image') parser.add_argument('-s', '--secondary',type=str, dest='secondary', required=True, help='Secondary image') + parser.add_argument('--fix-xml','--fix-image-xml', dest='fixImageXml', action='store_true', + help='Fix the image file path in the XML file. Enable this if input files havee been moved.') parser.add_argument('--op','--outprefix','--output-prefix', type=str, dest='outprefix', default='offset', required=True, @@ -166,12 +168,13 @@ def estimateOffsetField(reference, secondary, inps=None): return 0 # update file path in xml file - for fname in [reference, secondary]: - fname = os.path.abspath(fname) - img = IML.loadImage(fname)[0] - img.filename = fname - img.setAccessMode('READ') - img.renderHdr() + if inps.fixImageXml: + for fname in [reference, secondary]: + fname = os.path.abspath(fname) + img = IML.loadImage(fname)[0] + img.filename = fname + img.setAccessMode('READ') + img.renderHdr() ###Loading the secondary image object sim = isceobj.createSlcImage() @@ -382,14 +385,24 @@ def prepareGeometry(full_dir, out_dir, x_start, y_start, x_step, y_step, num_win x/y_step - int, output pixel step in column/row direction num_win_x/y - int, number of columns/rows """ + full_dir = os.path.abspath(full_dir) + out_dir = os.path.abspath(out_dir) + + # grab the file extension for full resolution file + full_exts = ['.rdr.full','.rdr'] if full_dir != out_dir else ['.rdr.full'] + full_exts = [e for e in full_exts if os.path.isfile(os.path.join(full_dir, '{f}{e}'.format(f=fbases[0], e=e)))] + if len(full_exts) == 0: + raise ValueError('No full resolution {}.rdr* file found in: {}'.format(fbases[0], full_dir)) + full_ext = full_exts[0] + print('-'*50) print('generate the corresponding multi-looked geometry datasets using gdal ...') - in_files = [os.path.join(full_dir, '{}.rdr.full'.format(i)) for i in fbases] + # input files + in_files = [os.path.join(full_dir, '{f}{e}'.format(f=f, e=full_ext)) for f in fbases] in_files = [i for i in in_files if os.path.isfile(i)] - if len(in_files) == 0: - raise ValueError('No full resolution geometry file found in: {}'.format(full_dir)) - fbases = [os.path.basename(i).split('.')[0] for i in in_files] + + # output files out_files = [os.path.join(out_dir, '{}.rdr'.format(i)) for i in fbases] os.makedirs(out_dir, exist_ok=True) From e499aec0558322fff2155cd352f7f60b8537982c Mon Sep 17 00:00:00 2001 From: Russell Grew Date: Mon, 3 May 2021 13:03:11 +1000 Subject: [PATCH 05/19] Swap lat/lon for proj6 and gdal3. --- contrib/stack/topsStack/plotBursts.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/stack/topsStack/plotBursts.py b/contrib/stack/topsStack/plotBursts.py index 5fae2e6..6ebb961 100755 --- a/contrib/stack/topsStack/plotBursts.py +++ b/contrib/stack/topsStack/plotBursts.py @@ -138,11 +138,11 @@ def main(iargs=None): lateNear = burst.orbit.rdr2geo(t1,r0) ring = ogr.Geometry(ogr.wkbLinearRing) - ring.AddPoint(earlyNear[1], earlyNear[0]) - ring.AddPoint(earlyFar[1], earlyFar[0]) - ring.AddPoint(lateFar[1], lateFar[0]) - ring.AddPoint(lateNear[1], lateNear[0]) - ring.AddPoint(earlyNear[1], earlyNear[0]) + ring.AddPoint(earlyNear[0], earlyNear[1]) + ring.AddPoint(earlyFar[0], earlyFar[1]) + ring.AddPoint(lateFar[0], lateFar[1]) + ring.AddPoint(lateNear[0], lateNear[1]) + ring.AddPoint(earlyNear[0], earlyNear[1]) feature = ogr.Feature(layer.GetLayerDefn()) feature.SetField('Name', 'IW{0}-{1}'.format(swath, ii)) From 2f8de43d280687c8bb0f349b8b3a96fcad51edf6 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Tue, 4 May 2021 13:06:54 +0800 Subject: [PATCH 06/19] Changes on the fetchOrbit.py to download orbit files from the new ESA website (#273) * Changes to fetchOrbit.py to download orbit files in the new ESA website I've made changes to the fetchOrbit.py to download the files on the https://scihub.copernicus.eu/gnss/#/home. I've used the generic credentials to query and download the orbit files. * Update fetchOrbit.py * Make output path generic I've used os.path.join on line no. 165 to make it more generic. * Set the verify option to true I've set the verify option to True to remove the warning regarding "Unverified HTTPS request". * Changes to the download link of the fetchOrbit.py The download link before uses the href link on the XML of the website. It was properly working right until yesterday when it didn't work properly. Instead I've edited the script to be consistent instead with the download link on the scihub copernicus GNSS website. I've tested it and it worked properly. * Update fetchOrbit.py --- contrib/stack/topsStack/fetchOrbit.py | 99 +++++++++++---------------- 1 file changed, 41 insertions(+), 58 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 7e7192f..fcd69db 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -1,22 +1,25 @@ #!/usr/bin/env python3 import numpy as np -import re import requests +import re import os import argparse import datetime from html.parser import HTMLParser -server = 'http://aux.sentinel1.eo.esa.int/' +server = 'https://scihub.copernicus.eu/gnss/' -orbitMap = [('precise', 'POEORB/'), - ('restituted', 'RESORB/')] +orbitMap = [('precise', 'AUX_POEORB'), + ('restituted', 'AUX_RESORB')] datefmt = "%Y%m%dT%H%M%S" queryfmt = "%Y-%m-%d" queryfmt2 = "%Y/%m/%d/" +#Generic credentials to query and download orbit files +credentials = ('gnssguest', 'gnssguest') + def cmdLineParse(): ''' @@ -55,38 +58,26 @@ def FileToTimeStamp(safename): class MyHTMLParser(HTMLParser): - def __init__(self, satName, url): + def __init__(self,url): HTMLParser.__init__(self) self.fileList = [] - self.in_td = False - self.in_a = False - self.in_table = False self._url = url - self.satName = satName - + def handle_starttag(self, tag, attrs): - if tag == 'td': - self.in_td = True - elif tag == 'a': - self.in_a = True - for name, val in attrs: - if name == "href": - if val.startswith("http"): - self._url = val.strip() - + for name, val in attrs: + if name == 'href': + if val.startswith("https://scihub.copernicus.eu/gnss/odata") and val.endswith(")/"): + pass + else: + downloadLink = val.strip() + downloadLink = downloadLink.split("/Products('Quicklook')") + downloadLink = downloadLink[0] + downloadLink[-1] + self._url = downloadLink + def handle_data(self, data): - if self.in_td and self.in_a: - if self.satName in data: - self.fileList.append((self._url, data.strip())) - - def handle_tag(self, tag): - if tag == 'td': - self.in_td = False - self.in_a = False - elif tag == 'a': - self.in_a = False - self._url = None - + if data.startswith("S1") and data.endswith(".EOF"): + self.fileList.append((self._url, data.strip())) + def download_file(url, outdir='.', session=None): ''' @@ -96,9 +87,9 @@ def download_file(url, outdir='.', session=None): if session is None: session = requests.session() - path = os.path.join(outdir, os.path.basename(url)) + path = outdir print('Downloading URL: ', url) - request = session.get(url, stream=True, verify=False) + request = session.get(url, stream=True, verify=True, auth=credentials) try: val = request.raise_for_status() @@ -139,37 +130,29 @@ if __name__ == '__main__': fileTS, satName, fileTSStart = FileToTimeStamp(inps.input) print('Reference time: ', fileTS) print('Satellite name: ', satName) - match = None session = requests.Session() for spec in orbitMap: oType = spec[0] - - if oType == 'precise': - end_date = fileTS + datetime.timedelta(days=20) - elif oType == 'restituted': - end_date = fileTS - else: - raise ValueError("Unexpected orbit type: '" + oType + "'") - end_date2 = end_date + datetime.timedelta(days=1) - urls = (server + spec[1] + end_date.strftime("%Y/%m/%d/") for end_date in (end_date, end_date2)) - + delta = datetime.timedelta(days=1) + timebef = (fileTS - delta).strftime(queryfmt) + timeaft = (fileTS + delta).strftime(queryfmt) + url = server + 'search?q=( beginPosition:[{0}T00:00:00.000Z TO {1}T23:59:59.999Z] AND endPosition:[{0}T00:00:00.000Z TO {1}T23:59:59.999Z] ) AND ( (platformname:Sentinel-1 AND filename:{2}_* AND producttype:{3}))&start=0&rows=100'.format(timebef,timeaft, satName,spec[1]) + success = False match = None - + try: - - for url in urls: - r = session.get(url, verify=False) - r.raise_for_status() - parser = MyHTMLParser(satName, url) - parser.feed(r.text) - - for resulturl, result in parser.fileList: - tbef, taft, mission = fileToRange(os.path.basename(result)) - if (tbef <= fileTSStart) and (taft >= fileTS): - match = os.path.join(resulturl, result) + r = session.get(url, verify=True, auth=credentials) + r.raise_for_status() + parser = MyHTMLParser(url) + parser.feed(r.text) + for resulturl, result in parser.fileList: + tbef, taft, mission = fileToRange(os.path.basename(result)) + if (tbef <= fileTSStart) and (taft >= fileTS): + matchFileName = result + match = os.path.join(server[0:-5],resulturl[36:]) if match is not None: success = True @@ -180,8 +163,8 @@ if __name__ == '__main__': break if match is not None: - - res = download_file(match, inps.outdir, session) + output = os.path.join(inps.outdir, matchFileName) + res = download_file(match, output, session) if res is False: print('Failed to download URL: ', match) else: From 8ba8f32f672fbe30cacd8c6600aa40560c41b8bf Mon Sep 17 00:00:00 2001 From: Russell Grew Date: Fri, 7 May 2021 08:39:13 +1000 Subject: [PATCH 07/19] gdal version smarts for lat/lon order --- contrib/stack/topsStack/plotBursts.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/contrib/stack/topsStack/plotBursts.py b/contrib/stack/topsStack/plotBursts.py index 6ebb961..b54da34 100755 --- a/contrib/stack/topsStack/plotBursts.py +++ b/contrib/stack/topsStack/plotBursts.py @@ -48,7 +48,7 @@ def main(iargs=None): inps=cmdLineParse(iargs) - from osgeo import ogr, osr + from osgeo import gdal, ogr, osr import matplotlib if inps.shapefile is not None: matplotlib.use('Agg') @@ -138,11 +138,18 @@ def main(iargs=None): lateNear = burst.orbit.rdr2geo(t1,r0) ring = ogr.Geometry(ogr.wkbLinearRing) - ring.AddPoint(earlyNear[0], earlyNear[1]) - ring.AddPoint(earlyFar[0], earlyFar[1]) - ring.AddPoint(lateFar[0], lateFar[1]) - ring.AddPoint(lateNear[0], lateNear[1]) - ring.AddPoint(earlyNear[0], earlyNear[1]) + if int(gdal.__version__[0]) >= 3: + ring.AddPoint(earlyNear[0], earlyNear[1]) + ring.AddPoint(earlyFar[0], earlyFar[1]) + ring.AddPoint(lateFar[0], lateFar[1]) + ring.AddPoint(lateNear[0], lateNear[1]) + ring.AddPoint(earlyNear[0], earlyNear[1]) + else: + ring.AddPoint(earlyNear[1], earlyNear[0]) + ring.AddPoint(earlyFar[1], earlyFar[0]) + ring.AddPoint(lateFar[1], lateFar[0]) + ring.AddPoint(lateNear[1], lateNear[0]) + ring.AddPoint(earlyNear[1], earlyNear[0]) feature = ogr.Feature(layer.GetLayerDefn()) feature.SetField('Name', 'IW{0}-{1}'.format(swath, ii)) From f48e71d138027459b78bb56ea0b74dbcf6d98e19 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Fri, 7 May 2021 15:57:36 +0800 Subject: [PATCH 08/19] Changes to line 155 of fetchOrbit.py I've made some changes to fetchOrbit.py due to changes made by ESA based on the issue presented in https://github.com/isce-framework/isce2/pull/273#issuecomment-833967310. --- contrib/stack/topsStack/fetchOrbit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index fcd69db..fc69ceb 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -152,7 +152,7 @@ if __name__ == '__main__': tbef, taft, mission = fileToRange(os.path.basename(result)) if (tbef <= fileTSStart) and (taft >= fileTS): matchFileName = result - match = os.path.join(server[0:-5],resulturl[36:]) + match = resulturl if match is not None: success = True From 21084da4dfd8f71511998c94a0c8e884eb97124b Mon Sep 17 00:00:00 2001 From: Russell Grew <42530666+RussellGrew@users.noreply.github.com> Date: Tue, 11 May 2021 16:00:12 +1000 Subject: [PATCH 09/19] Robust GDAL version check Co-authored-by: Ryan Burns --- contrib/stack/topsStack/plotBursts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/stack/topsStack/plotBursts.py b/contrib/stack/topsStack/plotBursts.py index b54da34..1187d4b 100755 --- a/contrib/stack/topsStack/plotBursts.py +++ b/contrib/stack/topsStack/plotBursts.py @@ -138,7 +138,8 @@ def main(iargs=None): lateNear = burst.orbit.rdr2geo(t1,r0) ring = ogr.Geometry(ogr.wkbLinearRing) - if int(gdal.__version__[0]) >= 3: + from distutils.version import StrictVersion + if StrictVersion(gdal.__version__) >= StrictVersion("3.0"): ring.AddPoint(earlyNear[0], earlyNear[1]) ring.AddPoint(earlyFar[0], earlyFar[1]) ring.AddPoint(lateFar[0], lateFar[1]) From a540c0116e64ab271272e8764a7e7653243adea8 Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Thu, 13 May 2021 10:59:16 -0700 Subject: [PATCH 10/19] Remove use of deprecated XML ElementTree getchildren method --- components/isceobj/Catalog/Catalog.py | 2 +- components/isceobj/Sensor/CEOS.py | 8 +- components/isceobj/Sensor/GRD/Radarsat2.py | 16 +-- components/isceobj/Sensor/GRD/Sentinel1.py | 14 +- components/isceobj/Sensor/GRD/Terrasarx.py | 16 +-- components/isceobj/Sensor/Radarsat2.py | 48 +++---- components/isceobj/Sensor/Sentinel1.py | 6 +- components/isceobj/Sensor/TOPS/Sentinel1.py | 14 +- components/isceobj/Sensor/TanDEMX.py | 147 ++++++++++---------- components/isceobj/Sensor/TerraSARX.py | 147 ++++++++++---------- components/isceobj/XmlUtil/XmlUtil.py | 2 +- components/isceobj/XmlUtil/xmlUtils.py | 2 +- contrib/stack/stripmapStack/prepSlcRSAT2.py | 2 +- 13 files changed, 211 insertions(+), 213 deletions(-) diff --git a/components/isceobj/Catalog/Catalog.py b/components/isceobj/Catalog/Catalog.py index 00a8d0f..d847b0d 100755 --- a/components/isceobj/Catalog/Catalog.py +++ b/components/isceobj/Catalog/Catalog.py @@ -246,7 +246,7 @@ from collections import UserDict def dict_to_xml(adict,file,nodeTag=None,elementTag=None): a = ET.Element(nodeTag) # something to hang nodes on a = dict_to_et(a,adict,nodeTag,elementTag) - et = a.getchildren()[0] + et = list(a)[0] indent(et) tree = ET.ElementTree(et) tree.write(file) diff --git a/components/isceobj/Sensor/CEOS.py b/components/isceobj/Sensor/CEOS.py index b1f2223..94efac7 100755 --- a/components/isceobj/Sensor/CEOS.py +++ b/components/isceobj/Sensor/CEOS.py @@ -44,7 +44,7 @@ class CEOSDB(object): self.metadata = {} if not xml == None: self.xmlFP = open(self.xml, 'r') - self.rootChildren = ET(file=self.xmlFP).getroot().getchildren() + self.rootChildren = list(ET(file=self.xmlFP).getroot()) else: self.xmlFP = None self.rootChildren = [] @@ -81,7 +81,7 @@ class CEOSDB(object): self.metadata[key] = [None]*loopCount for i in range(loopCount): struct = {} - for node in z.getchildren(): + for node in z: (subkey,data) = self.decodeNode(node) struct[subkey] = data self.metadata[key][i] = struct @@ -98,7 +98,7 @@ class CEOSDB(object): xmlFP = open(self.xml, 'r') self.root = ET(file=xmlFP).getroot() - for z in self.root.getchildren(): + for z in self.root: # If the tag name is 'rec', this is a plain old record if z.tag == 'rec': (key,data) = self.decodeNode(z) @@ -116,7 +116,7 @@ class CEOSDB(object): self.metadata[key] = [None]*loopCount for i in range(loopCount): struct = {} - for node in z.getchildren(): + for node in z: (subkey,data) = self.decodeNode(node) struct[subkey] = data self.metadata[key][i] = struct diff --git a/components/isceobj/Sensor/GRD/Radarsat2.py b/components/isceobj/Sensor/GRD/Radarsat2.py index 84d596e..be0f43c 100755 --- a/components/isceobj/Sensor/GRD/Radarsat2.py +++ b/components/isceobj/Sensor/GRD/Radarsat2.py @@ -276,8 +276,8 @@ class Radarsat2_GRD(Component): glist = (self.getxmlelement('imageAttributes/geographicInformation/geolocationGrid')) lat = [] lon = [] - for child in glist.getchildren(): - for grandchild in child.getchildren(): + for child in glist: + for grandchild in child: string = ElementTree.tostring(grandchild, encoding = 'unicode', method = 'xml') string = string.split("<")[1] string = string.split(">")[0] @@ -303,7 +303,7 @@ class Radarsat2_GRD(Component): if node.tag == 'stateVector': sv = StateVector() sv.configure() - for z in node.getchildren(): + for z in node: if z.tag == 'timeStamp': timeStamp = self.convertToDateTime(z.text) elif z.tag == 'xPosition': @@ -346,16 +346,16 @@ class Radarsat2_GRD(Component): if not node.tag == 'imageTiePoint': continue - for z in node.getchildren(): + for z in node: if z.tag == 'imageCoordinate': - for zz in z.getchildren(): + for zz in z: if zz.tag == 'line': line = float(zz.text) elif zz.tag == 'pixel': pixel = float(zz.text) if z.tag == 'geodeticCoordinate': - for zz in z.getchildren(): + for zz in z: if zz.tag == 'latitude': lat = float(zz.text) elif zz.tag == 'longitude': @@ -424,8 +424,8 @@ class Radarsat2_GRD(Component): lines = [] data = [] - for child in node.getchildren(): - for child in node.getchildren(): + for child in node: + for child in node: string = ElementTree.tostring(child, encoding = 'unicode', method = 'xml') string = string.split("<")[1] string = string.split(">")[0] diff --git a/components/isceobj/Sensor/GRD/Sentinel1.py b/components/isceobj/Sensor/GRD/Sentinel1.py index 71ba920..3b79146 100755 --- a/components/isceobj/Sensor/GRD/Sentinel1.py +++ b/components/isceobj/Sensor/GRD/Sentinel1.py @@ -406,7 +406,7 @@ class Sentinel1(Component): lat = [] lon = [] - for child in glist.getchildren(): + for child in glist: lat.append( float(child.find('latitude').text)) lon.append( float(child.find('longitude').text)) @@ -502,7 +502,7 @@ class Sentinel1(Component): frameOrbit = Orbit() frameOrbit.configure() - for child in node.getchildren(): + for child in node: timestamp = self.convertToDateTime(child.find('time').text) pos = [] vel = [] @@ -543,7 +543,7 @@ class Sentinel1(Component): tstart = self.product.sensingStart - margin tend = self.product.sensingStop + margin - for child in node.getchildren(): + for child in node: timestamp = self.convertToDateTime(child.find('UTC').text[4:]) if (timestamp >= tstart) and (timestamp < tend): @@ -606,7 +606,7 @@ class Sentinel1(Component): pixels = [] data = None - for ii, child in enumerate(node.getchildren()): + for ii, child in enumerate(node): pixnode = child.find('pixel') nump = int(pixnode.attrib['count']) @@ -685,7 +685,7 @@ class Sentinel1(Component): noise_range_lut_indices = np.zeros((num_vectors,)) noise_range_lut_values = np.zeros((num_vectors, self.product.numberOfSamples)) - for ii, child in enumerate(node.getchildren()): + for ii, child in enumerate(node): print("Processing range noise vector {}/{}".format(ii + 1, num_vectors)) pixnode = child.find('pixel') @@ -711,7 +711,7 @@ class Sentinel1(Component): noise_azimuth_lut_indices = defaultdict(list) noise_azimuth_lut_values = defaultdict(list) - for block_i, child in enumerate(node.getchildren()): + for block_i, child in enumerate(node): print("Processing azimuth noise vector {}/{}".format(block_i + 1, num_vectors)) linenode = child.find('line') signode = child.find("noiseAzimuthLut") @@ -861,7 +861,7 @@ class Sentinel1(Component): data = [] - for ii, child in enumerate(node.getchildren()): + for ii, child in enumerate(node): t0 = self.convertToDateTime(child.find('azimuthTime').text) lines.append( (t0-self.product.sensingStart).total_seconds()/self.product.azimuthTimeInterval) diff --git a/components/isceobj/Sensor/GRD/Terrasarx.py b/components/isceobj/Sensor/GRD/Terrasarx.py index f5aaace..fb99081 100755 --- a/components/isceobj/Sensor/GRD/Terrasarx.py +++ b/components/isceobj/Sensor/GRD/Terrasarx.py @@ -325,8 +325,8 @@ class Terrasar_GRD(Component): glist = (self.getxmlelement(self._xml2_root, 'geolocationGrid')) lat = [] lon = [] - for child in glist.getchildren(): - for grandchild in child.getchildren(): + for child in glist: + for grandchild in child: string = ElementTree.tostring(child, encoding = 'unicode', method = 'xml') string = string.split("<")[1] string = string.split(">")[0] @@ -352,7 +352,7 @@ class Terrasar_GRD(Component): if node.tag == 'stateVec': sv = StateVector() sv.configure() - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': timeStamp = self.convertToDateTime2(z.text) elif z.tag == 'posX': @@ -396,7 +396,7 @@ class Terrasar_GRD(Component): if not node.tag == 'gridPoint': continue - for zz in node.getchildren(): + for zz in node: if zz.tag == 't': az_time = float(zz.text) elif zz.tag == 'tau': @@ -429,9 +429,9 @@ class Terrasar_GRD(Component): if not node.tag == 'antennaPattern': continue - for z in node.getchildren(): + for z in node: if z.tag == 'elevationPattern': - for zz in z.getchildren(): + for zz in z: if zz.tag == 'gainExt': node = float(zz.text) node2.append(node) @@ -467,9 +467,9 @@ class Terrasar_GRD(Component): for node in self._xml_root.find('productSpecific'): if not node.tag == 'projectedImageInfo': continue - for z in node.getchildren(): + for z in node: if z.tag == 'slantToGroundRangeProjection': - for zz in z.getchildren(): + for zz in z: if zz.tag == 'coefficient': p = float(zz.text)*(SPEED_OF_LIGHT) pp.append(p) diff --git a/components/isceobj/Sensor/Radarsat2.py b/components/isceobj/Sensor/Radarsat2.py index 96a03cb..859153a 100755 --- a/components/isceobj/Sensor/Radarsat2.py +++ b/components/isceobj/Sensor/Radarsat2.py @@ -441,7 +441,7 @@ class _Product(Radarsat2Namespace): self.imageAttributes = _ImageAttributes() def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('productId'): self.productId = z.text elif z.tag == self.elementName('documentIdentifier'): @@ -483,7 +483,7 @@ class _SourceAttributes(Radarsat2Namespace): self.orbitAndAttitude = _OrbitAndAttitude() def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('satellite'): self.satellite = z.text elif z.tag == self.elementName('sensor'): @@ -548,7 +548,7 @@ class _RadarParameters(Radarsat2Namespace): def set_from_etnode(self,node): i = 0 - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('acquisitionType'): self.acquisitionType = z.text elif z.tag == self.elementName('beams'): @@ -608,7 +608,7 @@ class _ReferenceNoiseLevel(Radarsat2Namespace): self.noiseLevelValues = [] def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('pixelFirstNoiseValue'): self.pixelFirstNoiseValue = int(z.text) elif z.tag == self.elementName('stepSize'): @@ -660,7 +660,7 @@ class _OrbitAndAttitude(Radarsat2Namespace): self.attitudeInformation = _AttitudeInformation() def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('orbitInformation'): self.orbitInformation.set_from_etnode(z) elif z.tag == self.elementName('attitudeInformation'): @@ -685,7 +685,7 @@ class _OrbitInformation(Radarsat2Namespace): self.stateVectors = [] def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('passDirection'): self.passDirection = z.text elif z.tag == self.elementName('orbitDataSource'): @@ -722,7 +722,7 @@ class _StateVector(Radarsat2Namespace): self.zVelocity = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('timeStamp'): self.timeStamp = self.convertToDateTime(z.text) elif z.tag == self.elementName('xPosition'): @@ -766,7 +766,7 @@ class _AttitudeInformation(Radarsat2Namespace): self.attitudeAngles = [] def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('attitudeDataSource'): self.attitudeDataSource = z.text elif z.tag == self.elementName('attitudeOffsetApplied'): @@ -797,7 +797,7 @@ class _AttitudeAngles(Radarsat2Namespace): self.pitch = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('timeStamp'): self.timeStamp = self.convertToDateTime(z.text) elif z.tag == self.elementName('yaw'): @@ -833,7 +833,7 @@ class _ImageGenerationParameters(Radarsat2Namespace): self.payloadCharacteristicsFile = [] def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('generalProcessingInformation'): self.generalProcessingInformation.set_from_etnode(z) elif z.tag == self.elementName('sarProcessingInformation'): @@ -869,7 +869,7 @@ class _GeneralProcessingInformation(Radarsat2Namespace): self.softwareVersion = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('productType'): self.productType = z.text elif z.tag == self.elementName('_processingFacility'): @@ -925,7 +925,7 @@ class _SarProcessingInformation(Radarsat2Namespace): self._satelliteHeight = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('lutApplied'): self.lutApplied = z.text elif z.tag == self.elementName('numberOfLinesProcessed'): @@ -971,7 +971,7 @@ class _Window(Radarsat2Namespace): self.windowCoefficient = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('windowName'): self.windowName = z.text elif z.tag == self.elementName('windowCoefficient'): @@ -1000,7 +1000,7 @@ class _DopplerCentroid(Radarsat2Namespace): self.dopplerCentroidConfidence = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('timeOfDopplerCentroidEstimate'): self.timeOfDopplerCentroidEstimate = self.convertToDateTime(z.text) elif z.tag == self.elementName('dopplerAmbiguity'): @@ -1027,7 +1027,7 @@ class _DopplerRateValues(Radarsat2Namespace): self.dopplerRateValuesCoefficients = [] def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('dopplerRateReferenceTime'): self.dopplerRateReferenceTime = float(z.text) elif z.tag == self.elementName('dopplerRateValuesCoefficients'): @@ -1056,7 +1056,7 @@ class _SlantRangeToGroundRange(Radarsat2Namespace): self.groundToSlantRangeCoefficients = [] def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('zeroDopplerAzimuthTime'): self.zeroDopplerAzimuthTime = self.convertToDateTime(z.text) elif z.tag == self.elementName('slantRangeTimeToFirstRangeSample'): @@ -1072,7 +1072,7 @@ class _ImageAttributes(Radarsat2Namespace): self.fullResolutionImageData = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('productFormat'): self.productFormat = z.text elif z.tag == self.elementName('outputMediaInterleaving'): @@ -1097,7 +1097,7 @@ class _RasterAttributes(Radarsat2Namespace): self.pixelTimeOrdering = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('dataType'): self.dataType = z.text elif z.tag == self.elementName('bitsPerSample'): @@ -1123,7 +1123,7 @@ class _GeographicInformation(Radarsat2Namespace): self.referenceEllipsoidParameters = _ReferenceEllipsoidParameters() def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('geolocationGrid'): self.geolocationGrid.set_from_etnode(z) @@ -1133,7 +1133,7 @@ class _GeolocationGrid(Radarsat2Namespace): self.imageTiePoint = [] def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('imageTiePoint'): tp = _ImageTiePoint() tp.set_from_etnode(z) @@ -1147,7 +1147,7 @@ class _ImageTiePoint(Radarsat2Namespace): self.geodeticCoordinates = _GeodeticCoordinates() def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('imageCoordinate'): self.imageCoordinates.set_from_etnode(z) elif z.tag == self.elementName('geodeticCoordinate'): @@ -1170,7 +1170,7 @@ class _ImageCoordinates(Radarsat2Namespace): self.pixel = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('line'): self.line = float(z.text) elif z.tag == self.elementName('pixel'): @@ -1194,7 +1194,7 @@ class _GeodeticCoordinates(Radarsat2Namespace): self.height = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('latitude'): self.latitude = float(z.text) elif z.tag == self.elementName('longitude'): @@ -1223,7 +1223,7 @@ class _ReferenceEllipsoidParameters(Radarsat2Namespace): self.geodeticTerrainHeight = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == self.elementName('ellipsoidName'): self.ellipsoidName = z.text elif z.tag == self.elementName('semiMajorAxis'): diff --git a/components/isceobj/Sensor/Sentinel1.py b/components/isceobj/Sensor/Sentinel1.py index 7c3317b..6f3316c 100755 --- a/components/isceobj/Sensor/Sentinel1.py +++ b/components/isceobj/Sensor/Sentinel1.py @@ -370,7 +370,7 @@ class Sentinel1(Sensor): frameOrbit = Orbit() frameOrbit.setOrbitSource('Header') - for child in node.getchildren(): + for child in node: timestamp = self.convertToDateTime(child.find('time').text) pos = [] vel = [] @@ -416,7 +416,7 @@ class Sentinel1(Sensor): tstart = self.frame.getSensingStart() - margin tend = self.frame.getSensingStop() + margin - for child in node.getchildren(): + for child in node: timestamp = self.convertToDateTime(child.find('UTC').text[4:]) if (timestamp >= tstart) and (timestamp < tend): @@ -487,7 +487,7 @@ class Sentinel1(Sensor): tdiff = 1.0e9 dpoly = None - for index, burst in enumerate(node.getchildren()): + for index, burst in enumerate(node): refTime = self.convertToDateTime( burst.find('azimuthTime').text) delta = abs((refTime - self.frame.sensingMid).total_seconds()) diff --git a/components/isceobj/Sensor/TOPS/Sentinel1.py b/components/isceobj/Sensor/TOPS/Sentinel1.py index 1178b7a..e4fa1b9 100755 --- a/components/isceobj/Sensor/TOPS/Sentinel1.py +++ b/components/isceobj/Sensor/TOPS/Sentinel1.py @@ -635,7 +635,7 @@ class Sentinel1(Component): ''' burstList = self.getxmlelement('swathTiming/burstList') - for index, burst in enumerate(burstList.getchildren()): + for index, burst in enumerate(burstList): bb = self.product.bursts[index] bb.sensingStart = self.convertToDateTime(burst.find('azimuthTime').text) deltaT = datetime.timedelta(seconds=(bb.numberOfLines - 1)*bb.azimuthTimeInterval) @@ -670,7 +670,7 @@ class Sentinel1(Component): ####Read in fm rates separately fmrateList = self.getxmlelement('generalAnnotation/azimuthFmRateList') fmRates = [] - for index, burst in enumerate(fmrateList.getchildren()): + for index, burst in enumerate(fmrateList): r0 = 0.5 * Const.c * float(burst.find('t0').text) try: c0 = float(burst.find('c0').text) @@ -702,7 +702,7 @@ class Sentinel1(Component): dcList = self.getxmlelement('dopplerCentroid/dcEstimateList') dops = [ ] - for index, burst in enumerate(dcList.getchildren()): + for index, burst in enumerate(dcList): r0 = 0.5 * Const.c* float(burst.find('t0').text) refTime = self.convertToDateTime(burst.find('azimuthTime').text) @@ -729,7 +729,7 @@ class Sentinel1(Component): eapList = self.getxmlelement('antennaPattern/antennaPatternList') eaps = [] - for index, burst in enumerate(eapList.getchildren()): + for index, burst in enumerate(eapList): refTime = self.convertToDateTime(burst.find('azimuthTime').text) taus = [float(val) for val in burst.find('slantRangeTime').text.split()] angs = [float(val) for val in burst.find('elevationAngle').text.split()] @@ -801,7 +801,7 @@ class Sentinel1(Component): frameOrbit = Orbit() frameOrbit.configure() - for child in node.getchildren(): + for child in node: timestamp = self.convertToDateTime(child.find('time').text) pos = [] vel = [] @@ -847,7 +847,7 @@ class Sentinel1(Component): tstart = self.product.bursts[0].sensingStart - margin tend = self.product.bursts[-1].sensingStop + margin - for child in node.getchildren(): + for child in node: timestamp = self.convertToDateTime(child.find('UTC').text[4:]) if (timestamp >= tstart) and (timestamp < tend): @@ -889,7 +889,7 @@ class Sentinel1(Component): xml_root = ET.ElementTree(file=fp).getroot() res = xml_root.find('calibrationParamsList/calibrationParams') paramsList = xml_root.find('calibrationParamsList') - for par in (paramsList.getchildren()): + for par in paramsList: if (par.find('swath').text.strip() == ('IW'+str(burst.swathNumber))) and (par.find('polarisation').text == burst.polarization): self._delta_theta = float(par.find('elevationAntennaPattern/elevationAngleIncrement').text) Geap_IQ = [float(val) for val in par.find('elevationAntennaPattern/values').text.split()] diff --git a/components/isceobj/Sensor/TanDEMX.py b/components/isceobj/Sensor/TanDEMX.py index 64bed71..db6ee78 100755 --- a/components/isceobj/Sensor/TanDEMX.py +++ b/components/isceobj/Sensor/TanDEMX.py @@ -122,8 +122,7 @@ class TanDEMX(Component): raise IOError(strerr) self._xml_root = ElementTree(file=fp).getroot() - a = self._xml_root.getchildren() - for z in a: + for z in self._xml_root: if z.tag == 'generalHeader': self.generalHeader.set_from_etnode(z) if z.tag == 'productComponents': @@ -441,7 +440,7 @@ class _GeneralHeader(object): self.fileName = node.attrib['fileName'] self.fileVersion = node.attrib['fileVersion'] self.status = node.attrib['status'] - for z in node.getchildren(): + for z in node: if z.tag == 'itemName': self.itemName = z.text if z.tag == 'mission': @@ -512,7 +511,7 @@ class _ProductComponents(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'annotation': self.annotation.append(_Annotation()) self.annotation[-1].set_from_etnode(z) @@ -554,7 +553,7 @@ class _Annotation(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'type': self.type = z.text if z.tag == 'file': @@ -581,7 +580,7 @@ class _ImageData(object): def set_from_etnode(self,node): self.layerIndex = int(node.attrib['layerIndex']) - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.type = z.text if z.tag == 'file': @@ -610,7 +609,7 @@ class _QuickLooks(object): def set_from_etnode(self,node): self.layerIndex = int(node.attrib['layerIndex']) - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.type = z.text if z.tag == 'file': @@ -636,7 +635,7 @@ class _CompositeQuickLook(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'file': self.file.set_from_etnode(z) return @@ -656,7 +655,7 @@ class _BrowseImage(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'file': self.file.set_from_etnode(z) return @@ -676,7 +675,7 @@ class _MapPlot(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'file': self.file.set_from_etnode(z) return @@ -706,7 +705,7 @@ class _ProductInfo(object): self.sceneInfo = _SceneInfo() def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'generationInfo': self.generationInfo.set_from_etnode(z) if z.tag == 'missionInfo': @@ -752,7 +751,7 @@ class _GenerationInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'logicalProductID': self.logicalProductID = z.text if z.tag == 'receivingStation': @@ -800,7 +799,7 @@ class _QualityInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'qualityInspection': self.qualityInspection = z.text if z.tag == 'qualityRemark': @@ -830,7 +829,7 @@ class _MissionInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'mission': self.mission = z.text if z.tag == 'orbitPhase': @@ -872,7 +871,7 @@ class _PolarisationList(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text @@ -890,7 +889,7 @@ class _ImagingModeStripMap(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'azimuthBeamID': self.azimuthBeamID = z.text return @@ -908,7 +907,7 @@ class _ImagingModeSpecificInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'stripMap': self.stripMap.set_from_etnode(z) return @@ -933,7 +932,7 @@ class _AcquisitionInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'sensor': self.sensor = z.text if z.tag == 'imagingMode': @@ -980,7 +979,7 @@ class _ProductVariantInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'productType': self.productType = z.text if z.tag == 'productVariant': @@ -1014,7 +1013,7 @@ class _ImageRaster(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'numberOfRows': self.numberOfRows = int(z.text) if z.tag == 'numberOfColumns': @@ -1062,7 +1061,7 @@ class _ImageDataInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'imageRaster': self.imageRaster.set_from_etnode(z) return @@ -1084,7 +1083,7 @@ class _SceneInfoTime(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'timeGPS': @@ -1111,7 +1110,7 @@ class _SceneInfoRangeTime(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'firstPixel': self.firstPixel = float(z.text) if z.tag == 'lastPixel': @@ -1139,7 +1138,7 @@ class _SceneInfoSceneCornerCoord(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'refRow': self.refRow = int(z.text) if z.tag == 'refColumn': @@ -1188,7 +1187,7 @@ class _SceneCenterCoord(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'refRow': self.refRow = int(z.text) if z.tag == 'refColumn': @@ -1237,7 +1236,7 @@ class _SceneInfo(object): def set_from_etnode(self,node): iCorner = -1 - for z in node.getchildren(): + for z in node: if z.tag == 'sceneID': self.sceneID = z.text if z.tag == 'start': @@ -1282,7 +1281,7 @@ class _ProductSpecific(object): self.complexImageInfo = _ComplexImageInfo() return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'complexImageInfo': self.complexImageInfo.set_from_etnode(z) return @@ -1302,7 +1301,7 @@ class _ComplexImageInfo(object): self.quicklookDataStartWith = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'commonPRF': self.commonPRF = float(z.text) if z.tag == 'commonRSF': @@ -1348,7 +1347,7 @@ class _ProjectedSpacingRange(object): self.slantRange = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'groundNear': self.groundNear = float(z.text) if z.tag == 'groundFar': @@ -1381,7 +1380,7 @@ class _Platform(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'referenceData': self.referenceData.set_from_etnode(z) if z.tag == 'orbit': @@ -1410,7 +1409,7 @@ class _SARAntennaPosition(object): def set_from_etnode(self,node): self.DRAoffset = node.attrib['DRAoffset'] - for w in node.getchildren(): + for w in node: if w.tag == 'x': self.x = float(w.text) if w.tag == 'y': @@ -1438,7 +1437,7 @@ class _GPSAntennaPosition(object): def set_from_etnode(self,node): self.GPSreceiver = node.attrib['GPSreceiver'] self.unit = node.attrib['unit'] - for w in node.getchildren(): + for w in node: if w.tag == 'x': self.x = float(w.text) if w.tag == 'y': @@ -1469,7 +1468,7 @@ class _PlatformReferenceData(object): def set_from_etnode(self,node): iGPSAnt = -1 - for x in node.getchildren(): + for x in node: if x.tag == 'SARAntennaMechanicalBoresight': self.SARAntennaMechanicalBoresight = float(x.text) if x.tag == 'SARAntennaPosition': @@ -1497,7 +1496,7 @@ class _FirstStateTime(object): self.firstStateTimeGPSFraction = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'firstStateTimeUTC': self.firstStateTimeUTC = z.text if z.tag == 'firstStateTimeGPS': @@ -1523,7 +1522,7 @@ class _LastStateTime(object): self.lastStateTimeGPSFraction = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'lastStateTimeUTC': self.lastStateTimeUTC = z.text if z.tag == 'lastStateTimeGPS': @@ -1562,7 +1561,7 @@ class _OrbitHeader(object): self.dataGapIndicator = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'generationSystem': self.generationSystem = z.text self.generationSystemVersion = z.attrib['version'] @@ -1652,7 +1651,7 @@ class _StateVec(object): self.maneuver = node.attrib['maneuver'] self.num = int(node.attrib['num']) self.qualInd = int(node.attrib['qualInd']) - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = datetime.datetime.strptime(z.text,"%Y-%m-%dT%H:%M:%S.%f") if z.tag == 'timeGPS': @@ -1700,7 +1699,7 @@ class _Orbit(object): self.stateVec = () def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'orbitHeader': self.orbitHeader.set_from_etnode(z) if z.tag == 'stateVec': @@ -1737,7 +1736,7 @@ class _AttitudeData(object): self.maneuver = node.attrib['maneuver'] self.num = int(node.attrib['num']) self.qualInd = int(node.attrib['qualInd']) - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'timeGPS': @@ -1782,7 +1781,7 @@ class _FirstAttitudeTime(object): self.firstAttitudeTimeGPSFraction = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'firstAttitudeTimeUTC': self.firstAttitudeTimeUTC = z.text if z.tag == 'firstAttitudeTimeGPS': @@ -1808,7 +1807,7 @@ class _LastAttitudeTime(object): self.lastAttitudeTimeGPSFraction = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'lastAttitudeTimeUTC': self.lastAttitudeTimeUTC = z.text if z.tag == 'lastAttitudeTimeGPS': @@ -1834,7 +1833,7 @@ class _AttitudeDataRefFrame(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'FromFrame': self.FromFrame = z.text if z.tag == 'ToFrame': @@ -1870,7 +1869,7 @@ class _AttitudeHeader(object): self.steeringLawIndicator = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'generationSystem': self.generationSystem = z.text self.generationSystemVersion = z.attrib['version'] @@ -1948,7 +1947,7 @@ class _Attitude(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'attitudeHeader': self.attitudeHeader.set_from_etnode(z) if z.tag == 'attitudeData': @@ -1977,7 +1976,7 @@ class _Instrument(object): self.settings = _InstrumentSettings() def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'instrumentInfoCoordinateType': self.instrumentInfoCoordinateType = z.text if z.tag == 'radarParameters': @@ -2003,7 +2002,7 @@ class _RadarParameters(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'centerFrequency': self.centerFrequency = float(z.text) return @@ -2024,7 +2023,7 @@ class _RxGainSetting(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'startTimeUTC': self.startTimeUTC = z.text if z.tag == 'stopTimeUTC': @@ -2058,7 +2057,7 @@ class _DataSegment(object): def set_from_etnode(self,node): self.segmentID = int(node.attrib['segmentID']) - for z in node.getchildren(): + for z in node: if z.tag == 'startTimeUTC': self.startTimeUTC = z.text if z.tag == 'stopTimeUTC': @@ -2095,7 +2094,7 @@ class _SettingRecord(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'dataSegment': self.dataSegment.set_from_etnode(z) if z.tag == 'PRF': @@ -2157,7 +2156,7 @@ class _InstrumentSettings(object): self.settingRecord = () def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text if z.tag == 'DRAoffset': @@ -2245,7 +2244,7 @@ class _Processing(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'geometry': self.geometry.set_from_etnode(z) if z.tag == 'doppler': @@ -2279,7 +2278,7 @@ class _ProcessingGeometry(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'geometryCoordinateType': self.geometryCoordinateType = z.text if z.tag == 'velocityParameter': @@ -2315,7 +2314,7 @@ class _VelocityParameter(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'velocityParameterPolynomial': @@ -2342,7 +2341,7 @@ class _VelocityParameterPolynomial(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2385,7 +2384,7 @@ class _ZeroDopplerVelocity(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'velocity': self.velocity = float(z.text) return @@ -2405,7 +2404,7 @@ class _DopplerRate(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'dopplerRatePolynomial': @@ -2432,7 +2431,7 @@ class _DopplerRatePolynomial(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2477,7 +2476,7 @@ class _ProcessingDoppler(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'dopplerBasebandEstimationMethod': self.dopplerBasebandEstimationMethod = z.text if z.tag == 'dopplerGeometricEstimationMethod': @@ -2518,7 +2517,7 @@ class _ProcessingDopplerCentroid(object): def set_from_etnode(self,node): self.layerIndex = int(node.attrib['layerIndex']) - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text if z.tag == 'DRAoffset': @@ -2581,7 +2580,7 @@ class _DopplerEstimate(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'dopplerAtMidRange': @@ -2636,7 +2635,7 @@ class _BasebandDoppler(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2681,7 +2680,7 @@ class _GeometricDoppler(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2727,7 +2726,7 @@ class _CombinedDoppler(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2779,7 +2778,7 @@ class _ProcessingParameter(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'beamID': self.beamID = z.text if z.tag == 'processingInfoCoordinateType': @@ -2844,7 +2843,7 @@ class _RangeCompression(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'segmentInfo': self.segmentInfo.set_from_etnode(z) if z.tag == 'chirps': @@ -2869,7 +2868,7 @@ class _RCSegmentInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text if z.tag == 'dataSegment': @@ -2894,7 +2893,7 @@ class _RCDataSegment(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'startTimeUTC': self.startTimeUTC = z.text if z.tag == 'stopTimeUTC': @@ -2921,7 +2920,7 @@ class _RCChirps(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'referenceChirp': self.referenceChirp.set_from_etnode(z) return @@ -2949,7 +2948,7 @@ class _RCReferenceChirp(object): def set_from_etnode(self,node): self.pulseCode = int(node.attrib['pulseCode']) - for z in node.getchildren(): + for z in node: if z.tag == 'pulseType': self.pulseType = z.text if z.tag == 'chirpDesignator': @@ -3002,7 +3001,7 @@ class _RCChirpAmplitude(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -3047,7 +3046,7 @@ class _RCChirpPhase(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -3090,7 +3089,7 @@ class _CorrectedInstrumentDelay(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text if z.tag == 'DRAoffset': @@ -3143,7 +3142,7 @@ class _File(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'location': self.location.set_from_etnode(z) if z.tag == 'size': @@ -3172,7 +3171,7 @@ class _FileLocation(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'host': self.host = z.text if z.tag == 'path': diff --git a/components/isceobj/Sensor/TerraSARX.py b/components/isceobj/Sensor/TerraSARX.py index 867c860..65b289c 100755 --- a/components/isceobj/Sensor/TerraSARX.py +++ b/components/isceobj/Sensor/TerraSARX.py @@ -117,8 +117,7 @@ class TerraSARX(Sensor): raise IOError(strerr) self._xml_root = ElementTree(file=fp).getroot() - a = self._xml_root.getchildren() - for z in a: + for z in self._xml_root: if z.tag == 'generalHeader': self.generalHeader.set_from_etnode(z) if z.tag == 'productComponents': @@ -407,7 +406,7 @@ class _GeneralHeader(object): self.fileName = node.attrib['fileName'] self.fileVersion = node.attrib['fileVersion'] self.status = node.attrib['status'] - for z in node.getchildren(): + for z in node: if z.tag == 'itemName': self.itemName = z.text if z.tag == 'mission': @@ -478,7 +477,7 @@ class _ProductComponents(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'annotation': self.annotation.append(_Annotation()) self.annotation[-1].set_from_etnode(z) @@ -520,7 +519,7 @@ class _Annotation(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'type': self.type = z.text if z.tag == 'file': @@ -547,7 +546,7 @@ class _ImageData(object): def set_from_etnode(self,node): self.layerIndex = int(node.attrib['layerIndex']) - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.type = z.text if z.tag == 'file': @@ -576,7 +575,7 @@ class _QuickLooks(object): def set_from_etnode(self,node): self.layerIndex = int(node.attrib['layerIndex']) - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.type = z.text if z.tag == 'file': @@ -602,7 +601,7 @@ class _CompositeQuickLook(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'file': self.file.set_from_etnode(z) return @@ -622,7 +621,7 @@ class _BrowseImage(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'file': self.file.set_from_etnode(z) return @@ -642,7 +641,7 @@ class _MapPlot(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'file': self.file.set_from_etnode(z) return @@ -673,7 +672,7 @@ class _ProductInfo(object): self.sceneInfo = _SceneInfo() def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'generationInfo': self.generationInfo.set_from_etnode(z) if z.tag == 'missionInfo': @@ -719,7 +718,7 @@ class _GenerationInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'logicalProductID': self.logicalProductID = z.text if z.tag == 'receivingStation': @@ -767,7 +766,7 @@ class _QualityInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'qualityInspection': self.qualityInspection = z.text if z.tag == 'qualityRemark': @@ -797,7 +796,7 @@ class _MissionInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'mission': self.mission = z.text if z.tag == 'orbitPhase': @@ -839,7 +838,7 @@ class _PolarisationList(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text @@ -857,7 +856,7 @@ class _ImagingModeStripMap(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'azimuthBeamID': self.azimuthBeamID = z.text return @@ -875,7 +874,7 @@ class _ImagingModeSpecificInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'stripMap': self.stripMap.set_from_etnode(z) return @@ -900,7 +899,7 @@ class _AcquisitionInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'sensor': self.sensor = z.text if z.tag == 'imagingMode': @@ -947,7 +946,7 @@ class _ProductVariantInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'productType': self.productType = z.text if z.tag == 'productVariant': @@ -981,7 +980,7 @@ class _ImageRaster(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'numberOfRows': self.numberOfRows = int(z.text) if z.tag == 'numberOfColumns': @@ -1029,7 +1028,7 @@ class _ImageDataInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'imageRaster': self.imageRaster.set_from_etnode(z) return @@ -1051,7 +1050,7 @@ class _SceneInfoTime(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'timeGPS': @@ -1078,7 +1077,7 @@ class _SceneInfoRangeTime(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'firstPixel': self.firstPixel = float(z.text) if z.tag == 'lastPixel': @@ -1106,7 +1105,7 @@ class _SceneInfoSceneCornerCoord(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'refRow': self.refRow = int(z.text) if z.tag == 'refColumn': @@ -1155,7 +1154,7 @@ class _SceneCenterCoord(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'refRow': self.refRow = int(z.text) if z.tag == 'refColumn': @@ -1204,7 +1203,7 @@ class _SceneInfo(object): def set_from_etnode(self,node): iCorner = -1 - for z in node.getchildren(): + for z in node: if z.tag == 'sceneID': self.sceneID = z.text if z.tag == 'start': @@ -1249,7 +1248,7 @@ class _ProductSpecific(object): self.complexImageInfo = _ComplexImageInfo() return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'complexImageInfo': self.complexImageInfo.set_from_etnode(z) return @@ -1269,7 +1268,7 @@ class _ComplexImageInfo(object): self.quicklookDataStartWith = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'commonPRF': self.commonPRF = float(z.text) if z.tag == 'commonRSF': @@ -1315,7 +1314,7 @@ class _ProjectedSpacingRange(object): self.slantRange = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'groundNear': self.groundNear = float(z.text) if z.tag == 'groundFar': @@ -1348,7 +1347,7 @@ class _Platform(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'referenceData': self.referenceData.set_from_etnode(z) if z.tag == 'orbit': @@ -1377,7 +1376,7 @@ class _SARAntennaPosition(object): def set_from_etnode(self,node): self.DRAoffset = node.attrib['DRAoffset'] - for w in node.getchildren(): + for w in node: if w.tag == 'x': self.x = float(w.text) if w.tag == 'y': @@ -1405,7 +1404,7 @@ class _GPSAntennaPosition(object): def set_from_etnode(self,node): self.GPSreceiver = node.attrib['GPSreceiver'] self.unit = node.attrib['unit'] - for w in node.getchildren(): + for w in node: if w.tag == 'x': self.x = float(w.text) if w.tag == 'y': @@ -1436,7 +1435,7 @@ class _PlatformReferenceData(object): def set_from_etnode(self,node): iGPSAnt = -1 - for x in node.getchildren(): + for x in node: if x.tag == 'SARAntennaMechanicalBoresight': self.SARAntennaMechanicalBoresight = float(x.text) if x.tag == 'SARAntennaPosition': @@ -1464,7 +1463,7 @@ class _FirstStateTime(object): self.firstStateTimeGPSFraction = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'firstStateTimeUTC': self.firstStateTimeUTC = z.text if z.tag == 'firstStateTimeGPS': @@ -1490,7 +1489,7 @@ class _LastStateTime(object): self.lastStateTimeGPSFraction = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'lastStateTimeUTC': self.lastStateTimeUTC = z.text if z.tag == 'lastStateTimeGPS': @@ -1529,7 +1528,7 @@ class _OrbitHeader(object): self.dataGapIndicator = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'generationSystem': self.generationSystem = z.text self.generationSystemVersion = z.attrib['version'] @@ -1619,7 +1618,7 @@ class _StateVec(object): self.maneuver = node.attrib['maneuver'] self.num = int(node.attrib['num']) self.qualInd = int(node.attrib['qualInd']) - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = datetime.datetime.strptime(z.text,"%Y-%m-%dT%H:%M:%S.%f") if z.tag == 'timeGPS': @@ -1667,7 +1666,7 @@ class _Orbit(object): self.stateVec = () def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'orbitHeader': self.orbitHeader.set_from_etnode(z) if z.tag == 'stateVec': @@ -1704,7 +1703,7 @@ class _AttitudeData(object): self.maneuver = node.attrib['maneuver'] self.num = int(node.attrib['num']) self.qualInd = int(node.attrib['qualInd']) - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'timeGPS': @@ -1749,7 +1748,7 @@ class _FirstAttitudeTime(object): self.firstAttitudeTimeGPSFraction = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'firstAttitudeTimeUTC': self.firstAttitudeTimeUTC = z.text if z.tag == 'firstAttitudeTimeGPS': @@ -1775,7 +1774,7 @@ class _LastAttitudeTime(object): self.lastAttitudeTimeGPSFraction = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'lastAttitudeTimeUTC': self.lastAttitudeTimeUTC = z.text if z.tag == 'lastAttitudeTimeGPS': @@ -1801,7 +1800,7 @@ class _AttitudeDataRefFrame(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'FromFrame': self.FromFrame = z.text if z.tag == 'ToFrame': @@ -1837,7 +1836,7 @@ class _AttitudeHeader(object): self.steeringLawIndicator = None def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'generationSystem': self.generationSystem = z.text self.generationSystemVersion = z.attrib['version'] @@ -1915,7 +1914,7 @@ class _Attitude(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'attitudeHeader': self.attitudeHeader.set_from_etnode(z) if z.tag == 'attitudeData': @@ -1944,7 +1943,7 @@ class _Instrument(object): self.settings = _InstrumentSettings() def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'instrumentInfoCoordinateType': self.instrumentInfoCoordinateType = z.text if z.tag == 'radarParameters': @@ -1970,7 +1969,7 @@ class _RadarParameters(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'centerFrequency': self.centerFrequency = float(z.text) return @@ -1991,7 +1990,7 @@ class _RxGainSetting(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'startTimeUTC': self.startTimeUTC = z.text if z.tag == 'stopTimeUTC': @@ -2025,7 +2024,7 @@ class _DataSegment(object): def set_from_etnode(self,node): self.segmentID = int(node.attrib['segmentID']) - for z in node.getchildren(): + for z in node: if z.tag == 'startTimeUTC': self.startTimeUTC = z.text if z.tag == 'stopTimeUTC': @@ -2062,7 +2061,7 @@ class _SettingRecord(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'dataSegment': self.dataSegment.set_from_etnode(z) if z.tag == 'PRF': @@ -2124,7 +2123,7 @@ class _InstrumentSettings(object): self.settingRecord = () def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text if z.tag == 'DRAoffset': @@ -2212,7 +2211,7 @@ class _Processing(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'geometry': self.geometry.set_from_etnode(z) if z.tag == 'doppler': @@ -2246,7 +2245,7 @@ class _ProcessingGeometry(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'geometryCoordinateType': self.geometryCoordinateType = z.text if z.tag == 'velocityParameter': @@ -2282,7 +2281,7 @@ class _VelocityParameter(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'velocityParameterPolynomial': @@ -2309,7 +2308,7 @@ class _VelocityParameterPolynomial(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2352,7 +2351,7 @@ class _ZeroDopplerVelocity(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'velocity': self.velocity = float(z.text) return @@ -2372,7 +2371,7 @@ class _DopplerRate(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'dopplerRatePolynomial': @@ -2399,7 +2398,7 @@ class _DopplerRatePolynomial(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2444,7 +2443,7 @@ class _ProcessingDoppler(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'dopplerBasebandEstimationMethod': self.dopplerBasebandEstimationMethod = z.text if z.tag == 'dopplerGeometricEstimationMethod': @@ -2485,7 +2484,7 @@ class _ProcessingDopplerCentroid(object): def set_from_etnode(self,node): self.layerIndex = int(node.attrib['layerIndex']) - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text if z.tag == 'DRAoffset': @@ -2548,7 +2547,7 @@ class _DopplerEstimate(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'timeUTC': self.timeUTC = z.text if z.tag == 'dopplerAtMidRange': @@ -2603,7 +2602,7 @@ class _BasebandDoppler(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2648,7 +2647,7 @@ class _GeometricDoppler(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2694,7 +2693,7 @@ class _CombinedDoppler(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -2746,7 +2745,7 @@ class _ProcessingParameter(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'beamID': self.beamID = z.text if z.tag == 'processingInfoCoordinateType': @@ -2811,7 +2810,7 @@ class _RangeCompression(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'segmentInfo': self.segmentInfo.set_from_etnode(z) if z.tag == 'chirps': @@ -2836,7 +2835,7 @@ class _RCSegmentInfo(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text if z.tag == 'dataSegment': @@ -2861,7 +2860,7 @@ class _RCDataSegment(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'startTimeUTC': self.startTimeUTC = z.text if z.tag == 'stopTimeUTC': @@ -2888,7 +2887,7 @@ class _RCChirps(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'referenceChirp': self.referenceChirp.set_from_etnode(z) return @@ -2916,7 +2915,7 @@ class _RCReferenceChirp(object): def set_from_etnode(self,node): self.pulseCode = int(node.attrib['pulseCode']) - for z in node.getchildren(): + for z in node: if z.tag == 'pulseType': self.pulseType = z.text if z.tag == 'chirpDesignator': @@ -2969,7 +2968,7 @@ class _RCChirpAmplitude(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -3014,7 +3013,7 @@ class _RCChirpPhase(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'validityRangeMin': self.validityRangeMin = float(z.text) if z.tag == 'validityRangeMax': @@ -3057,7 +3056,7 @@ class _CorrectedInstrumentDelay(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'polLayer': self.polLayer = z.text if z.tag == 'DRAoffset': @@ -3110,7 +3109,7 @@ class _File(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'location': self.location.set_from_etnode(z) if z.tag == 'size': @@ -3135,7 +3134,7 @@ class _FileLocation(object): return def set_from_etnode(self,node): - for z in node.getchildren(): + for z in node: if z.tag == 'host': self.host = z.text if z.tag == 'path': diff --git a/components/isceobj/XmlUtil/XmlUtil.py b/components/isceobj/XmlUtil/XmlUtil.py index 7cd0bf7..b54a0f9 100755 --- a/components/isceobj/XmlUtil/XmlUtil.py +++ b/components/isceobj/XmlUtil/XmlUtil.py @@ -154,7 +154,7 @@ class XmlUtil: else: keyWord = var.find('name').text - listChildren = var.getchildren() + listChildren = list(var) tmpDict = {} for description in listChildren: if(description.tag == 'name'): diff --git a/components/isceobj/XmlUtil/xmlUtils.py b/components/isceobj/XmlUtil/xmlUtils.py index b6c87d7..be636df 100755 --- a/components/isceobj/XmlUtil/xmlUtils.py +++ b/components/isceobj/XmlUtil/xmlUtils.py @@ -90,7 +90,7 @@ class OrderedDict(UserDict): def dict_to_xml(adict,file): a = ET.Element('') # something to hang nodes on a = dict_to_et(a,adict) - et = a.getchildren()[0] + et = list(a)[0] indent(et) tree = ET.ElementTree(et) tree.write(file) diff --git a/contrib/stack/stripmapStack/prepSlcRSAT2.py b/contrib/stack/stripmapStack/prepSlcRSAT2.py index 7d65423..fe517ec 100755 --- a/contrib/stack/stripmapStack/prepSlcRSAT2.py +++ b/contrib/stack/stripmapStack/prepSlcRSAT2.py @@ -44,7 +44,7 @@ def get_Date(RSAT2folder): tree = etree.parse(RSAT2file) root = tree.getroot() for attributes in root.iter('{http://www.rsi.ca/rs2/prod/xml/schemas}sourceAttributes'): - attribute_list = attributes.getchildren() + attribute_list = list(attributes) for attribute in attribute_list: if attribute.tag=='{http://www.rsi.ca/rs2/prod/xml/schemas}rawDataStartTime': date = attribute.text From 6fdfd1365ec9b1a9acb7059d3f5a62cd7665a743 Mon Sep 17 00:00:00 2001 From: Scott Staniewicz Date: Wed, 26 May 2021 16:37:50 -0500 Subject: [PATCH 11/19] logging.conf: avoid empty file creation Keeps the larger size from previous commit, but sets `delay=True` in `RotatingFileHandler` https://docs.python.org/3/library/logging.handlers.html#logging.handlers.RotatingFileHandler Arguments are: `RotatingFileHandler(filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False, errors=None)` --- defaults/logging/logging.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/defaults/logging/logging.conf b/defaults/logging/logging.conf index 1b966fc..8f10b48 100755 --- a/defaults/logging/logging.conf +++ b/defaults/logging/logging.conf @@ -20,8 +20,8 @@ propagate=0 [handler_fileHandler] class=handlers.RotatingFileHandler formatter=simpleFormatter -# Filename, file mode, maximum file size in bytes,number of backups to keep -args=('isce.log','a',1000048576,5) +# Filename, file mode, maximum file size in bytes,number of backups to keep, encoding, delay creation +args=('isce.log','a',1000048576,5,None,True) [handler_consoleHandler] class=StreamHandler From f20e8d33f805b7b8472239a3874087afe0c476e8 Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Thu, 27 May 2021 11:28:18 -0700 Subject: [PATCH 12/19] Add topsApp preprocess test to CI We can run topsApp up through the preprocess step without needing any data. This will at least verify that the config is parsed properly. --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index fa16804..e320cd8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,6 +53,10 @@ jobs: topsApp.py --help --steps stripmapApp.py --help --steps python3 -c "import isce" + # Create dummy ref/secondary configs for topsApp + ln -s ../examples/input_files/reference_TOPS_SENTINEL1.xml reference.xml + ln -s reference.xml secondary.xml + topsApp.py --steps --end=preprocess ../examples/input_files/topsApp.xml test: docker: From ef29fecb92a0fd8629d94251f7e58ac71fc9dba4 Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Fri, 28 May 2021 09:33:52 +0800 Subject: [PATCH 13/19] Proposed fix on a bug in create_cmds.py (#291) I'm proposing a fix to create_cmds.py since I found a minor bug in line 165 which doesn't remove the excluded dates during image stacking. Instead line 165 removes dates based on the dates not included from the dates to be included parameter on the alosStack.xml. The image folder contains the ALOS-2 images with dates 191005, 191019, 191102, 191214, 191228. I've tested the changes and it worked properly. Co-authored-by: Ryan Burns --- contrib/stack/alosStack/create_cmds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/stack/alosStack/create_cmds.py b/contrib/stack/alosStack/create_cmds.py index 6ef5240..faacab6 100755 --- a/contrib/stack/alosStack/create_cmds.py +++ b/contrib/stack/alosStack/create_cmds.py @@ -162,7 +162,7 @@ def formPairs(idir, numberOfSubsequentDates, pairTimeSpanMinimum=None, pairTimeS for pair in pairsProcess: rdate = pair.split('-')[0] sdate = pair.split('-')[1] - if (rdate not in datesIncluded) and (sdate not in datesIncluded): + if (rdate not in datesExcluded) and (sdate not in datesExcluded): pairsProcess2.append(pair) pairsProcess = pairsProcess2 From cd66b264e32d6a64496f450677cce5f2b9ff53ab Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Fri, 28 May 2021 12:05:57 -0700 Subject: [PATCH 14/19] Release 2.5.2 --- release_history.py | 1 + 1 file changed, 1 insertion(+) diff --git a/release_history.py b/release_history.py index 0c601b7..a12c2ef 100755 --- a/release_history.py +++ b/release_history.py @@ -64,6 +64,7 @@ releases = (Tag('1.0.0', '739', '20120814'), Tag('2.4.2', '', '20201116'), Tag('2.5.0', '', '20210304'), Tag('2.5.1', '', '20210305'), + Tag('2.5.2', '', '20210528'), ) From e5ce2c0a29e1f57540e1e2e9a0cf7ba29da3f58f Mon Sep 17 00:00:00 2001 From: Bryan Marfito Date: Fri, 2 Jul 2021 11:42:39 +0800 Subject: [PATCH 15/19] Change the syntax to remove warning --- contrib/stack/topsStack/SentinelWrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/stack/topsStack/SentinelWrapper.py b/contrib/stack/topsStack/SentinelWrapper.py index 8433d94..cf6fc8a 100755 --- a/contrib/stack/topsStack/SentinelWrapper.py +++ b/contrib/stack/topsStack/SentinelWrapper.py @@ -110,7 +110,7 @@ class ConfigParser: # Looks for string between $ sysmbols in the common subheading in config file def __parseString(self, iString): - if iString is '': + if iString == '': return iString elif isinstance(self.common, (dict)): # Case when "common" parameters are read from the configuration file From 80411d38aa686ca01c4f93e804d91720f75a8f74 Mon Sep 17 00:00:00 2001 From: Lijun Zhu Date: Tue, 13 Apr 2021 10:08:29 -0700 Subject: [PATCH 16/19] PyCuAmpcor: add sum_area_table based correlation surface normalization to allow arbitrary window size (used to be limited by the 1024 max cuda threads) --- contrib/PyCuAmpcor/CMakeLists.txt | 2 + contrib/PyCuAmpcor/examples/cuDenseOffsets.py | 10 - contrib/PyCuAmpcor/src/Makefile | 9 +- contrib/PyCuAmpcor/src/SConscript | 1 + contrib/PyCuAmpcor/src/cuAmpcorChunk.cu | 18 +- contrib/PyCuAmpcor/src/cuAmpcorChunk.h | 5 + contrib/PyCuAmpcor/src/cuAmpcorController.cu | 2 +- contrib/PyCuAmpcor/src/cuAmpcorUtil.h | 11 +- contrib/PyCuAmpcor/src/cuCorrNormalization.cu | 68 +++++- .../PyCuAmpcor/src/cuCorrNormalizationSAT.cu | 201 ++++++++++++++++++ contrib/PyCuAmpcor/src/cuCorrNormalizer.cu | 107 ++++++++++ contrib/PyCuAmpcor/src/cuCorrNormalizer.h | 91 ++++++++ 12 files changed, 507 insertions(+), 18 deletions(-) create mode 100644 contrib/PyCuAmpcor/src/cuCorrNormalizationSAT.cu create mode 100644 contrib/PyCuAmpcor/src/cuCorrNormalizer.cu create mode 100644 contrib/PyCuAmpcor/src/cuCorrNormalizer.h diff --git a/contrib/PyCuAmpcor/CMakeLists.txt b/contrib/PyCuAmpcor/CMakeLists.txt index cd5acfd..24054d4 100644 --- a/contrib/PyCuAmpcor/CMakeLists.txt +++ b/contrib/PyCuAmpcor/CMakeLists.txt @@ -21,6 +21,8 @@ cython_add_module(PyCuAmpcor src/cuArraysPadding.cu src/cuCorrFrequency.cu src/cuCorrNormalization.cu + src/cuCorrNormalizationSAT.cu + src/cuCorrNormalizer.cu src/cuCorrTimeDomain.cu src/cuDeramp.cu src/cuEstimateStats.cu diff --git a/contrib/PyCuAmpcor/examples/cuDenseOffsets.py b/contrib/PyCuAmpcor/examples/cuDenseOffsets.py index c1d374d..8c3fb18 100755 --- a/contrib/PyCuAmpcor/examples/cuDenseOffsets.py +++ b/contrib/PyCuAmpcor/examples/cuDenseOffsets.py @@ -143,16 +143,6 @@ def cmdLineParse(iargs = None): parser = createParser() inps = parser.parse_args(args=iargs) - # check oversampled window size - if (inps.winwidth + 2 * inps.srcwidth ) * inps.raw_oversample > 1024: - msg = 'The oversampled window width, ' \ - 'as computed by (winwidth+2*srcwidth)*raw_oversample, ' \ - 'exceeds the current implementation limit of 1,024. ' \ - f'Please reduce winwidth: {inps.winwidth}, ' \ - f'srcwidth: {inps.srcwidth}, ' \ - f'or raw_oversample: {inps.raw_oversample}.' - raise ValueError(msg) - return inps diff --git a/contrib/PyCuAmpcor/src/Makefile b/contrib/PyCuAmpcor/src/Makefile index bf3be71..e19d643 100644 --- a/contrib/PyCuAmpcor/src/Makefile +++ b/contrib/PyCuAmpcor/src/Makefile @@ -15,7 +15,8 @@ NVCC=nvcc DEPS = cudaUtil.h cudaError.h cuArrays.h GDALImage.h cuAmpcorParameter.h OBJS = GDALImage.o cuArrays.o cuArraysCopy.o cuArraysPadding.o cuOverSampler.o \ cuSincOverSampler.o cuDeramp.o cuOffset.o \ - cuCorrNormalization.o cuAmpcorParameter.o cuCorrTimeDomain.o cuCorrFrequency.o \ + cuCorrNormalization.o cuCorrNormalizationSAT.o cuCorrNormalizer.o \ + cuAmpcorParameter.o cuCorrTimeDomain.o cuCorrFrequency.o \ cuAmpcorChunk.o cuAmpcorController.o cuEstimateStats.o all: pyampcor @@ -47,6 +48,12 @@ cuOffset.o: cuOffset.cu $(DEPS) cuCorrNormalization.o: cuCorrNormalization.cu $(DEPS) $(NVCC) $(NVCCFLAGS) -c -o $@ cuCorrNormalization.cu +cuCorrNormalizationSAT.o: cuCorrNormalizationSAT.cu $(DEPS) + $(NVCC) $(NVCCFLAGS) -c -o $@ cuCorrNormalizationSAT.cu + +cuCorrNormalizer.o: cuCorrNormalizer.cu $(DEPS) + $(NVCC) $(NVCCFLAGS) -c -o $@ cuCorrNormalizer.cu + cuAmpcorParameter.o: cuAmpcorParameter.cu $(NVCC) $(NVCCFLAGS) -c -o $@ cuAmpcorParameter.cu diff --git a/contrib/PyCuAmpcor/src/SConscript b/contrib/PyCuAmpcor/src/SConscript index c88b4d4..9ab50c3 100644 --- a/contrib/PyCuAmpcor/src/SConscript +++ b/contrib/PyCuAmpcor/src/SConscript @@ -11,6 +11,7 @@ listFiles = ['GDALImage.cu', 'cuArrays.cu', 'cuArraysCopy.cu', 'cuArraysPadding.cu', 'cuOverSampler.cu', 'cuSincOverSampler.cu', 'cuDeramp.cu', 'cuOffset.cu', 'cuCorrNormalization.cu', + 'cuCorrNormalizationSAT.cu', 'cuCorrNormalizer.cu' 'cuAmpcorParameter.cu', 'cuCorrTimeDomain.cu', 'cuAmpcorController.cu', 'cuCorrFrequency.cu', 'cuAmpcorChunk.cu', 'cuEstimateStats.cu'] diff --git a/contrib/PyCuAmpcor/src/cuAmpcorChunk.cu b/contrib/PyCuAmpcor/src/cuAmpcorChunk.cu index c7c894d..8ac887e 100644 --- a/contrib/PyCuAmpcor/src/cuAmpcorChunk.cu +++ b/contrib/PyCuAmpcor/src/cuAmpcorChunk.cu @@ -54,7 +54,7 @@ void cuAmpcorChunk::run(int idxDown_, int idxAcross_) #endif // normalize the correlation surface - cuCorrNormalize(r_referenceBatchRaw, r_secondaryBatchRaw, r_corrBatchRaw, stream); + corrNormalizerRaw->execute(r_corrBatchRaw, r_referenceBatchRaw, r_secondaryBatchRaw, stream); #ifdef CUAMPCOR_DEBUG // dump the normalized correlation surface @@ -155,7 +155,7 @@ void cuAmpcorChunk::run(int idxDown_, int idxAcross_) #endif // normalize the correlation surface - cuCorrNormalize(r_referenceBatchOverSampled, r_secondaryBatchOverSampled, r_corrBatchZoomIn, stream); + corrNormalizerOverSampled->execute(r_corrBatchZoomIn, r_referenceBatchOverSampled, r_secondaryBatchOverSampled, stream); #ifdef CUAMPCOR_DEBUG // dump the oversampled correlation surface (normalized) @@ -549,10 +549,22 @@ cuAmpcorChunk::cuAmpcorChunk(cuAmpcorParameter *param_, GDALImage *reference_, G stream); cuCorrFreqDomain_OverSampled = new cuFreqCorrelator( param->searchWindowSizeHeight, param->searchWindowSizeWidth, - param->numberWindowDownInChunk*param->numberWindowAcrossInChunk, + param->numberWindowDownInChunk * param->numberWindowAcrossInChunk, stream); } + corrNormalizerRaw = new cuNormalizer( + param->searchWindowSizeHeightRaw, + param->searchWindowSizeWidthRaw, + param->numberWindowDownInChunk * param->numberWindowAcrossInChunk + ); + + corrNormalizerOverSampled = new cuNormalizer( + param->searchWindowSizeHeight, + param->searchWindowSizeWidth, + param->numberWindowDownInChunk * param->numberWindowAcrossInChunk + ); + #ifdef CUAMPCOR_DEBUG std::cout << "all objects in chunk are created ...\n"; diff --git a/contrib/PyCuAmpcor/src/cuAmpcorChunk.h b/contrib/PyCuAmpcor/src/cuAmpcorChunk.h index 0476282..99cd9ee 100644 --- a/contrib/PyCuAmpcor/src/cuAmpcorChunk.h +++ b/contrib/PyCuAmpcor/src/cuAmpcorChunk.h @@ -14,6 +14,7 @@ #include "cuOverSampler.h" #include "cuSincOverSampler.h" #include "cuCorrFrequency.h" +#include "cuCorrNormalizer.h" /** @@ -64,6 +65,10 @@ private: // cross-correlation processor with frequency domain algorithm cuFreqCorrelator *cuCorrFreqDomain, *cuCorrFreqDomain_OverSampled; + // correlation surface normalizer + cuNormalizer *corrNormalizerRaw; + cuNormalizer *corrNormalizerOverSampled; + // save offset results in different stages cuArrays *offsetInit; cuArrays *offsetZoomIn; diff --git a/contrib/PyCuAmpcor/src/cuAmpcorController.cu b/contrib/PyCuAmpcor/src/cuAmpcorController.cu index 625a6a4..1e04d03 100644 --- a/contrib/PyCuAmpcor/src/cuAmpcorController.cu +++ b/contrib/PyCuAmpcor/src/cuAmpcorController.cu @@ -102,7 +102,7 @@ void cuAmpcorController::runAmpcor() << nChunksDown << " x " << nChunksAcross << std::endl; // iterative over chunks down - int message_interval = nChunksDown/10; + int message_interval = std::max(nChunksDown/10, 1); for(int i = 0; i *images, cudaStream_t stream); void cuArraysPadding(cuArrays *image1, cuArrays *image2, cudaStream_t stream); void cuArraysPaddingMany(cuArrays *image1, cuArrays *image2, cudaStream_t stream); -//in cuCorrNormalization.cu: utities to normalize the cross correlation function +//in cuCorrNormalization.cu: utilities to normalize the cross correlation function void cuArraysSubtractMean(cuArrays *images, cudaStream_t stream); void cuCorrNormalize(cuArrays *templates, cuArrays *images, cuArrays *results, cudaStream_t stream); +void cuCorrNormalize64(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); +void cuCorrNormalize128(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); +void cuCorrNormalize256(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); +void cuCorrNormalize512(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); +void cuCorrNormalize1024(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); + +// in cuCorrNormalizationSAT.cu: to normalize the cross correlation function with sum area table +void cuCorrNormalizeSAT(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, + cuArrays * referenceSum2, cuArrays *secondarySAT, cuArrays *secondarySAT2, cudaStream_t stream); //in cuOffset.cu: utitilies for determining the max locaiton of cross correlations or the offset void cuArraysMaxloc2D(cuArrays *images, cuArrays *maxloc, cuArrays *maxval, cudaStream_t stream); diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalization.cu b/contrib/PyCuAmpcor/src/cuCorrNormalization.cu index 5ab49c1..e32fa5e 100644 --- a/contrib/PyCuAmpcor/src/cuCorrNormalization.cu +++ b/contrib/PyCuAmpcor/src/cuCorrNormalization.cu @@ -321,7 +321,6 @@ __global__ void cuCorrNormalize_kernel( * @param[in] stream cudaStream * @warning The current implementation uses one thread for one column, therefore, * the secondary window width is limited to <=1024, the max threads in a block. - * @todo an implementation for arbitrary window width, might not be as efficient */ void cuCorrNormalize(cuArrays *templates, cuArrays *images, cuArrays *results, cudaStream_t stream) { @@ -376,7 +375,72 @@ void cuCorrNormalize(cuArrays *templates, cuArrays *images, cuArra throw; } - } +void cuCorrNormalize64(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + const int nImages = correlation->count; + const dim3 grid(1, 1, nImages); + const float invReferenceSize = 1.0f/reference->size; + cuCorrNormalize_kernel< 6><<>>(nImages, + reference->devData, reference->height, reference->width, reference->size, + secondary->devData, secondary->height, secondary->width, secondary->size, + correlation->devData, correlation->height, correlation->width, correlation->size, + invReferenceSize); + getLastCudaError("cuCorrNormalize kernel error"); +} + +void cuCorrNormalize128(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + const int nImages = correlation->count; + const dim3 grid(1, 1, nImages); + const float invReferenceSize = 1.0f/reference->size; + cuCorrNormalize_kernel< 7><<>>(nImages, + reference->devData, reference->height, reference->width, reference->size, + secondary->devData, secondary->height, secondary->width, secondary->size, + correlation->devData, correlation->height, correlation->width, correlation->size, + invReferenceSize); + getLastCudaError("cuCorrNormalize kernel error"); +} + +void cuCorrNormalize256(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + const int nImages = correlation->count; + const dim3 grid(1, 1, nImages); + const float invReferenceSize = 1.0f/reference->size; + cuCorrNormalize_kernel< 8><<>>(nImages, + reference->devData, reference->height, reference->width, reference->size, + secondary->devData, secondary->height, secondary->width, secondary->size, + correlation->devData, correlation->height, correlation->width, correlation->size, + invReferenceSize); + getLastCudaError("cuCorrNormalize kernel error"); +} + +void cuCorrNormalize512(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + const int nImages = correlation->count; + const dim3 grid(1, 1, nImages); + const float invReferenceSize = 1.0f/reference->size; + cuCorrNormalize_kernel< 9><<>>(nImages, + reference->devData, reference->height, reference->width, reference->size, + secondary->devData, secondary->height, secondary->width, secondary->size, + correlation->devData, correlation->height, correlation->width, correlation->size, + invReferenceSize); + getLastCudaError("cuCorrNormalize kernel error"); +} + +void cuCorrNormalize1024(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + const int nImages = correlation->count; + const dim3 grid(1, 1, nImages); + const float invReferenceSize = 1.0f/reference->size; + cuCorrNormalize_kernel< 10><<>>(nImages, + reference->devData, reference->height, reference->width, reference->size, + secondary->devData, secondary->height, secondary->width, secondary->size, + correlation->devData, correlation->height, correlation->width, correlation->size, + invReferenceSize); + getLastCudaError("cuCorrNormalize kernel error"); +} + + // end of file diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalizationSAT.cu b/contrib/PyCuAmpcor/src/cuCorrNormalizationSAT.cu new file mode 100644 index 0000000..20bd1e4 --- /dev/null +++ b/contrib/PyCuAmpcor/src/cuCorrNormalizationSAT.cu @@ -0,0 +1,201 @@ +/* + * @file cuCorrNormalizationSAT.cu + * @brief Utilities to normalize the 2D correlation surface with the sum area table + * + */ + +#include +#include +// my declarations +#include "cuAmpcorUtil.h" +// for FLT_EPSILON +#include + +// alias for cuda cooperative groups +namespace cg = cooperative_groups; + + +/** + * cuda kernel for sum value^2 (std) + * compute the sum value square (std) of the reference image + * @param[out] sum2 sum of value square + * @param[in] images the reference images + * @param[in] n total elements in one image nx*ny + * @param[in] batch number of images + * @note use one thread block for each image, blockIdx.x is image index + **/ + +__global__ void sum_square_kernel(float *sum2, const float *images, int n, int batch) +{ + // get block id for each image + int imageid = blockIdx.x; + const float *image = images + imageid*n; + + // get the thread block + cg::thread_block cta = cg::this_thread_block(); + // get the shared memory + extern float __shared__ sdata[]; + + // get the current thread + int tid = cta.thread_rank(); + + // stride over grid and add the values to shared memory + sdata[tid] = 0; + + for(int i = tid; i < n; i += cta.size() ) { + auto value = image[i]; + sdata[tid] += value*value; + } + + cg::sync(cta); + + // partition thread block into tiles in size 32 (warp) + cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta); + + // reduce in each tile with warp + sdata[tid] = cg::reduce(tile32, sdata[tid], cg::plus()); + cg::sync(cta); + + // reduce all tiles with thread 0 + if(tid == 0) { + float sum = 0.0; + for (int i = 0; i < cta.size(); i += tile32.size()) + sum += sdata[i]; + // assign the value to results + sum2[imageid] = sum; + } +} + +/** + * cuda kernel for 2d sum area table + * Compute the (inclusive) sum area table of the value and value^2 of a batch of 2d images. + * @param[out] sat the sum area table + * @param[out] sat2 the sum area table of value^2 + * @param[in] data search images + * @param[in] nx image height (subleading dimension) + * @param[in] ny image width (leading dimension) + * @param[in] batch number of images + **/ + +__global__ void sat2d_kernel(float *sat, float * sat2, const float *data, int nx, int ny, int batch) +{ + // get block id for each image + int imageid = blockIdx.x; + + // get the thread id for each row/column + int tid = threadIdx.x; + + // compute prefix-sum along row at first + // the number of rows may be bigger than the number of threads, iterate + for (int row = tid; row < nx; row += blockDim.x) { + // running sum for value and value^2 + float sum = 0.0f; + float sum2 = 0.0f; + // starting position for this row + int index = (imageid*nx+row)*ny; + // iterative over column + for (int i=0; i 0 && ty > 0) ? sat[(tx-1)*secondaryNY+(ty-1)] : 0.0; + float topright = (tx > 0 ) ? sat[(tx-1)*secondaryNY+(ty+referenceNY-1)] : 0.0; + float bottomleft = (ty > 0) ? sat[(tx+referenceNX-1)*secondaryNY+(ty-1)] : 0.0; + float bottomright = sat[(tx+referenceNX-1)*secondaryNY+(ty+referenceNY-1)]; + // get the sum + float secondarySum = bottomright + topleft - topright - bottomleft; + // sum of value^2 + const float *sat2 = secondarySat2 + imageid*secondaryNX*secondaryNY; + // get sat2 values for four corners + topleft = (tx > 0 && ty > 0) ? sat2[(tx-1)*secondaryNY+(ty-1)] : 0.0; + topright = (tx > 0 ) ? sat2[(tx-1)*secondaryNY+(ty+referenceNY-1)] : 0.0; + bottomleft = (ty > 0) ? sat2[(tx+referenceNX-1)*secondaryNY+(ty-1)] : 0.0; + bottomright = sat2[(tx+referenceNX-1)*secondaryNY+(ty+referenceNY-1)]; + float secondarySum2 = bottomright + topleft - topright - bottomleft; + + // compute the normalization + float norm2 = (secondarySum2-secondarySum*secondarySum/(referenceNX*referenceNY))*refSum2; + // normalize the correlation surface + correlation[(imageid*corNX+tx)*corNY+ty] *= rsqrtf(norm2 + FLT_EPSILON); + } +} + + +void cuCorrNormalizeSAT(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, + cuArrays * referenceSum2, cuArrays *secondarySat, cuArrays *secondarySat2, cudaStream_t stream) +{ + // compute the std of reference image + // note that the mean is already subtracted + int nthreads = 256; + int sMemSize = nthreads*sizeof(float); + int nblocks = reference->count; + sum_square_kernel<<>>(referenceSum2->devData, reference->devData, + reference->width * reference->height, reference->count); + getLastCudaError("reference image sum_square kernel error"); + + // compute the sum area table of the search images + sat2d_kernel<<>>(secondarySat->devData, secondarySat2->devData, secondary->devData, + secondary->height, secondary->width, secondary->count); + getLastCudaError("search image sat kernel error"); + + nthreads = NTHREADS2D; + dim3 blockSize(nthreads, nthreads, 1); + dim3 gridSize(IDIVUP(correlation->height,nthreads), IDIVUP(correlation->width,nthreads), correlation->count); + cuCorrNormalizeSAT_kernel<<>>(correlation->devData, + referenceSum2->devData, secondarySat->devData, secondarySat2->devData, + correlation->height, correlation->width, + reference->height, reference->width, + secondary->height, secondary->width); + getLastCudaError("cuCorrNormalizeSAT_kernel kernel error"); +} \ No newline at end of file diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu b/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu new file mode 100644 index 0000000..b5058b6 --- /dev/null +++ b/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu @@ -0,0 +1,107 @@ +/* + * @file cuNormalizer.cu + * @brief processors to normalize the correlation surface + * + */ + +#include "cuCorrNormalizer.h" +#include "cuAmpcorUtil.h" + +cuNormalizer::cuNormalizer(int secondaryNX, int secondaryNY, int count) +{ + // depending on NY, choose different processor + if(secondaryNY <= 64) { + processor = new cuNormalize64(); + } + else if (secondaryNY <= 128) { + processor = new cuNormalize128(); + } + else if (secondaryNY <= 256) { + processor = new cuNormalize256(); + } + else if (secondaryNY <= 512) { + processor = new cuNormalize512(); + } + else if (secondaryNY <= 1024) { + processor = new cuNormalize1024(); + } + else { + processor = new cuNormalizeSAT(secondaryNX, secondaryNY, count); + } +} + +cuNormalizer::~cuNormalizer() +{ + delete processor; +} + +void cuNormalizer::execute(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + processor->execute(correlation, reference, secondary, stream); +} + +/** + * + * + **/ + +cuNormalizeSAT::cuNormalizeSAT(int secondaryNX, int secondaryNY, int count) +{ + // allocate the work array + // reference sum square + referenceSum2 = new cuArrays(1, 1, count); + referenceSum2->allocate(); + + // secondary sum and sum square + secondarySAT = new cuArrays(secondaryNX, secondaryNY, count); + secondarySAT->allocate(); + secondarySAT2 = new cuArrays(secondaryNX, secondaryNY, count); + secondarySAT2->allocate(); +}; + +cuNormalizeSAT::~cuNormalizeSAT() +{ + delete referenceSum2; + delete secondarySAT; + delete secondarySAT2; +} + +void cuNormalizeSAT::execute(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + cuCorrNormalizeSAT(correlation, reference, secondary, + referenceSum2, secondarySAT, secondarySAT2, stream); +} + +void cuNormalize64::execute(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + cuCorrNormalize64(correlation, reference, secondary, stream); +} + +void cuNormalize128::execute(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + cuCorrNormalize128(correlation, reference, secondary, stream); +} + +void cuNormalize256::execute(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + cuCorrNormalize256(correlation, reference, secondary, stream); +} + +void cuNormalize512::execute(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + cuCorrNormalize512(correlation, reference, secondary, stream); +} + +void cuNormalize1024::execute(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +{ + cuCorrNormalize1024(correlation, reference, secondary, stream); +} + +// end of file \ No newline at end of file diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalizer.h b/contrib/PyCuAmpcor/src/cuCorrNormalizer.h new file mode 100644 index 0000000..aa45a16 --- /dev/null +++ b/contrib/PyCuAmpcor/src/cuCorrNormalizer.h @@ -0,0 +1,91 @@ +/* + * @file cuNormalizer.h + * @brief normalize the correlation surface + * + * cuNormalizeProcessor is an abstract class for processors to normalize the correlation surface. + * It has different implementations wrt different image sizes. + * cuNormalize64, 128, ... 1024 use a shared memory accelerated algorithm, which are limited by the number of cuda threads in a block. + * cuNormalizeSAT uses the sum area table based algorithm, which applies to any size (used for >1024). + * cuNormalizer is a wrapper class which determines which processor to use. + */ + +#ifndef __CUNORMALIZER_H +#define __CUNORMALIZER_H + +#include "cuArrays.h" +#include "cudaUtil.h" + +/** + * Abstract class interface for correlation surface normalization processor + * with different implementations + */ +class cuNormalizeProcessor { +public: + // default constructor and destructor + cuNormalizeProcessor() {} + ~cuNormalizeProcessor() {} + // execute interface + virtual void execute(cuArrays * correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) = 0; +}; + +class cuNormalizer { +private: + cuNormalizeProcessor *processor; +public: + // disable the default constructor + cuNormalizer() = delete; + // constructor with the secondary dimension + cuNormalizer(int secondaryNX, int secondaryNY, int count); + // destructor + ~cuNormalizer(); + // execute correlation surface normalization + void execute(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, + cudaStream_t stream); +}; + + +class cuNormalize64 : public cuNormalizeProcessor +{ +public: + void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; +}; + +class cuNormalize128 : public cuNormalizeProcessor +{ +public: + void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; +}; + +class cuNormalize256 : public cuNormalizeProcessor +{ +public: + void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; +}; + +class cuNormalize512 : public cuNormalizeProcessor +{ +public: + void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; +}; + +class cuNormalize1024 : public cuNormalizeProcessor +{ +public: + void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; +}; + +class cuNormalizeSAT : public cuNormalizeProcessor +{ +private: + cuArrays *referenceSum2; + cuArrays *secondarySAT; + cuArrays *secondarySAT2; + +public: + cuNormalizeSAT(int secondaryNX, int secondaryNY, int count); + ~cuNormalizeSAT(); + void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; +}; + +#endif +// end of file \ No newline at end of file From b0641ef2f59ccaf8736dabe9493a474cc3539a50 Mon Sep 17 00:00:00 2001 From: Lijun Zhu Date: Tue, 11 May 2021 15:32:47 -0700 Subject: [PATCH 17/19] PyCuAmpcor: add an implementation of cuCorrNormalizationSAT to support cuda 9/10 --- .../PyCuAmpcor/src/cuCorrNormalizationSAT.cu | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalizationSAT.cu b/contrib/PyCuAmpcor/src/cuCorrNormalizationSAT.cu index 20bd1e4..7f7c1bd 100644 --- a/contrib/PyCuAmpcor/src/cuCorrNormalizationSAT.cu +++ b/contrib/PyCuAmpcor/src/cuCorrNormalizationSAT.cu @@ -5,7 +5,11 @@ */ #include + +#if __CUDACC_VER_MAJOR__ >= 11 #include +#endif + // my declarations #include "cuAmpcorUtil.h" // for FLT_EPSILON @@ -25,6 +29,9 @@ namespace cg = cooperative_groups; * @note use one thread block for each image, blockIdx.x is image index **/ + +#if __CUDACC_VER_MAJOR__ >= 11 +// use cg::reduce for NVCC 11 and above __global__ void sum_square_kernel(float *sum2, const float *images, int n, int batch) { // get block id for each image @@ -66,6 +73,69 @@ __global__ void sum_square_kernel(float *sum2, const float *images, int n, int b } } +#else +// use warp-shuffle reduction for NVCC 9 & 10 +__global__ void sum_square_kernel(float *sum2, const float *images, int n, int batch) +{ + // get block id for each image + int imageid = blockIdx.x; + const float *image = images + imageid*n; + + // get the thread block + cg::thread_block cta = cg::this_thread_block(); + // get the shared memory + extern float __shared__ sdata[]; + + // get the current thread + unsigned int tid = cta.thread_rank(); + unsigned int blockSize = cta.size(); + + // stride over grid and add the values to the shared memory + float sum = 0; + + for(int i = tid; i < n; i += blockSize ) { + auto value = image[i]; + sum += value*value; + } + sdata[tid] = sum; + cg::sync(cta); + + // do reduction in shared memory in log2 steps + if ((blockSize >= 512) && (tid < 256)) { + sdata[tid] = sum = sum + sdata[tid + 256]; + } + cg::sync(cta); + + if ((blockSize >= 256) && (tid < 128)) { + sdata[tid] = sum = sum + sdata[tid + 128]; + } + cg::sync(cta); + + if ((blockSize >= 128) && (tid < 64)) { + sdata[tid] = sum = sum + sdata[tid + 64]; + } + cg::sync(cta); + + // partition thread block into tiles in size 32 (warp) + cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta); + + // reduce within warp + if(tid < 32) { + if(blockSize >=64) sum += sdata[tid + 32]; + for (int offset = tile32.size()/2; offset >0; offset /=2) { + sum += tile32.shfl_down(sum, offset); + } + } + + // return results with thread 0 + if(tid == 0) { + // assign the value to results + sum2[imageid] = sum; + } +} +#endif // __CUDACC_VER_MAJOR check + + /** * cuda kernel for 2d sum area table * Compute the (inclusive) sum area table of the value and value^2 of a batch of 2d images. From 14d708863de69107bbad04b6c29c6d8f0433e8cd Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Mon, 19 Jul 2021 16:34:35 -0700 Subject: [PATCH 18/19] Templatize normalization routines --- contrib/PyCuAmpcor/src/cuAmpcorUtil.h | 8 +- contrib/PyCuAmpcor/src/cuCorrNormalization.cu | 79 ++++++------------- contrib/PyCuAmpcor/src/cuCorrNormalizer.cu | 45 ++++------- contrib/PyCuAmpcor/src/cuCorrNormalizer.h | 31 +------- 4 files changed, 46 insertions(+), 117 deletions(-) diff --git a/contrib/PyCuAmpcor/src/cuAmpcorUtil.h b/contrib/PyCuAmpcor/src/cuAmpcorUtil.h index ea49534..69f09ce 100644 --- a/contrib/PyCuAmpcor/src/cuAmpcorUtil.h +++ b/contrib/PyCuAmpcor/src/cuAmpcorUtil.h @@ -61,11 +61,9 @@ void cuArraysPaddingMany(cuArrays *image1, cuArrays *image2, cud //in cuCorrNormalization.cu: utilities to normalize the cross correlation function void cuArraysSubtractMean(cuArrays *images, cudaStream_t stream); void cuCorrNormalize(cuArrays *templates, cuArrays *images, cuArrays *results, cudaStream_t stream); -void cuCorrNormalize64(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); -void cuCorrNormalize128(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); -void cuCorrNormalize256(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); -void cuCorrNormalize512(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); -void cuCorrNormalize1024(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); + +template +void cuCorrNormalizeFixed(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream); // in cuCorrNormalizationSAT.cu: to normalize the cross correlation function with sum area table void cuCorrNormalizeSAT(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalization.cu b/contrib/PyCuAmpcor/src/cuCorrNormalization.cu index e32fa5e..b6796d2 100644 --- a/contrib/PyCuAmpcor/src/cuCorrNormalization.cu +++ b/contrib/PyCuAmpcor/src/cuCorrNormalization.cu @@ -377,64 +377,20 @@ void cuCorrNormalize(cuArrays *templates, cuArrays *images, cuArra } -void cuCorrNormalize64(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) +template struct Log2; +template<> struct Log2<64> { static const int value = 6; }; +template<> struct Log2<128> { static const int value = 7; }; +template<> struct Log2<256> { static const int value = 8; }; +template<> struct Log2<512> { static const int value = 9; }; +template<> struct Log2<1024> { static const int value = 10; }; + +template +void cuCorrNormalizeFixed(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) { const int nImages = correlation->count; const dim3 grid(1, 1, nImages); const float invReferenceSize = 1.0f/reference->size; - cuCorrNormalize_kernel< 6><<>>(nImages, - reference->devData, reference->height, reference->width, reference->size, - secondary->devData, secondary->height, secondary->width, secondary->size, - correlation->devData, correlation->height, correlation->width, correlation->size, - invReferenceSize); - getLastCudaError("cuCorrNormalize kernel error"); -} - -void cuCorrNormalize128(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) -{ - const int nImages = correlation->count; - const dim3 grid(1, 1, nImages); - const float invReferenceSize = 1.0f/reference->size; - cuCorrNormalize_kernel< 7><<>>(nImages, - reference->devData, reference->height, reference->width, reference->size, - secondary->devData, secondary->height, secondary->width, secondary->size, - correlation->devData, correlation->height, correlation->width, correlation->size, - invReferenceSize); - getLastCudaError("cuCorrNormalize kernel error"); -} - -void cuCorrNormalize256(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) -{ - const int nImages = correlation->count; - const dim3 grid(1, 1, nImages); - const float invReferenceSize = 1.0f/reference->size; - cuCorrNormalize_kernel< 8><<>>(nImages, - reference->devData, reference->height, reference->width, reference->size, - secondary->devData, secondary->height, secondary->width, secondary->size, - correlation->devData, correlation->height, correlation->width, correlation->size, - invReferenceSize); - getLastCudaError("cuCorrNormalize kernel error"); -} - -void cuCorrNormalize512(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) -{ - const int nImages = correlation->count; - const dim3 grid(1, 1, nImages); - const float invReferenceSize = 1.0f/reference->size; - cuCorrNormalize_kernel< 9><<>>(nImages, - reference->devData, reference->height, reference->width, reference->size, - secondary->devData, secondary->height, secondary->width, secondary->size, - correlation->devData, correlation->height, correlation->width, correlation->size, - invReferenceSize); - getLastCudaError("cuCorrNormalize kernel error"); -} - -void cuCorrNormalize1024(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) -{ - const int nImages = correlation->count; - const dim3 grid(1, 1, nImages); - const float invReferenceSize = 1.0f/reference->size; - cuCorrNormalize_kernel< 10><<>>(nImages, + cuCorrNormalize_kernel::value><<>>(nImages, reference->devData, reference->height, reference->width, reference->size, secondary->devData, secondary->height, secondary->width, secondary->size, correlation->devData, correlation->height, correlation->width, correlation->size, @@ -442,5 +398,20 @@ void cuCorrNormalize1024(cuArrays *correlation, cuArrays *referenc getLastCudaError("cuCorrNormalize kernel error"); } +template void cuCorrNormalizeFixed<64>(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, + cudaStream_t stream); +template void cuCorrNormalizeFixed<128>(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, + cudaStream_t stream); +template void cuCorrNormalizeFixed<256>(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, + cudaStream_t stream); +template void cuCorrNormalizeFixed<512>(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, + cudaStream_t stream); +template void cuCorrNormalizeFixed<1024>(cuArrays *correlation, + cuArrays *reference, cuArrays *secondary, + cudaStream_t stream); // end of file diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu b/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu index b5058b6..783080d 100644 --- a/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu +++ b/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu @@ -11,19 +11,19 @@ cuNormalizer::cuNormalizer(int secondaryNX, int secondaryNY, int count) { // depending on NY, choose different processor if(secondaryNY <= 64) { - processor = new cuNormalize64(); + processor = new cuNormalizeFixed<64>(); } else if (secondaryNY <= 128) { - processor = new cuNormalize128(); + processor = new cuNormalizeFixed<128>(); } else if (secondaryNY <= 256) { - processor = new cuNormalize256(); + processor = new cuNormalizeFixed<256>(); } else if (secondaryNY <= 512) { - processor = new cuNormalize512(); + processor = new cuNormalizeFixed<512>(); } else if (secondaryNY <= 1024) { - processor = new cuNormalize1024(); + processor = new cuNormalizeFixed<1024>(); } else { processor = new cuNormalizeSAT(secondaryNX, secondaryNY, count); @@ -74,34 +74,17 @@ void cuNormalizeSAT::execute(cuArrays *correlation, referenceSum2, secondarySAT, secondarySAT2, stream); } -void cuNormalize64::execute(cuArrays *correlation, +template +void cuNormalizeFixed::execute(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) { - cuCorrNormalize64(correlation, reference, secondary, stream); + cuCorrNormalizeFixed(correlation, reference, secondary, stream); } -void cuNormalize128::execute(cuArrays *correlation, - cuArrays *reference, cuArrays *secondary, cudaStream_t stream) -{ - cuCorrNormalize128(correlation, reference, secondary, stream); -} +template class cuNormalizeFixed<64>; +template class cuNormalizeFixed<128>; +template class cuNormalizeFixed<256>; +template class cuNormalizeFixed<512>; +template class cuNormalizeFixed<1024>; -void cuNormalize256::execute(cuArrays *correlation, - cuArrays *reference, cuArrays *secondary, cudaStream_t stream) -{ - cuCorrNormalize256(correlation, reference, secondary, stream); -} - -void cuNormalize512::execute(cuArrays *correlation, - cuArrays *reference, cuArrays *secondary, cudaStream_t stream) -{ - cuCorrNormalize512(correlation, reference, secondary, stream); -} - -void cuNormalize1024::execute(cuArrays *correlation, - cuArrays *reference, cuArrays *secondary, cudaStream_t stream) -{ - cuCorrNormalize1024(correlation, reference, secondary, stream); -} - -// end of file \ No newline at end of file +// end of file diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalizer.h b/contrib/PyCuAmpcor/src/cuCorrNormalizer.h index aa45a16..043accd 100644 --- a/contrib/PyCuAmpcor/src/cuCorrNormalizer.h +++ b/contrib/PyCuAmpcor/src/cuCorrNormalizer.h @@ -4,7 +4,7 @@ * * cuNormalizeProcessor is an abstract class for processors to normalize the correlation surface. * It has different implementations wrt different image sizes. - * cuNormalize64, 128, ... 1024 use a shared memory accelerated algorithm, which are limited by the number of cuda threads in a block. + * cuNormalizeFixed<64/128/.../1024> use a shared memory accelerated algorithm, which are limited by the number of cuda threads in a block. * cuNormalizeSAT uses the sum area table based algorithm, which applies to any size (used for >1024). * cuNormalizer is a wrapper class which determines which processor to use. */ @@ -44,31 +44,8 @@ public: }; -class cuNormalize64 : public cuNormalizeProcessor -{ -public: - void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; -}; - -class cuNormalize128 : public cuNormalizeProcessor -{ -public: - void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; -}; - -class cuNormalize256 : public cuNormalizeProcessor -{ -public: - void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; -}; - -class cuNormalize512 : public cuNormalizeProcessor -{ -public: - void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; -}; - -class cuNormalize1024 : public cuNormalizeProcessor +template +class cuNormalizeFixed : public cuNormalizeProcessor { public: void execute(cuArrays * correlation, cuArrays *reference, cuArrays *search, cudaStream_t stream) override; @@ -88,4 +65,4 @@ public: }; #endif -// end of file \ No newline at end of file +// end of file From a0ce9d55cb39999c91bd42ca9eae305e498ddce0 Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Mon, 19 Jul 2021 16:53:22 -0700 Subject: [PATCH 19/19] Replace cuNormalizer holder class with unique_ptr --- contrib/PyCuAmpcor/src/cuAmpcorChunk.cu | 9 ++++--- contrib/PyCuAmpcor/src/cuAmpcorChunk.h | 4 +-- contrib/PyCuAmpcor/src/cuCorrNormalizer.cu | 31 ++++++---------------- contrib/PyCuAmpcor/src/cuCorrNormalizer.h | 20 +++----------- 4 files changed, 19 insertions(+), 45 deletions(-) diff --git a/contrib/PyCuAmpcor/src/cuAmpcorChunk.cu b/contrib/PyCuAmpcor/src/cuAmpcorChunk.cu index 8ac887e..fda5f89 100644 --- a/contrib/PyCuAmpcor/src/cuAmpcorChunk.cu +++ b/contrib/PyCuAmpcor/src/cuAmpcorChunk.cu @@ -553,17 +553,18 @@ cuAmpcorChunk::cuAmpcorChunk(cuAmpcorParameter *param_, GDALImage *reference_, G stream); } - corrNormalizerRaw = new cuNormalizer( + corrNormalizerRaw = std::unique_ptr(newCuNormalizer( param->searchWindowSizeHeightRaw, param->searchWindowSizeWidthRaw, param->numberWindowDownInChunk * param->numberWindowAcrossInChunk - ); + )); - corrNormalizerOverSampled = new cuNormalizer( + corrNormalizerOverSampled = + std::unique_ptr(newCuNormalizer( param->searchWindowSizeHeight, param->searchWindowSizeWidth, param->numberWindowDownInChunk * param->numberWindowAcrossInChunk - ); + )); #ifdef CUAMPCOR_DEBUG diff --git a/contrib/PyCuAmpcor/src/cuAmpcorChunk.h b/contrib/PyCuAmpcor/src/cuAmpcorChunk.h index 99cd9ee..0e853ac 100644 --- a/contrib/PyCuAmpcor/src/cuAmpcorChunk.h +++ b/contrib/PyCuAmpcor/src/cuAmpcorChunk.h @@ -66,8 +66,8 @@ private: cuFreqCorrelator *cuCorrFreqDomain, *cuCorrFreqDomain_OverSampled; // correlation surface normalizer - cuNormalizer *corrNormalizerRaw; - cuNormalizer *corrNormalizerOverSampled; + std::unique_ptr corrNormalizerRaw; + std::unique_ptr corrNormalizerOverSampled; // save offset results in different stages cuArrays *offsetInit; diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu b/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu index 783080d..7391203 100644 --- a/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu +++ b/contrib/PyCuAmpcor/src/cuCorrNormalizer.cu @@ -7,45 +7,30 @@ #include "cuCorrNormalizer.h" #include "cuAmpcorUtil.h" -cuNormalizer::cuNormalizer(int secondaryNX, int secondaryNY, int count) +cuNormalizeProcessor* +newCuNormalizer(int secondaryNX, int secondaryNY, int count) { // depending on NY, choose different processor if(secondaryNY <= 64) { - processor = new cuNormalizeFixed<64>(); + return new cuNormalizeFixed<64>(); } else if (secondaryNY <= 128) { - processor = new cuNormalizeFixed<128>(); + return new cuNormalizeFixed<128>(); } else if (secondaryNY <= 256) { - processor = new cuNormalizeFixed<256>(); + return new cuNormalizeFixed<256>(); } else if (secondaryNY <= 512) { - processor = new cuNormalizeFixed<512>(); + return new cuNormalizeFixed<512>(); } else if (secondaryNY <= 1024) { - processor = new cuNormalizeFixed<1024>(); + return new cuNormalizeFixed<1024>(); } else { - processor = new cuNormalizeSAT(secondaryNX, secondaryNY, count); + return new cuNormalizeSAT(secondaryNX, secondaryNY, count); } } -cuNormalizer::~cuNormalizer() -{ - delete processor; -} - -void cuNormalizer::execute(cuArrays *correlation, - cuArrays *reference, cuArrays *secondary, cudaStream_t stream) -{ - processor->execute(correlation, reference, secondary, stream); -} - -/** - * - * - **/ - cuNormalizeSAT::cuNormalizeSAT(int secondaryNX, int secondaryNY, int count) { // allocate the work array diff --git a/contrib/PyCuAmpcor/src/cuCorrNormalizer.h b/contrib/PyCuAmpcor/src/cuCorrNormalizer.h index 043accd..e042d4b 100644 --- a/contrib/PyCuAmpcor/src/cuCorrNormalizer.h +++ b/contrib/PyCuAmpcor/src/cuCorrNormalizer.h @@ -22,26 +22,14 @@ class cuNormalizeProcessor { public: // default constructor and destructor - cuNormalizeProcessor() {} - ~cuNormalizeProcessor() {} + cuNormalizeProcessor() = default; + virtual ~cuNormalizeProcessor() = default; // execute interface virtual void execute(cuArrays * correlation, cuArrays *reference, cuArrays *secondary, cudaStream_t stream) = 0; }; -class cuNormalizer { -private: - cuNormalizeProcessor *processor; -public: - // disable the default constructor - cuNormalizer() = delete; - // constructor with the secondary dimension - cuNormalizer(int secondaryNX, int secondaryNY, int count); - // destructor - ~cuNormalizer(); - // execute correlation surface normalization - void execute(cuArrays *correlation, cuArrays *reference, cuArrays *secondary, - cudaStream_t stream); -}; +// factory with the secondary dimension +cuNormalizeProcessor* newCuNormalizer(int NX, int NY, int count); template