KJ135
/
ISCE_INSAR
mirror of http://172.16.0.12:5000/WBJY/ISCE_INSAR.git
4
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bash script edits to make stackSentinel.py faster in places (#225) 

* plumb in num-process for time consuming steps

* refine num-process speed enhancements

* refine num-process speed enhancements

* dedicated function for writing batch processes

* refine num-process speed enhancements

* fix indents

Co-authored-by: Russell Grew <russell.grew@douglaspartners.com.au>
LT1AB
Russell Grew 2021-02-05 08:58:36 +11:00 committed by GitHub
parent 0c54c9d422
commit e8da2d947a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 107 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
contrib/stack/topsStack/Stack.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
######################## ########################
#Author: Heresh Fattahi #Author: Heresh Fattahi
@ -13,6 +13,7 @@ defoMax = '2'
maxNodes = 72 maxNodes = 72
class config(object): class config(object):
""" """
A class representing the config file A class representing the config file
@ -242,6 +243,7 @@ class config(object):
self.f.write('defomax : ' + self.defoMax + '\n') self.f.write('defomax : ' + self.defoMax + '\n')
self.f.write('rlks : ' + self.rangeLooks + '\n') self.f.write('rlks : ' + self.rangeLooks + '\n')
self.f.write('alks : ' + self.azimuthLooks + '\n') self.f.write('alks : ' + self.azimuthLooks + '\n')
self.f.write('numProcess : ' + self.numProcess + '\n')
def denseOffset(self, function): def denseOffset(self, function):
self.f.write('###################################'+'\n') self.f.write('###################################'+'\n')
@ -258,6 +260,18 @@ class config(object):
#self.f.write('ww : 256\n') #self.f.write('ww : 256\n')
#self.f.write('wh : 128\n') #self.f.write('wh : 128\n')
def write_wrapper_config2run_file(self, configName, line_cnt, numProcess = 1):
# dispassionate list of commands for single process
if numProcess == 1:
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
# aggregate background commands between wait blocks for speed gains
elif numProcess > 1:
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + ' &\n')
if line_cnt == numProcess:
self.runf.write('wait\n\n')
line_cnt = 0
return line_cnt
def finalize(self): def finalize(self):
self.f.close() self.f.close()
@ -286,6 +300,7 @@ class run(object):
swath_path = self.work_dir swath_path = self.work_dir
os.makedirs(self.config_path, exist_ok=True) os.makedirs(self.config_path, exist_ok=True)
line_cnt = 0
for slcdate in acquisitionDates: for slcdate in acquisitionDates:
configName = os.path.join(self.config_path,'config_unpack_'+slcdate) configName = os.path.join(self.config_path,'config_unpack_'+slcdate)
configObj = config(configName) configObj = config(configName)
@ -300,8 +315,10 @@ class run(object):
configObj.Sentinel1_TOPS('[Function-1]') configObj.Sentinel1_TOPS('[Function-1]')
configObj.topo('[Function-2]') configObj.topo('[Function-2]')
configObj.finalize() configObj.finalize()
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
def unpackStackReferenceSLC(self, safe_dict): def unpackStackReferenceSLC(self, safe_dict):
swath_path = self.work_dir swath_path = self.work_dir
@ -320,11 +337,14 @@ class run(object):
configObj.Sentinel1_TOPS('[Function-1]') configObj.Sentinel1_TOPS('[Function-1]')
configObj.topo('[Function-2]') configObj.topo('[Function-2]')
configObj.finalize() configObj.finalize()
line_cnt = 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict): def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict):
line_cnt = 0
for secondary in secondaryList: for secondary in secondaryList:
configName = os.path.join(self.config_path,'config_secondary_'+secondary) configName = os.path.join(self.config_path,'config_secondary_'+secondary)
outdir = os.path.join(self.work_dir,'secondarys/'+secondary) outdir = os.path.join(self.work_dir,'secondarys/'+secondary)
@ -337,11 +357,14 @@ class run(object):
configObj.outDir = outdir configObj.outDir = outdir
configObj.Sentinel1_TOPS('[Function-1]') configObj.Sentinel1_TOPS('[Function-1]')
configObj.finalize() configObj.finalize()
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
del configObj del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName+'\n')
def averageBaseline(self, stackReferenceDate, secondaryList): def averageBaseline(self, stackReferenceDate, secondaryList):
line_cnt = 0
for secondary in secondaryList: for secondary in secondaryList:
configName = os.path.join(self.config_path,'config_baseline_'+secondary) configName = os.path.join(self.config_path,'config_baseline_'+secondary)
configObj = config(configName) configObj = config(configName)
@ -351,10 +374,14 @@ class run(object):
configObj.baselineFile = os.path.join(self.work_dir,'baselines/' + stackReferenceDate +'_' + secondary + '/' + stackReferenceDate +'_'+ secondary + '.txt') configObj.baselineFile = os.path.join(self.work_dir,'baselines/' + stackReferenceDate +'_' + secondary + '/' + stackReferenceDate +'_'+ secondary + '.txt')
configObj.computeAverageBaseline('[Function-1]') configObj.computeAverageBaseline('[Function-1]')
configObj.finalize() configObj.finalize()
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
del configObj del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName+'\n')
def gridBaseline(self, stackReferenceDate, secondaryList): def gridBaseline(self, stackReferenceDate, secondaryList):
line_cnt = 0
for secondary in secondaryList: for secondary in secondaryList:
configName = os.path.join(self.config_path,'config_baselinegrid_'+secondary) configName = os.path.join(self.config_path,'config_baselinegrid_'+secondary)
configObj = config(configName) configObj = config(configName)
@ -364,8 +391,10 @@ class run(object):
configObj.baselineFile = os.path.join(self.work_dir, 'merged/baselines/' + secondary + '/' + secondary ) configObj.baselineFile = os.path.join(self.work_dir, 'merged/baselines/' + secondary + '/' + secondary )
configObj.computeGridBaseline('[Function-1]') configObj.computeGridBaseline('[Function-1]')
configObj.finalize() configObj.finalize()
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName+'\n')
# also add the reference in itself to be consistent with the SLC dir # also add the reference in itself to be consistent with the SLC dir
configName = os.path.join(self.config_path,'config_baselinegrid_reference') configName = os.path.join(self.config_path,'config_baselinegrid_reference')
configObj = config(configName) configObj = config(configName)
@ -375,8 +404,10 @@ class run(object):
configObj.baselineFile = os.path.join(self.work_dir, 'merged/baselines/' + stackReferenceDate + '/' + stackReferenceDate) configObj.baselineFile = os.path.join(self.work_dir, 'merged/baselines/' + stackReferenceDate + '/' + stackReferenceDate)
configObj.computeGridBaseline('[Function-1]') configObj.computeGridBaseline('[Function-1]')
configObj.finalize() configObj.finalize()
line_cnt = 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName+'\n')
def extractOverlaps(self): def extractOverlaps(self):
@ -386,6 +417,7 @@ class run(object):
def geo2rdr_offset(self, secondaryList, fullBurst='False'): def geo2rdr_offset(self, secondaryList, fullBurst='False'):
line_cnt = 0
for secondary in secondaryList: for secondary in secondaryList:
reference = self.reference_date reference = self.reference_date
if fullBurst == 'True': if fullBurst == 'True':
@ -407,10 +439,14 @@ class run(object):
configObj.overlapTrueOrFalse = 'True' configObj.overlapTrueOrFalse = 'True'
configObj.geo2rdr('[Function-1]') configObj.geo2rdr('[Function-1]')
configObj.finalize() configObj.finalize()
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
del configObj del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
def resample_with_carrier(self, secondaryList, fullBurst='False'): def resample_with_carrier(self, secondaryList, fullBurst='False'):
line_cnt = 0
for secondary in secondaryList: for secondary in secondaryList:
reference = self.reference_date reference = self.reference_date
if fullBurst == 'True': if fullBurst == 'True':
@ -433,13 +469,17 @@ class run(object):
configObj.overlapTrueOrFalse = 'True' configObj.overlapTrueOrFalse = 'True'
configObj.resamp_withCarrier('[Function-1]') configObj.resamp_withCarrier('[Function-1]')
configObj.finalize() configObj.finalize()
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
del configObj del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
def pairs_misregistration(self, dateList, safe_dict): def pairs_misregistration(self, dateList, safe_dict):
# generating overlap interferograms, estimate azimuth misregistration for each pair: # generating overlap interferograms, estimate azimuth misregistration for each pair:
pairs = [] pairs = []
num_overlap_connections = int(self.num_overlap_connections) + 1 num_overlap_connections = int(self.num_overlap_connections) + 1
for i in range(len(dateList)-1): for i in range(len(dateList)-1):
for j in range(i+1,i+num_overlap_connections): for j in range(i+1,i+num_overlap_connections):
if j<len(dateList): if j<len(dateList):
@ -450,6 +490,8 @@ class run(object):
safe_dict[date].slc_overlap = os.path.join(self.work_dir , 'coreg_secondarys/'+date) safe_dict[date].slc_overlap = os.path.join(self.work_dir , 'coreg_secondarys/'+date)
safe_dict[self.reference_date].slc = os.path.join(self.work_dir , 'reference') safe_dict[self.reference_date].slc = os.path.join(self.work_dir , 'reference')
safe_dict[self.reference_date].slc_overlap = os.path.join(self.work_dir , 'reference') safe_dict[self.reference_date].slc_overlap = os.path.join(self.work_dir , 'reference')
line_cnt = 0
for pair in pairs: for pair in pairs:
reference = pair[0] reference = pair[0]
secondary = pair[1] secondary = pair[1]
@ -479,11 +521,12 @@ class run(object):
configObj.rangeMisreg('[Function-4]') configObj.rangeMisreg('[Function-4]')
configObj.finalize() configObj.finalize()
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
del configObj
######################## ########################
def timeseries_misregistration(self): def timeseries_misregistration(self):
#inverting the misregistration offsets of the overlap pairs to estimate the offsets of each date #inverting the misregistration offsets of the overlap pairs to estimate the offsets of each date
self.runf.write(self.text_cmd + 'invertMisreg.py -i ' + os.path.join(self.work_dir,'misreg/azimuth/pairs/') + ' -o ' + os.path.join(self.work_dir,'misreg/azimuth/dates/') + '\n') self.runf.write(self.text_cmd + 'invertMisreg.py -i ' + os.path.join(self.work_dir,'misreg/azimuth/pairs/') + ' -o ' + os.path.join(self.work_dir,'misreg/azimuth/dates/') + '\n')
@ -499,6 +542,8 @@ class run(object):
for date in dateList: for date in dateList:
safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_secondarys/'+date) safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_secondarys/'+date)
safe_dict[self.reference_date].slc = os.path.join(self.work_dir , 'reference') safe_dict[self.reference_date].slc = os.path.join(self.work_dir , 'reference')
line_cnt = 0
for pair in pairs: for pair in pairs:
reference = pair[0] reference = pair[0]
secondary = pair[1] secondary = pair[1]
@ -514,14 +559,19 @@ class run(object):
configObj.overlapTrueOrFalse = 'False' configObj.overlapTrueOrFalse = 'False'
configObj.generateIgram('[Function-1]') configObj.generateIgram('[Function-1]')
configObj.finalize() configObj.finalize()
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
del configObj del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
def igram_mergeBurst(self, dateList, safe_dict, pairs): def igram_mergeBurst(self, dateList, safe_dict, pairs):
for date in dateList: for date in dateList:
safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_secondarys/'+date) safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_secondarys/'+date)
safe_dict[self.reference_date].slc = os.path.join(self.work_dir , 'reference') safe_dict[self.reference_date].slc = os.path.join(self.work_dir , 'reference')
line_cnt = 0
for pair in pairs: for pair in pairs:
reference = pair[0] reference = pair[0]
secondary = pair[1] secondary = pair[1]
@ -543,12 +593,14 @@ class run(object):
configObj.stack = os.path.join(self.work_dir, 'stack') configObj.stack = os.path.join(self.work_dir, 'stack')
configObj.mergeBurst('[Function-1]') configObj.mergeBurst('[Function-1]')
configObj.finalize() configObj.finalize()
del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
del configObj
def mergeSecondarySLC(self, secondaryList, virtual='True'): def mergeSecondarySLC(self, secondaryList, virtual='True'):
line_cnt = 0
for secondary in secondaryList: for secondary in secondaryList:
configName = os.path.join(self.config_path,'config_merge_' + secondary) configName = os.path.join(self.config_path,'config_merge_' + secondary)
configObj = config(configName) configObj = config(configName)
@ -566,7 +618,11 @@ class run(object):
configObj.stack = os.path.join(self.work_dir, 'stack') configObj.stack = os.path.join(self.work_dir, 'stack')
configObj.mergeBurst('[Function-1]') configObj.mergeBurst('[Function-1]')
configObj.finalize() configObj.finalize()
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj
def mergeReference(self, stackReference, virtual='True'): def mergeReference(self, stackReference, virtual='True'):
@ -585,7 +641,10 @@ class run(object):
configObj.multiLook = 'False' configObj.multiLook = 'False'
configObj.mergeBurst('[Function-1]') configObj.mergeBurst('[Function-1]')
configObj.finalize() configObj.finalize()
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
line_cnt = 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj
geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr'] geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr']
multiookToolDict = {'lat*rdr': 'gdal', 'lon*rdr': 'gdal', 'los*rdr': 'gdal' , 'hgt*rdr':"gdal", 'shadowMask*rdr':"isce",'incLocal*rdr':"gdal"} multiookToolDict = {'lat*rdr': 'gdal', 'lon*rdr': 'gdal', 'los*rdr': 'gdal' , 'hgt*rdr':"gdal", 'shadowMask*rdr':"isce",'incLocal*rdr':"gdal"}
@ -610,10 +669,14 @@ class run(object):
configObj.stack = os.path.join(self.work_dir, 'stack') configObj.stack = os.path.join(self.work_dir, 'stack')
configObj.mergeBurst('[Function-1]') configObj.mergeBurst('[Function-1]')
configObj.finalize() configObj.finalize()
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj
def mergeSLC(self, aquisitionDates, virtual='True'): def mergeSLC(self, aquisitionDates, virtual='True'):
line_cnt = 0
for slcdate in aquisitionDates: for slcdate in aquisitionDates:
configName = os.path.join(self.config_path,'config_merge_' + slcdate) configName = os.path.join(self.config_path,'config_merge_' + slcdate)
configObj = config(configName) configObj = config(configName)
@ -630,9 +693,14 @@ class run(object):
configObj.stack = os.path.join(self.work_dir, 'stack') configObj.stack = os.path.join(self.work_dir, 'stack')
configObj.mergeBurst('[Function-1]') configObj.mergeBurst('[Function-1]')
configObj.finalize() configObj.finalize()
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr'] geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr']
g_line_cnt = 0
for i in range(len(geometryList)): for i in range(len(geometryList)):
pattern = geometryList[i] pattern = geometryList[i]
configName = os.path.join(self.config_path,'config_merge_' + slcdate + '_' +pattern.split('*')[0]) configName = os.path.join(self.config_path,'config_merge_' + slcdate + '_' +pattern.split('*')[0])
@ -650,10 +718,14 @@ class run(object):
configObj.stack = os.path.join(self.work_dir, 'stack') configObj.stack = os.path.join(self.work_dir, 'stack')
configObj.mergeBurst('[Function-1]') configObj.mergeBurst('[Function-1]')
configObj.finalize() configObj.finalize()
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
g_line_cnt += 1
g_line_cnt = configObj.write_wrapper_config2run_file(configName, g_line_cnt)
del configObj
def filter_coherence(self, pairs): def filter_coherence(self, pairs):
line_cnt = 0
for pair in pairs: for pair in pairs:
reference = pair[0] reference = pair[0]
secondary = pair[1] secondary = pair[1]
@ -673,9 +745,15 @@ class run(object):
#configObj.filtStrength = str(self.filtStrength) #configObj.filtStrength = str(self.filtStrength)
configObj.FilterAndCoherence('[Function-1]') configObj.FilterAndCoherence('[Function-1]')
configObj.finalize() configObj.finalize()
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
del configObj
def unwrap(self, pairs): def unwrap(self, pairs):
line_cnt = 0
for pair in pairs: for pair in pairs:
reference = pair[0] reference = pair[0]
secondary = pair[1] secondary = pair[1]
@ -693,10 +771,14 @@ class run(object):
configObj.unwMethod = self.unwMethod configObj.unwMethod = self.unwMethod
configObj.unwrap('[Function-1]') configObj.unwrap('[Function-1]')
configObj.finalize() configObj.finalize()
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
del configObj
def denseOffsets(self, pairs): def denseOffsets(self, pairs):
line_cnt = 0
for pair in pairs: for pair in pairs:
reference = pair[0] reference = pair[0]
secondary = pair[1] secondary = pair[1]
@ -709,9 +791,10 @@ class run(object):
configObj.output = os.path.join(self.work_dir , 'merged/dense_offsets/'+reference+'_'+secondary + '/' + reference+'_'+secondary) configObj.output = os.path.join(self.work_dir , 'merged/dense_offsets/'+reference+'_'+secondary + '/' + reference+'_'+secondary)
configObj.denseOffset('[Function-1]') configObj.denseOffset('[Function-1]')
configObj.finalize() configObj.finalize()
del configObj
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj
def finalize(self): def finalize(self):
self.runf.close() self.runf.close()
@ -966,8 +1049,6 @@ class sentinelSLC(object):
self.orbit = orbitFile[0] self.orbit = orbitFile[0]
self.orbitType = 'restituted' self.orbitType = 'restituted'
# an example for writing job files when using clusters # an example for writing job files when using clusters
""" """

2
contrib/stack/topsStack/stackSentinel.py
View File

@ -167,7 +167,7 @@ def createParser():
help='Allow App to use GPU when available') help='Allow App to use GPU when available')
parser.add_argument('--num-proc', '--num-process', dest='numProcess', type=int, default=1, parser.add_argument('--num-proc', '--num-process', dest='numProcess', type=int, default=1,
help='number of parallel processes (for topo only) (default: %(default)s).') help='number of parallel processes (where applicable) (default: %(default)s).')
parser.add_argument('-u', '--unw_method', dest='unwMethod', type=str, default='snaphu', choices=['icu', 'snaphu'], parser.add_argument('-u', '--unw_method', dest='unwMethod', type=str, default='snaphu', choices=['icu', 'snaphu'],
help='Unwrapping method (default: %(default)s).') help='Unwrapping method (default: %(default)s).')