From 97b7800fdad0dbed21c319457d55bc8479c12820 Mon Sep 17 00:00:00 2001 From: Sara Mirzaee Date: Thu, 9 Apr 2020 01:13:36 -0500 Subject: [PATCH] make a separate run_file for some functions to better allocate time and memory --- contrib/stack/topsStack/Stack.py | 107 ++++++++++++++--------- contrib/stack/topsStack/stackSentinel.py | 34 +++++-- 2 files changed, 92 insertions(+), 49 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index a9e5238..4251566 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -9,6 +9,7 @@ import datetime noMCF = 'False' +rmFilter = 'True' defoMax = '2' maxNodes = 72 @@ -223,6 +224,7 @@ class config(object): self.f.write('defomax : ' + self.defoMax + '\n') self.f.write('rlks : ' + self.rangeLooks + '\n') self.f.write('alks : ' + self.azimuthLooks + '\n') + self.f.write('rmfilter : ' + self.rmFilter + '\n') self.f.write('method : ' + self.unwMethod + '\n') def unwrapSnaphu(self, function): @@ -285,7 +287,7 @@ class run(object): os.makedirs(self.config_path) for slcdate in acquisitionDates: - configName = os.path.join(self.config_path,'config_'+slcdate) + configName = os.path.join(self.config_path,'config_unpack_'+slcdate) configObj = config(configName) configObj.configure(self) configObj.dirName = safe_dict[slcdate].safe_file @@ -381,11 +383,15 @@ class run(object): self.runf.write(self.text_cmd + 'subsetMaster.py -m ' + os.path.join(self.work_dir, 'master') + ' -g ' + os.path.join(self.work_dir, 'geom_master') + '\n') - def overlap_geo2rdr_resample(self, slaveList): + + def geo2rdr_offset(self, slaveList, fullBurst='False'): for slave in slaveList: master = self.master_date - configName = os.path.join(self.config_path ,'config_resamp_overlap_'+slave) + if fullBurst == 'True': + configName = os.path.join(self.config_path, 'config_fullBurst_geo2rdr_' + slave) + else: + configName = os.path.join(self.config_path, 'config_overlap_geo2rdr_'+slave) ########### configObj = config(configName) configObj.configure(self) @@ -393,16 +399,42 @@ class run(object): configObj.masterDir = os.path.join(self.work_dir, 'master') configObj.geom_master = os.path.join(self.work_dir, 'geom_master') configObj.coregSlaveDir = os.path.join(self.work_dir, 'coreg_slaves/'+slave) - configObj.overlapTrueOrFalse = 'True' + if fullBurst == 'True': + configObj.misreg_az = os.path.join(self.work_dir, 'misreg/azimuth/dates/' + slave + '.txt') + configObj.misreg_rng = os.path.join(self.work_dir, 'misreg/range/dates/' + slave + '.txt') + configObj.overlapTrueOrFalse = 'False' + else: + configObj.overlapTrueOrFalse = 'True' configObj.geo2rdr('[Function-1]') - ########### - configObj.interferogram_prefix = 'coarse' - configObj.masterDir = os.path.join(self.work_dir,'master') - configObj.resamp_withCarrier('[Function-2]') - ########### configObj.finalize() del configObj - self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') + self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') + + def resample_with_carrier(self, slaveList, fullBurst='False'): + for slave in slaveList: + master = self.master_date + if fullBurst == 'True': + configName = os.path.join(self.config_path, 'config_fullBurst_resample_' + slave) + else: + configName = os.path.join(self.config_path, 'config_overlap_resample_' + slave) + ########### + configObj = config(configName) + configObj.configure(self) + configObj.slaveDir = os.path.join(self.work_dir, 'slaves/' + slave) + configObj.masterDir = os.path.join(self.work_dir, 'master') + configObj.coregSlaveDir = os.path.join(self.work_dir, 'coreg_slaves/' + slave) + configObj.interferogram_prefix = 'coarse' + configObj.masterDir = os.path.join(self.work_dir, 'master') + if fullBurst == 'True': + configObj.misreg_az = os.path.join(self.work_dir, 'misreg/azimuth/dates/' + slave + '.txt') + configObj.misreg_rng = os.path.join(self.work_dir, 'misreg/range/dates/' + slave + '.txt') + configObj.overlapTrueOrFalse = 'False' + else: + configObj.overlapTrueOrFalse = 'True' + configObj.resamp_withCarrier('[Function-1]') + configObj.finalize() + del configObj + self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') def pairs_misregistration(self, dateList, safe_dict): # generating overlap interferograms, estimate azimuth misregistration for each pair: @@ -457,37 +489,13 @@ class run(object): self.runf.write(self.text_cmd + 'invertMisreg.py -i ' + os.path.join(self.work_dir,'misreg/azimuth/pairs/') + ' -o ' + os.path.join(self.work_dir,'misreg/azimuth/dates/') + '\n') self.runf.write(self.text_cmd + 'invertMisreg.py -i ' + os.path.join(self.work_dir,'misreg/range/pairs/') + ' -o ' + os.path.join(self.work_dir,'misreg/range/dates/') + '\n') - def geo2rdr_resample(self, slaveList): - # geometry offsets and resampling each full burst slave SLC - for slave in slaveList: - master = self.master_date - configName = os.path.join(self.config_path , 'config_resamp_' + slave) - ########### - configObj = config(configName) - configObj.configure(self) - configObj.slaveDir = os.path.join(self.work_dir, 'slaves/' + slave) - configObj.masterDir = os.path.join(self.work_dir, 'master') - configObj.geom_master = os.path.join(self.work_dir, 'geom_master') - configObj.coregSlaveDir = os.path.join(self.work_dir, 'coreg_slaves/' + slave) - configObj.misreg_az = os.path.join(self.work_dir, 'misreg/azimuth/dates/' + slave + '.txt') - configObj.misreg_rng = os.path.join(self.work_dir, 'misreg/range/dates/' + slave + '.txt') - configObj.overlapTrueOrFalse = 'False' - configObj.geo2rdr('[Function-1]') - ########### - configObj.interferogram_prefix = 'coarse' - configObj.masterDir = os.path.join(self.work_dir, 'master') - configObj.resamp_withCarrier('[Function-2]') - ########### - configObj.finalize() - del configObj - self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') - def extractStackValidRegion(self): masterDir = os.path.join(self.work_dir, 'master') coregSlaveDir = os.path.join(self.work_dir, 'coreg_slaves') self.runf.write(self.text_cmd + 'extractCommonValidRegion.py -m ' + masterDir + ' -s ' + coregSlaveDir + '\n') - def burstIgram_mergeBurst(self, dateList, safe_dict, pairs): + def generate_burstIgram(self, dateList, safe_dict, pairs): + for date in dateList: safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_slaves/'+date) safe_dict[self.master_date].slc = os.path.join(self.work_dir , 'master') @@ -495,8 +503,7 @@ class run(object): master = pair[0] slave = pair[1] interferogramDir = os.path.join(self.work_dir, 'interferograms/' + master + '_' + slave) - mergedDir = os.path.join(self.work_dir, 'merged/interferograms/' + master + '_' + slave) - configName = os.path.join(self.config_path ,'config_igram_' + master + '_' + slave) + configName = os.path.join(self.config_path ,'config_generate_igram_' + master + '_' + slave) configObj = config(configName) configObj.configure(self) configObj.masterDir = safe_dict[master].slc @@ -506,9 +513,26 @@ class run(object): configObj.flatten = 'False' configObj.overlapTrueOrFalse = 'False' configObj.generateIgram('[Function-1]') + configObj.finalize() + del configObj - configObj.master = configObj.interferogramDir - configObj.dirName = configObj.master + self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') + + def igram_mergeBurst(self, dateList, safe_dict, pairs): + for date in dateList: + safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_slaves/'+date) + safe_dict[self.master_date].slc = os.path.join(self.work_dir , 'master') + for pair in pairs: + master = pair[0] + slave = pair[1] + interferogramDir = os.path.join(self.work_dir, 'interferograms/' + master + '_' + slave) + mergedDir = os.path.join(self.work_dir, 'merged/interferograms/' + master + '_' + slave) + configName = os.path.join(self.config_path ,'config_merge_igram_' + master + '_' + slave) + configObj = config(configName) + configObj.configure(self) + configObj.interferogram_prefix = 'fine' + configObj.master = interferogramDir + configObj.dirName = interferogramDir configObj.namePattern = 'fine*int' configObj.mergedFile = mergedDir + '/' + configObj.interferogram_prefix + '.int' configObj.mergeBurstsMethod = 'top' @@ -517,7 +541,7 @@ class run(object): configObj.useVirtualFiles = 'True' configObj.multiLook = 'True' configObj.stack = os.path.join(self.work_dir, 'stack') - configObj.mergeBurst('[Function-2]') + configObj.mergeBurst('[Function-1]') configObj.finalize() del configObj @@ -661,6 +685,7 @@ class run(object): configObj.cohName = os.path.join(mergedDir,'filt_fine.cor') configObj.unwName = os.path.join(mergedDir,'filt_fine.unw') configObj.noMCF = noMCF + configObj.rmFilter = rmFilter configObj.master = os.path.join(self.work_dir,'master') configObj.defoMax = defoMax configObj.unwMethod = self.unwMethod diff --git a/contrib/stack/topsStack/stackSentinel.py b/contrib/stack/topsStack/stackSentinel.py index 0103500..7c75f93 100755 --- a/contrib/stack/topsStack/stackSentinel.py +++ b/contrib/stack/topsStack/stackSentinel.py @@ -457,10 +457,16 @@ def slcStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, upd runObj.extractOverlaps() runObj.finalize() - i+=1 + i += 1 runObj = run() - runObj.configure(inps, 'run_' + str(i) + "_overlap_geo2rdr_resample") - runObj.overlap_geo2rdr_resample(slaveDates) + runObj.configure(inps, 'run_' + str(i) + "_overlap_geo2rdr") + runObj.geo2rdr_offset(slaveDates) + runObj.finalize() + + i += 1 + runObj = run() + runObj.configure(inps, 'run_' + str(i) + "_overlap_resample") + runObj.resample_with_carrier(slaveDates) runObj.finalize() i+=1 @@ -478,10 +484,16 @@ def slcStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, upd runObj.timeseries_misregistration() runObj.finalize() - i+=1 + i += 1 runObj = run() - runObj.configure(inps, 'run_' + str(i) + "_geo2rdr_resample") - runObj.geo2rdr_resample(slaveDates) + runObj.configure(inps, 'run_' + str(i) + "_fullBurst_geo2rdr") + runObj.geo2rdr_offset(slaveDates, fullBurst='True') + runObj.finalize() + + i += 1 + runObj = run() + runObj.configure(inps, 'run_' + str(i) + "_fullBurst_resample") + runObj.resample_with_carrier(slaveDates, fullBurst='True') runObj.finalize() i+=1 @@ -546,8 +558,14 @@ def interferogramStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe i+=1 runObj = run() - runObj.configure(inps, 'run_' + str(i) + "_merge_burst_igram") - runObj.burstIgram_mergeBurst(acquisitionDates, safe_dict, pairs) + runObj.configure(inps, 'run_' + str(i) + "_generate_burst_igram") + runObj.generate_burstIgram(acquisitionDates, safe_dict, pairs) + runObj.finalize() + + i += 1 + runObj = run() + runObj.configure(inps, 'run_' + str(i) + "_merge_burst_igram") + runObj.igram_mergeBurst(acquisitionDates, safe_dict, pairs) runObj.finalize() i+=1