From 97b7800fdad0dbed21c319457d55bc8479c12820 Mon Sep 17 00:00:00 2001 From: Sara Mirzaee Date: Thu, 9 Apr 2020 01:13:36 -0500 Subject: [PATCH 1/5] make a separate run_file for some functions to better allocate time and memory --- contrib/stack/topsStack/Stack.py | 107 ++++++++++++++--------- contrib/stack/topsStack/stackSentinel.py | 34 +++++-- 2 files changed, 92 insertions(+), 49 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index a9e5238..4251566 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -9,6 +9,7 @@ import datetime noMCF = 'False' +rmFilter = 'True' defoMax = '2' maxNodes = 72 @@ -223,6 +224,7 @@ class config(object): self.f.write('defomax : ' + self.defoMax + '\n') self.f.write('rlks : ' + self.rangeLooks + '\n') self.f.write('alks : ' + self.azimuthLooks + '\n') + self.f.write('rmfilter : ' + self.rmFilter + '\n') self.f.write('method : ' + self.unwMethod + '\n') def unwrapSnaphu(self, function): @@ -285,7 +287,7 @@ class run(object): os.makedirs(self.config_path) for slcdate in acquisitionDates: - configName = os.path.join(self.config_path,'config_'+slcdate) + configName = os.path.join(self.config_path,'config_unpack_'+slcdate) configObj = config(configName) configObj.configure(self) configObj.dirName = safe_dict[slcdate].safe_file @@ -381,11 +383,15 @@ class run(object): self.runf.write(self.text_cmd + 'subsetMaster.py -m ' + os.path.join(self.work_dir, 'master') + ' -g ' + os.path.join(self.work_dir, 'geom_master') + '\n') - def overlap_geo2rdr_resample(self, slaveList): + + def geo2rdr_offset(self, slaveList, fullBurst='False'): for slave in slaveList: master = self.master_date - configName = os.path.join(self.config_path ,'config_resamp_overlap_'+slave) + if fullBurst == 'True': + configName = os.path.join(self.config_path, 'config_fullBurst_geo2rdr_' + slave) + else: + configName = os.path.join(self.config_path, 'config_overlap_geo2rdr_'+slave) ########### configObj = config(configName) configObj.configure(self) @@ -393,16 +399,42 @@ class run(object): configObj.masterDir = os.path.join(self.work_dir, 'master') configObj.geom_master = os.path.join(self.work_dir, 'geom_master') configObj.coregSlaveDir = os.path.join(self.work_dir, 'coreg_slaves/'+slave) - configObj.overlapTrueOrFalse = 'True' + if fullBurst == 'True': + configObj.misreg_az = os.path.join(self.work_dir, 'misreg/azimuth/dates/' + slave + '.txt') + configObj.misreg_rng = os.path.join(self.work_dir, 'misreg/range/dates/' + slave + '.txt') + configObj.overlapTrueOrFalse = 'False' + else: + configObj.overlapTrueOrFalse = 'True' configObj.geo2rdr('[Function-1]') - ########### - configObj.interferogram_prefix = 'coarse' - configObj.masterDir = os.path.join(self.work_dir,'master') - configObj.resamp_withCarrier('[Function-2]') - ########### configObj.finalize() del configObj - self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') + self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') + + def resample_with_carrier(self, slaveList, fullBurst='False'): + for slave in slaveList: + master = self.master_date + if fullBurst == 'True': + configName = os.path.join(self.config_path, 'config_fullBurst_resample_' + slave) + else: + configName = os.path.join(self.config_path, 'config_overlap_resample_' + slave) + ########### + configObj = config(configName) + configObj.configure(self) + configObj.slaveDir = os.path.join(self.work_dir, 'slaves/' + slave) + configObj.masterDir = os.path.join(self.work_dir, 'master') + configObj.coregSlaveDir = os.path.join(self.work_dir, 'coreg_slaves/' + slave) + configObj.interferogram_prefix = 'coarse' + configObj.masterDir = os.path.join(self.work_dir, 'master') + if fullBurst == 'True': + configObj.misreg_az = os.path.join(self.work_dir, 'misreg/azimuth/dates/' + slave + '.txt') + configObj.misreg_rng = os.path.join(self.work_dir, 'misreg/range/dates/' + slave + '.txt') + configObj.overlapTrueOrFalse = 'False' + else: + configObj.overlapTrueOrFalse = 'True' + configObj.resamp_withCarrier('[Function-1]') + configObj.finalize() + del configObj + self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') def pairs_misregistration(self, dateList, safe_dict): # generating overlap interferograms, estimate azimuth misregistration for each pair: @@ -457,37 +489,13 @@ class run(object): self.runf.write(self.text_cmd + 'invertMisreg.py -i ' + os.path.join(self.work_dir,'misreg/azimuth/pairs/') + ' -o ' + os.path.join(self.work_dir,'misreg/azimuth/dates/') + '\n') self.runf.write(self.text_cmd + 'invertMisreg.py -i ' + os.path.join(self.work_dir,'misreg/range/pairs/') + ' -o ' + os.path.join(self.work_dir,'misreg/range/dates/') + '\n') - def geo2rdr_resample(self, slaveList): - # geometry offsets and resampling each full burst slave SLC - for slave in slaveList: - master = self.master_date - configName = os.path.join(self.config_path , 'config_resamp_' + slave) - ########### - configObj = config(configName) - configObj.configure(self) - configObj.slaveDir = os.path.join(self.work_dir, 'slaves/' + slave) - configObj.masterDir = os.path.join(self.work_dir, 'master') - configObj.geom_master = os.path.join(self.work_dir, 'geom_master') - configObj.coregSlaveDir = os.path.join(self.work_dir, 'coreg_slaves/' + slave) - configObj.misreg_az = os.path.join(self.work_dir, 'misreg/azimuth/dates/' + slave + '.txt') - configObj.misreg_rng = os.path.join(self.work_dir, 'misreg/range/dates/' + slave + '.txt') - configObj.overlapTrueOrFalse = 'False' - configObj.geo2rdr('[Function-1]') - ########### - configObj.interferogram_prefix = 'coarse' - configObj.masterDir = os.path.join(self.work_dir, 'master') - configObj.resamp_withCarrier('[Function-2]') - ########### - configObj.finalize() - del configObj - self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') - def extractStackValidRegion(self): masterDir = os.path.join(self.work_dir, 'master') coregSlaveDir = os.path.join(self.work_dir, 'coreg_slaves') self.runf.write(self.text_cmd + 'extractCommonValidRegion.py -m ' + masterDir + ' -s ' + coregSlaveDir + '\n') - def burstIgram_mergeBurst(self, dateList, safe_dict, pairs): + def generate_burstIgram(self, dateList, safe_dict, pairs): + for date in dateList: safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_slaves/'+date) safe_dict[self.master_date].slc = os.path.join(self.work_dir , 'master') @@ -495,8 +503,7 @@ class run(object): master = pair[0] slave = pair[1] interferogramDir = os.path.join(self.work_dir, 'interferograms/' + master + '_' + slave) - mergedDir = os.path.join(self.work_dir, 'merged/interferograms/' + master + '_' + slave) - configName = os.path.join(self.config_path ,'config_igram_' + master + '_' + slave) + configName = os.path.join(self.config_path ,'config_generate_igram_' + master + '_' + slave) configObj = config(configName) configObj.configure(self) configObj.masterDir = safe_dict[master].slc @@ -506,9 +513,26 @@ class run(object): configObj.flatten = 'False' configObj.overlapTrueOrFalse = 'False' configObj.generateIgram('[Function-1]') + configObj.finalize() + del configObj - configObj.master = configObj.interferogramDir - configObj.dirName = configObj.master + self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n') + + def igram_mergeBurst(self, dateList, safe_dict, pairs): + for date in dateList: + safe_dict[date].slc = os.path.join(self.work_dir, 'coreg_slaves/'+date) + safe_dict[self.master_date].slc = os.path.join(self.work_dir , 'master') + for pair in pairs: + master = pair[0] + slave = pair[1] + interferogramDir = os.path.join(self.work_dir, 'interferograms/' + master + '_' + slave) + mergedDir = os.path.join(self.work_dir, 'merged/interferograms/' + master + '_' + slave) + configName = os.path.join(self.config_path ,'config_merge_igram_' + master + '_' + slave) + configObj = config(configName) + configObj.configure(self) + configObj.interferogram_prefix = 'fine' + configObj.master = interferogramDir + configObj.dirName = interferogramDir configObj.namePattern = 'fine*int' configObj.mergedFile = mergedDir + '/' + configObj.interferogram_prefix + '.int' configObj.mergeBurstsMethod = 'top' @@ -517,7 +541,7 @@ class run(object): configObj.useVirtualFiles = 'True' configObj.multiLook = 'True' configObj.stack = os.path.join(self.work_dir, 'stack') - configObj.mergeBurst('[Function-2]') + configObj.mergeBurst('[Function-1]') configObj.finalize() del configObj @@ -661,6 +685,7 @@ class run(object): configObj.cohName = os.path.join(mergedDir,'filt_fine.cor') configObj.unwName = os.path.join(mergedDir,'filt_fine.unw') configObj.noMCF = noMCF + configObj.rmFilter = rmFilter configObj.master = os.path.join(self.work_dir,'master') configObj.defoMax = defoMax configObj.unwMethod = self.unwMethod diff --git a/contrib/stack/topsStack/stackSentinel.py b/contrib/stack/topsStack/stackSentinel.py index 0103500..7c75f93 100755 --- a/contrib/stack/topsStack/stackSentinel.py +++ b/contrib/stack/topsStack/stackSentinel.py @@ -457,10 +457,16 @@ def slcStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, upd runObj.extractOverlaps() runObj.finalize() - i+=1 + i += 1 runObj = run() - runObj.configure(inps, 'run_' + str(i) + "_overlap_geo2rdr_resample") - runObj.overlap_geo2rdr_resample(slaveDates) + runObj.configure(inps, 'run_' + str(i) + "_overlap_geo2rdr") + runObj.geo2rdr_offset(slaveDates) + runObj.finalize() + + i += 1 + runObj = run() + runObj.configure(inps, 'run_' + str(i) + "_overlap_resample") + runObj.resample_with_carrier(slaveDates) runObj.finalize() i+=1 @@ -478,10 +484,16 @@ def slcStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, upd runObj.timeseries_misregistration() runObj.finalize() - i+=1 + i += 1 runObj = run() - runObj.configure(inps, 'run_' + str(i) + "_geo2rdr_resample") - runObj.geo2rdr_resample(slaveDates) + runObj.configure(inps, 'run_' + str(i) + "_fullBurst_geo2rdr") + runObj.geo2rdr_offset(slaveDates, fullBurst='True') + runObj.finalize() + + i += 1 + runObj = run() + runObj.configure(inps, 'run_' + str(i) + "_fullBurst_resample") + runObj.resample_with_carrier(slaveDates, fullBurst='True') runObj.finalize() i+=1 @@ -546,8 +558,14 @@ def interferogramStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe i+=1 runObj = run() - runObj.configure(inps, 'run_' + str(i) + "_merge_burst_igram") - runObj.burstIgram_mergeBurst(acquisitionDates, safe_dict, pairs) + runObj.configure(inps, 'run_' + str(i) + "_generate_burst_igram") + runObj.generate_burstIgram(acquisitionDates, safe_dict, pairs) + runObj.finalize() + + i += 1 + runObj = run() + runObj.configure(inps, 'run_' + str(i) + "_merge_burst_igram") + runObj.igram_mergeBurst(acquisitionDates, safe_dict, pairs) runObj.finalize() i+=1 From dc97d160a4e8df60ca3642b23e0af2645a0d52a0 Mon Sep 17 00:00:00 2001 From: Sara Mirzaee Date: Thu, 9 Apr 2020 01:14:26 -0500 Subject: [PATCH 2/5] add fine.unw to remove the effect of filtering --- contrib/stack/topsStack/unwrap.py | 63 ++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/contrib/stack/topsStack/unwrap.py b/contrib/stack/topsStack/unwrap.py index c297dc8..fd3931e 100755 --- a/contrib/stack/topsStack/unwrap.py +++ b/contrib/stack/topsStack/unwrap.py @@ -38,9 +38,11 @@ from isceobj.Constants import SPEED_OF_LIGHT import argparse import os import pickle -import sys +import gdal +import numpy as np #import shelve import s1a_isce_utils as ut +from isceobj.Util.ImageUtil import ImageLib as IML def createParser(): ''' @@ -70,6 +72,9 @@ def createParser(): parser.add_argument('-m', '--method', dest='method', type=str, default='icu', help='unwrapping method') + parser.add_argument('--rmfilter', action='store_true', default=False, + help='remove the effect of filtering from final unwrapped interferograms') + return parser @@ -276,6 +281,53 @@ def runUnwrapIcu(infile, outfile): #unwImage.finalizeImage() unwImage.renderHdr() + +def remove_filter(intfile, filtfile, unwfile): + + outunw = os.path.abspath(unwfile).split('filt_') + outunw = outunw[0] + outunw[1] + + ds_unw = gdal.Open(unwfile + ".vrt", gdal.GA_ReadOnly) + unw_phase = ds_unw.ReadAsArray() + + bands, length, width = unw_phase.shape + print(bands, length, width) + outfile = IML.memmap(outunw, mode='write', nchannels=2, + nxx=width, nyy=length, scheme='BIL', dataType='f') + + outfile.bands[0][:, :] = unw_phase[0, :, :] + + ds_filt = gdal.Open(filtfile + ".vrt", gdal.GA_ReadOnly) + filt_phase = np.angle(ds_filt.ReadAsArray()) + + integer_jumps = unw_phase[1, :, :] - filt_phase + + del unw_phase, filt_phase + + ds_int = gdal.Open(intfile + ".vrt", gdal.GA_ReadOnly) + int_phase = np.angle(ds_int.ReadAsArray()) + + out_phase = integer_jumps + int_phase + + del int_phase, integer_jumps + + outfile.bands[1][:, :] = out_phase[:, :] + + unwImage = isceobj.Image.createImage() + unwImage.setFilename(outunw) + unwImage.setWidth(width) + unwImage.setLength(length) + unwImage.imageType = 'unw' + unwImage.bands = 2 + unwImage.scheme = 'BIL' + unwImage.dataType = 'FLOAT' + unwImage.setAccessMode('read') + #unwImage.createImage() + unwImage.renderHdr() + #unwImage.finalizeImage() + + return + def main(iargs=None): ''' The main driver. @@ -284,6 +336,7 @@ def main(iargs=None): inps = cmdLineParse(iargs) interferogramDir = os.path.dirname(inps.intfile) print ('unwrapping method : ' , inps.method) + ''' if inps.method == 'snaphu': if inps.nomcf: fncall = runUnwrap @@ -297,6 +350,14 @@ def main(iargs=None): elif inps.method == 'icu': runUnwrapIcu(inps.intfile, inps.unwfile) + ''' + + if inps.rmfilter: + filtfile = os.path.abspath(inps.intfile) + intfile = filtfile.split('filt_') + intfile = intfile[0] + intfile[1] + + remove_filter(intfile, filtfile, inps.unwfile) if __name__ == '__main__': From e1729de5a8efd9cd0b8b5db78166c3661f87d57a Mon Sep 17 00:00:00 2001 From: Sara Mirzaee <37273875+mirzaees@users.noreply.github.com> Date: Sat, 11 Apr 2020 11:50:10 -0400 Subject: [PATCH 3/5] fix bug: some lines were commented by mistake --- contrib/stack/topsStack/unwrap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/stack/topsStack/unwrap.py b/contrib/stack/topsStack/unwrap.py index fd3931e..f5026ee 100755 --- a/contrib/stack/topsStack/unwrap.py +++ b/contrib/stack/topsStack/unwrap.py @@ -336,7 +336,7 @@ def main(iargs=None): inps = cmdLineParse(iargs) interferogramDir = os.path.dirname(inps.intfile) print ('unwrapping method : ' , inps.method) - ''' + if inps.method == 'snaphu': if inps.nomcf: fncall = runUnwrap @@ -350,7 +350,7 @@ def main(iargs=None): elif inps.method == 'icu': runUnwrapIcu(inps.intfile, inps.unwfile) - ''' + if inps.rmfilter: filtfile = os.path.abspath(inps.intfile) From fe097c906a0deb44f707c5e6442529e724cdad17 Mon Sep 17 00:00:00 2001 From: Sara Mirzaee Date: Fri, 1 May 2020 14:44:59 -0500 Subject: [PATCH 4/5] add --rmFilter to stackSentinel.py --- contrib/stack/topsStack/Stack.py | 3 +-- contrib/stack/topsStack/stackSentinel.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index 4251566..7c40561 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -9,7 +9,6 @@ import datetime noMCF = 'False' -rmFilter = 'True' defoMax = '2' maxNodes = 72 @@ -685,7 +684,7 @@ class run(object): configObj.cohName = os.path.join(mergedDir,'filt_fine.cor') configObj.unwName = os.path.join(mergedDir,'filt_fine.unw') configObj.noMCF = noMCF - configObj.rmFilter = rmFilter + configObj.rmFilter = self.rmFilter configObj.master = os.path.join(self.work_dir,'master') configObj.defoMax = defoMax configObj.unwMethod = self.unwMethod diff --git a/contrib/stack/topsStack/stackSentinel.py b/contrib/stack/topsStack/stackSentinel.py index 7c75f93..c70b819 100755 --- a/contrib/stack/topsStack/stackSentinel.py +++ b/contrib/stack/topsStack/stackSentinel.py @@ -162,6 +162,9 @@ def createParser(): parser.add_argument('-useGPU', '--useGPU', dest='useGPU',action='store_true', default=False, help='Allow App to use GPU when available') + parser.add_argument('-rmFilter', '--rmFilter', dest='rmFilter', action='store_true', default=False, + help='Make an extra unwrap file in which filtering effect is removed') + return parser def cmdLineParse(iargs = None): From 08e384994526eaa4f165e697e0c543553c0e283d Mon Sep 17 00:00:00 2001 From: Sara Mirzaee Date: Fri, 1 May 2020 14:45:48 -0500 Subject: [PATCH 5/5] looping over lines instead of reading the whole file --- contrib/stack/topsStack/unwrap.py | 37 +++++++++---------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/contrib/stack/topsStack/unwrap.py b/contrib/stack/topsStack/unwrap.py index fd3931e..7880858 100755 --- a/contrib/stack/topsStack/unwrap.py +++ b/contrib/stack/topsStack/unwrap.py @@ -288,30 +288,19 @@ def remove_filter(intfile, filtfile, unwfile): outunw = outunw[0] + outunw[1] ds_unw = gdal.Open(unwfile + ".vrt", gdal.GA_ReadOnly) - unw_phase = ds_unw.ReadAsArray() + width = ds_unw.RasterXSize + length = ds_unw.RasterYSize - bands, length, width = unw_phase.shape - print(bands, length, width) - outfile = IML.memmap(outunw, mode='write', nchannels=2, - nxx=width, nyy=length, scheme='BIL', dataType='f') + unw_phase = np.memmap(unwfile, dtype='f', mode='r', shape=(2, length, width)) + filt_phase = np.memmap(filtfile, dtype='f', mode='r', shape=(length, width)) + int_phase = np.memmap(intfile, dtype='f', mode='r', shape=(length, width)) - outfile.bands[0][:, :] = unw_phase[0, :, :] + outfile = np.memmap(outunw, dtype='f', mode='w+', shape=(2, length, width)) - ds_filt = gdal.Open(filtfile + ".vrt", gdal.GA_ReadOnly) - filt_phase = np.angle(ds_filt.ReadAsArray()) - - integer_jumps = unw_phase[1, :, :] - filt_phase - - del unw_phase, filt_phase - - ds_int = gdal.Open(intfile + ".vrt", gdal.GA_ReadOnly) - int_phase = np.angle(ds_int.ReadAsArray()) - - out_phase = integer_jumps + int_phase - - del int_phase, integer_jumps - - outfile.bands[1][:, :] = out_phase[:, :] + for line in range(length): + integer_jumps = unw_phase[1, line, :] - np.angle(filt_phase[line, :]) + outfile[1, line, :] = integer_jumps + np.angle(int_phase[line, :]) + outfile[0, line, :] = unw_phase[0, line, :] unwImage = isceobj.Image.createImage() unwImage.setFilename(outunw) @@ -322,9 +311,7 @@ def remove_filter(intfile, filtfile, unwfile): unwImage.scheme = 'BIL' unwImage.dataType = 'FLOAT' unwImage.setAccessMode('read') - #unwImage.createImage() unwImage.renderHdr() - #unwImage.finalizeImage() return @@ -334,9 +321,8 @@ def main(iargs=None): ''' inps = cmdLineParse(iargs) - interferogramDir = os.path.dirname(inps.intfile) print ('unwrapping method : ' , inps.method) - ''' + if inps.method == 'snaphu': if inps.nomcf: fncall = runUnwrap @@ -350,7 +336,6 @@ def main(iargs=None): elif inps.method == 'icu': runUnwrapIcu(inps.intfile, inps.unwfile) - ''' if inps.rmfilter: filtfile = os.path.abspath(inps.intfile)