From 7db07f3768940b195ddb9d2a898b6c6f55b7de47 Mon Sep 17 00:00:00 2001 From: piyushrpt Date: Sun, 31 May 2020 13:28:24 -0700 Subject: [PATCH 01/10] Suppress zero sized log file creation --- defaults/logging/logging.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/defaults/logging/logging.conf b/defaults/logging/logging.conf index ebd829e..cdc158a 100644 --- a/defaults/logging/logging.conf +++ b/defaults/logging/logging.conf @@ -21,7 +21,8 @@ propagate=0 class=handlers.RotatingFileHandler formatter=simpleFormatter # Filename, file mode, maximum file size in bytes,number of backups to keep -args=('isce.log','a',1048576,5) +# encoding, delay +args=('isce.log','a',1048576,5,None,True) [handler_consoleHandler] class=StreamHandler From 87cffaaadda66875900515ffb6edfeebda417ebe Mon Sep 17 00:00:00 2001 From: sssangha Date: Wed, 3 Jun 2020 17:24:09 -0700 Subject: [PATCH 02/10] Baseline bug fix --- components/isceobj/TopsProc/runComputeBaseline.py | 2 ++ components/zerodop/baseline/Baseline.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/components/isceobj/TopsProc/runComputeBaseline.py b/components/isceobj/TopsProc/runComputeBaseline.py index 7aa0eb9..970084d 100644 --- a/components/isceobj/TopsProc/runComputeBaseline.py +++ b/components/isceobj/TopsProc/runComputeBaseline.py @@ -77,6 +77,8 @@ def runComputeBaseline(self): mxyz = np.array(masterSV.getPosition()) mvel = np.array(masterSV.getVelocity()) sxyz = np.array(slaveSV.getPosition()) + mvelunit = mvel / np.linalg.norm(mvel) + sxyz = sxyz - np.dot ( sxyz-mxyz, mvelunit) * mvelunit aa = np.linalg.norm(sxyz-mxyz) costheta = (rng*rng + aa*aa - slvrng*slvrng)/(2.*rng*aa) diff --git a/components/zerodop/baseline/Baseline.py b/components/zerodop/baseline/Baseline.py index b26c74d..8b2b7b0 100755 --- a/components/zerodop/baseline/Baseline.py +++ b/components/zerodop/baseline/Baseline.py @@ -103,6 +103,8 @@ class Baseline(Component): mxyz = np.array(masterSV.getPosition()) mvel = np.array(masterSV.getVelocity()) sxyz = np.array(slaveSV.getPosition()) + mvelunit = mvel / np.linalg.norm(mvel) + sxyz = sxyz - np.dot ( sxyz-mxyz, mvelunit) * mvelunit aa = np.linalg.norm(sxyz-mxyz) From 167ac5ac6cae66b9727e5348c1e6e560492ce422 Mon Sep 17 00:00:00 2001 From: CunrenLiang <56097947+CunrenLiang@users.noreply.github.com> Date: Sat, 6 Jun 2020 00:11:02 -0700 Subject: [PATCH 03/10] create an account for downloading DEM --- examples/input_files/alos2/alos2_tutorial.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/input_files/alos2/alos2_tutorial.txt b/examples/input_files/alos2/alos2_tutorial.txt index 77099e6..0a36d90 100644 --- a/examples/input_files/alos2/alos2_tutorial.txt +++ b/examples/input_files/alos2/alos2_tutorial.txt @@ -92,7 +92,15 @@ in the data. Normally we only process HH polarization, so you can only extract H zip files to save space. If you want to process other polarizations, extract those polarization instead. 2. DEM and water body -If you only process one InSAR pair, there is no need to download DEM and water body. The program will + +You MUST FIRST have an account to download DEM and water body, no matter you download them manually or let +the program download them automatically. See +https://github.com/isce-framework/isce2#notes-on-digital-elevation-models +or +https://github.com/isce-framework/isce2 +for more details. + +If you only process one InSAR pair, there is no need to download DEM and water body manually. The program will do it for you. However, if you want to process a stack of interferograms, we recommend downloading DEM and water body by yourself and set the parameters in the input file; otherwise, the program will download DEM and water body each time it processes a pair. See input file on how to download DEM and water From 2b8fdb3acbb9b9e5052d15a27c6232bfd2249bc2 Mon Sep 17 00:00:00 2001 From: CunrenLiang <56097947+CunrenLiang@users.noreply.github.com> Date: Sat, 6 Jun 2020 00:12:26 -0700 Subject: [PATCH 04/10] fix bug in baseline computation --- components/isceobj/Alos2Proc/runPreprocessor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/isceobj/Alos2Proc/runPreprocessor.py b/components/isceobj/Alos2Proc/runPreprocessor.py index 2f46b10..3d39d5a 100644 --- a/components/isceobj/Alos2Proc/runPreprocessor.py +++ b/components/isceobj/Alos2Proc/runPreprocessor.py @@ -494,6 +494,10 @@ def runPreprocessor(self): mvel = np.array(masterSV.getVelocity()) sxyz = np.array(slaveSV.getPosition()) + #to fix abrupt change near zero in baseline grid. JUN-05-2020 + mvelunit = mvel / np.linalg.norm(mvel) + sxyz = sxyz - np.dot ( sxyz-mxyz, mvelunit) * mvelunit + aa = np.linalg.norm(sxyz-mxyz) costheta = (x[1]*x[1] + aa*aa - slvrng*slvrng)/(2.*x[1]*aa) From 5bc4cd64667e0d437c6189f5b697b028b5483012 Mon Sep 17 00:00:00 2001 From: CunrenLiang <56097947+CunrenLiang@users.noreply.github.com> Date: Sat, 6 Jun 2020 00:12:54 -0700 Subject: [PATCH 05/10] fix bug in baseline computation --- components/isceobj/Alos2burstProc/runPreprocessor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/isceobj/Alos2burstProc/runPreprocessor.py b/components/isceobj/Alos2burstProc/runPreprocessor.py index c971810..d71cacb 100644 --- a/components/isceobj/Alos2burstProc/runPreprocessor.py +++ b/components/isceobj/Alos2burstProc/runPreprocessor.py @@ -417,6 +417,10 @@ def runPreprocessor(self): mvel = np.array(masterSV.getVelocity()) sxyz = np.array(slaveSV.getPosition()) + #to fix abrupt change near zero in baseline grid. JUN-05-2020 + mvelunit = mvel / np.linalg.norm(mvel) + sxyz = sxyz - np.dot ( sxyz-mxyz, mvelunit) * mvelunit + aa = np.linalg.norm(sxyz-mxyz) costheta = (x[1]*x[1] + aa*aa - slvrng*slvrng)/(2.*x[1]*aa) From acf0bbb55cb3dfc3a4341039d1a12466ee797b67 Mon Sep 17 00:00:00 2001 From: sssangha Date: Sat, 6 Jun 2020 20:44:00 -0700 Subject: [PATCH 06/10] Negative valid samples/lines now set to 0 --- contrib/stack/topsStack/s1a_isce_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/stack/topsStack/s1a_isce_utils.py b/contrib/stack/topsStack/s1a_isce_utils.py index 9d3c3fc..30261fb 100755 --- a/contrib/stack/topsStack/s1a_isce_utils.py +++ b/contrib/stack/topsStack/s1a_isce_utils.py @@ -193,6 +193,12 @@ def adjustCommonValidRegion(master,slave): master.firstValidLine = max(master.firstValidLine, slave.firstValidLine) master.firstValidSample = max(master.firstValidSample, slave.firstValidSample) + #set to 0 to avoid negative values + if master.firstValidLine<0: + master.firstValidLine=0 + if master.firstValidSample<0: + master.firstValidSample=0 + master.numValidLines = igram_lastValidLine - master.firstValidLine + 1 master.numValidSamples = igram_lastValidSample - master.firstValidSample + 1 From a6e95fa2d3ce17a459c64bd99d5c0c9d866af19d Mon Sep 17 00:00:00 2001 From: piyushrpt Date: Sun, 7 Jun 2020 12:52:06 -0700 Subject: [PATCH 07/10] Updating circleci config to handle alpine changes --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31b9e03..c154bf5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -143,7 +143,7 @@ jobs: name: Install dependencies command: | apk add --no-cache \ - python-dev py-pip bash pigz build-base libffi-dev openssl-dev + python3-dev py3-pip bash pigz build-base libffi-dev openssl-dev pip install \ docker-compose awscli - run: @@ -170,7 +170,7 @@ jobs: name: Install dependencies command: | apk add --no-cache \ - python-dev py-pip bash pigz build-base libffi-dev openssl-dev + python3-dev py3-pip bash pigz build-base libffi-dev openssl-dev pip install \ docker-compose awscli - run: @@ -196,7 +196,7 @@ jobs: name: Install dependencies command: | apk add --no-cache \ - python-dev py-pip bash pigz build-base libffi-dev openssl-dev + python3-dev py3-pip bash pigz build-base libffi-dev openssl-dev pip install \ docker-compose awscli - run: From bb03d8c6523805c7e837750f93234a9afde551b0 Mon Sep 17 00:00:00 2001 From: shitong01 Date: Fri, 15 May 2020 11:31:45 +0800 Subject: [PATCH 08/10] changed made by @cheryltwj to fix orbit file download edge case --- contrib/stack/topsStack/Stack.py | 4 ++-- contrib/stack/topsStack/fetchOrbit.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index 70a4845..aa4105c 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -925,7 +925,7 @@ class sentinelSLC(object): orbit_start_date_time = datetime.datetime.strptime(fields[6].replace('V',''), datefmt) orbit_stop_date_time = datetime.datetime.strptime(fields[7].replace('.EOF',''), datefmt) - if self.start_date_time > orbit_start_date_time and self.start_date_time < orbit_stop_date_time: + if self.start_date_time > orbit_start_date_time and self.stop_date_time < orbit_stop_date_time: self.orbit = os.path.join(orbitDir,orbit) self.orbitType = 'precise' match = True @@ -944,7 +944,7 @@ class sentinelSLC(object): fields = os.path.basename(orbitFile).split('_') orbit_start_date_time = datetime.datetime.strptime(fields[6].replace('V',''), datefmt) orbit_stop_date_time = datetime.datetime.strptime(fields[7].replace('.EOF',''), datefmt) - if self.start_date_time > orbit_start_date_time and self.start_date_time < orbit_stop_date_time: + if self.start_date_time > orbit_start_date_time and self.stop_date_time < orbit_stop_date_time: print ("restituted orbit already exists.") self.orbit = orbitFile self.orbitType = 'restituted' diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index 0244b87..c33324a 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -38,9 +38,11 @@ def FileToTimeStamp(safename): ''' safename = os.path.basename(safename) fields = safename.split('_') + sstamp = [] # sstamp for getting SAFE file start time, not needed for orbit file timestamps try: tstamp = datetime.datetime.strptime(fields[-4], datefmt) + sstamp = datetime.datetime.strptime(fields[-5], datefmt) except: p = re.compile(r'(?<=_)\d{8}') dt2 = p.search(safename).group() @@ -48,7 +50,7 @@ def FileToTimeStamp(safename): satName = fields[0] - return tstamp, satName + return tstamp, satName, sstamp class MyHTMLParser(HTMLParser): @@ -144,7 +146,7 @@ if __name__ == '__main__': inps = cmdLineParse() - fileTS, satName = FileToTimeStamp(inps.input) + fileTS, satName, , fileTSStart = FileToTimeStamp(inps.input) print('Reference time: ', fileTS) print('Satellite name: ', satName) @@ -198,7 +200,7 @@ if __name__ == '__main__': for result in results: tbef, taft, mission = fileToRange(os.path.basename(result)) - if (tbef <= fileTS) and (taft >= fileTS): + if (tbef <= fileTSStart) and (taft >= fileTS): datestr2 = FileToTimeStamp(result)[0].strftime(queryfmt2) match = (server2 + spec[1].replace('aux_', '').upper() + '/' +datestr2+ result + '.EOF') From 0fb49e383edff1e22791d134cba514cb5ace6b43 Mon Sep 17 00:00:00 2001 From: shitong01 Date: Fri, 15 May 2020 11:56:14 +0800 Subject: [PATCH 09/10] fix syntax --- contrib/stack/topsStack/fetchOrbit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index c33324a..cfa43e4 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -146,7 +146,7 @@ if __name__ == '__main__': inps = cmdLineParse() - fileTS, satName, , fileTSStart = FileToTimeStamp(inps.input) + fileTS, satName, fileTSStart = FileToTimeStamp(inps.input) print('Reference time: ', fileTS) print('Satellite name: ', satName) From edea69d4b6216f4ac729eba78f12547807a2751a Mon Sep 17 00:00:00 2001 From: Cheryl Date: Wed, 20 May 2020 13:33:50 +0800 Subject: [PATCH 10/10] added buffer margin to orbit times in get_orbit --- contrib/stack/topsStack/Stack.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index aa4105c..8290ab8 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -912,7 +912,8 @@ class sentinelSLC(object): self.SNWE=[min(lats),max(lats),min(lons),max(lons)] - def get_orbit(self, orbitDir, workDir): + def get_orbit(self, orbitDir, workDir, margin=60.0): + margin = datetime.timedelta(seconds=margin) datefmt = "%Y%m%dT%H%M%S" orbit_files = glob.glob(os.path.join(orbitDir, self.platform + '*.EOF')) if len(orbit_files) == 0: @@ -922,10 +923,10 @@ class sentinelSLC(object): for orbit in orbit_files: orbit = os.path.basename(orbit) fields = orbit.split('_') - orbit_start_date_time = datetime.datetime.strptime(fields[6].replace('V',''), datefmt) - orbit_stop_date_time = datetime.datetime.strptime(fields[7].replace('.EOF',''), datefmt) + orbit_start_date_time = datetime.datetime.strptime(fields[6].replace('V',''), datefmt) + margin + orbit_stop_date_time = datetime.datetime.strptime(fields[7].replace('.EOF',''), datefmt) - margin - if self.start_date_time > orbit_start_date_time and self.stop_date_time < orbit_stop_date_time: + if self.start_date_time >= orbit_start_date_time and self.stop_date_time < orbit_stop_date_time: self.orbit = os.path.join(orbitDir,orbit) self.orbitType = 'precise' match = True @@ -944,7 +945,7 @@ class sentinelSLC(object): fields = os.path.basename(orbitFile).split('_') orbit_start_date_time = datetime.datetime.strptime(fields[6].replace('V',''), datefmt) orbit_stop_date_time = datetime.datetime.strptime(fields[7].replace('.EOF',''), datefmt) - if self.start_date_time > orbit_start_date_time and self.stop_date_time < orbit_stop_date_time: + if self.start_date_time >= orbit_start_date_time and self.stop_date_time < orbit_stop_date_time: print ("restituted orbit already exists.") self.orbit = orbitFile self.orbitType = 'restituted'