From 58f5044b92374a0cd471dd4e09908d3499f7421f Mon Sep 17 00:00:00 2001 From: Eric Jameson Fielding Date: Sun, 5 May 2019 23:31:27 -0700 Subject: [PATCH 01/24] modified prepareUAVSAR_coregStack to support segment of stack --- contrib/stack/README_stripmapStack.txt | 2 +- .../stripmapStack/prepareUAVSAR_coregStack.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/stack/README_stripmapStack.txt b/contrib/stack/README_stripmapStack.txt index e6ae915..fd6c377 100644 --- a/contrib/stack/README_stripmapStack.txt +++ b/contrib/stack/README_stripmapStack.txt @@ -31,7 +31,7 @@ fixImageXml.py -f -i demLat_S37_S31_Lon_W072_W069.dem.wgs84 mkdir download cd download -6- Download the data that that you want to process to the downlowd directory. +6- Download the data that that you want to process to the download directory. 7- once all data have been downloaded, we need to unzip them and move them to different folders and getting ready for unpacking and then SLC generation. This can be done by running the following command in a directory above "download": diff --git a/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py b/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py index 1c175b3..91571b8 100755 --- a/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py +++ b/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# modified to work for different UAVSAR stack segments EJF 2019/05/04 import os import glob @@ -14,15 +15,15 @@ def createParser(): Create command line parser. ''' - parser = argparse.ArgumentParser(description='Unzip Alos zip files.') + parser = argparse.ArgumentParser(description='Prepare UAVSAR SLC Stack files.') parser.add_argument('-i', '--input', dest='input', type=str, required=True, - help='directory which has all dates as directories. Inside each date, zip files are expected.') + help='directory which has all dates.') parser.add_argument('-d', '--dop_file', dest='dopFile', type=str, required=True, - help='Doppler file for the stack.') + help='Doppler file for the stack. Needs to be in directory where command is run.') parser.add_argument('-o', '--output', dest='output', type=str, required=True, help='output directory which will be used for unpacking.') - parser.add_argument('-t', '--text_cmd', dest='text_cmd', type=str, default='source ~/.bash_profile;' - , help='text command to be added to the beginning of each line of the run files. Example : source ~/.bash_profile;') + parser.add_argument('-s', '--segment', dest='segment', type=str, default='1', + help='segment of the UAVSAR stack to prepare. For "s2" use "2", etc. Default is "1" ') return parser @@ -64,14 +65,13 @@ def main(iargs=None): inps = cmdLineParse(iargs) outputDir = os.path.abspath(inps.output) - run_unPack = 'run_unPackAlos' ####################################### - slc_files = glob.glob(os.path.join(inps.input, '*_s5_1x1.slc')) + slc_files = glob.glob(os.path.join(inps.input, '*_s'+segment+'_1x1.slc')) for file in slc_files: imgDate = get_Date(file) print (imgDate) - annFile = file.replace('_s5_1x1.slc','')+'.ann' + annFile = file.replace('_s'+segment+'_1x1.slc','')+'.ann' print (annFile) imgDir = os.path.join(outputDir,imgDate) if not os.path.exists(imgDir): From 9a947b3d14d92a893eb7c4f81f9c05125343e467 Mon Sep 17 00:00:00 2001 From: Eric Jameson Fielding Date: Sun, 5 May 2019 23:39:11 -0700 Subject: [PATCH 02/24] more changes to support segments of UAVSAR stack --- contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py b/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py index 91571b8..f42c3b4 100755 --- a/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py +++ b/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py @@ -67,11 +67,11 @@ def main(iargs=None): outputDir = os.path.abspath(inps.output) ####################################### - slc_files = glob.glob(os.path.join(inps.input, '*_s'+segment+'_1x1.slc')) + slc_files = glob.glob(os.path.join(inps.input, '*_s'+inps.segment+'_1x1.slc')) for file in slc_files: imgDate = get_Date(file) print (imgDate) - annFile = file.replace('_s'+segment+'_1x1.slc','')+'.ann' + annFile = file.replace('_s'+inps.segment+'_1x1.slc','')+'.ann' print (annFile) imgDir = os.path.join(outputDir,imgDate) if not os.path.exists(imgDir): From 8c2e0f94af68c16e1db3a1b3733f539602359893 Mon Sep 17 00:00:00 2001 From: Eric Jameson Fielding Date: Mon, 6 May 2019 00:06:10 -0700 Subject: [PATCH 03/24] changed prepareUAVSAR to rename SLCs --- contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py b/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py index f42c3b4..ec743ec 100755 --- a/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py +++ b/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py @@ -81,7 +81,8 @@ def main(iargs=None): print (cmd) os.system(cmd) - cmd = 'mv ' + file + ' ' + imgDir + slcFile = os.path.join(imgDir, imgDate+'.slc') + cmd = 'mv ' + file + ' ' + slcFile print(cmd) os.system(cmd) @@ -90,7 +91,6 @@ def main(iargs=None): os.system(cmd) shelveFile = os.path.join(imgDir, 'data') - slcFile = os.path.join(imgDir, os.path.basename(file)) write_xml(shelveFile, slcFile) if __name__ == '__main__': From 50703cb8172e1e2be9efcdcfe1f75f8509c78c42 Mon Sep 17 00:00:00 2001 From: Eric Jameson Fielding Date: Wed, 8 May 2019 15:05:08 -0700 Subject: [PATCH 04/24] Modified stackStripMap and Stack.py to work with UAVSAR_STACK data --- contrib/stack/stripmapStack/Stack.py | 13 ++++++++----- contrib/stack/stripmapStack/stackStripMap.py | 5 ++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/contrib/stack/stripmapStack/Stack.py b/contrib/stack/stripmapStack/Stack.py index 74d0df4..f937884 100755 --- a/contrib/stack/stripmapStack/Stack.py +++ b/contrib/stack/stripmapStack/Stack.py @@ -658,7 +658,7 @@ def baselinePair(baselineDir, master, slave): print('Baseline at top/bottom: %f %f'%(bObj.pBaselineTop,bObj.pBaselineBottom)) return (bObj.pBaselineTop+bObj.pBaselineBottom)/2. -def baselineStack(inps,stackMaster,acqDates): +def baselineStack(inps,stackMaster,acqDates,doBaselines=True): from collections import OrderedDict baselineDir = os.path.join(inps.workDir,'baselines') if not os.path.exists(baselineDir): @@ -671,7 +671,10 @@ def baselineStack(inps,stackMaster,acqDates): for slv in acqDates: if slv != stackMaster: slave = os.path.join(inps.slcDir, slv) - baselineDict[slv]=baselinePair(baselineDir, master, slave) + if doBaselines: + baselineDict[slv]=baselinePair(baselineDir, master, slave) + else: + baselineDict[slv] = 0.0 # set slave baselines to zero if not calculated t = datetime.datetime.strptime(slv, datefmt) timeDict[slv] = t - t0 else: @@ -680,11 +683,11 @@ def baselineStack(inps,stackMaster,acqDates): return baselineDict, timeDict -def selectPairs(inps,stackMaster, slaveDates, acuisitionDates): - baselineDict, timeDict = baselineStack(inps, stackMaster, acuisitionDates) +def selectPairs(inps,stackMaster, slaveDates, acuisitionDates,doBaselines=True): + + baselineDict, timeDict = baselineStack(inps, stackMaster, acuisitionDates,doBaselines) for slave in slaveDates: print (slave,' : ' , baselineDict[slave]) - numDates = len(acuisitionDates) pairs = [] for i in range(numDates-1): diff --git a/contrib/stack/stripmapStack/stackStripMap.py b/contrib/stack/stripmapStack/stackStripMap.py index 07c1f68..b7495dd 100755 --- a/contrib/stack/stripmapStack/stackStripMap.py +++ b/contrib/stack/stripmapStack/stackStripMap.py @@ -318,7 +318,10 @@ def main(iargs=None): if not os.path.exists(runDir): os.makedirs(runDir) - pairs = selectPairs(inps,stackMasterDate, slaveDates, acquisitionDates) + if inps.sensor.lower() == 'uavsar_stack': # don't try to calculate baselines for UAVSAR_STACK data + pairs = selectPairs(inps,stackMasterDate, slaveDates, acquisitionDates,doBaselines=False) + else: + pairs = selectPairs(inps,stackMasterDate, slaveDates, acquisitionDates,doBaselines=True) print ('number of pairs: ', len(pairs)) ###If only a summary is requested quit after this From d267903d66bfe99ddf2f7f17e49ca352a4ba78e1 Mon Sep 17 00:00:00 2001 From: "Eric J. Fielding" Date: Wed, 8 May 2019 17:07:16 -0700 Subject: [PATCH 05/24] added back the "--text_cmd" command argument parsing --- contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py b/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py index ec743ec..95c28d6 100755 --- a/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py +++ b/contrib/stack/stripmapStack/prepareUAVSAR_coregStack.py @@ -24,6 +24,8 @@ def createParser(): help='output directory which will be used for unpacking.') parser.add_argument('-s', '--segment', dest='segment', type=str, default='1', help='segment of the UAVSAR stack to prepare. For "s2" use "2", etc. Default is "1" ') + parser.add_argument('-t', '--text_cmd', dest='text_cmd', type=str, default='source ~/.bash_profile;', + help='text command to be added to the beginning of each line of the run files. Default: source ~/.bash_profile;') return parser From 919ce84b9bb1124bd31fb5d04b1eb4e77ed9b0bd Mon Sep 17 00:00:00 2001 From: Eric Jameson Fielding Date: Wed, 22 May 2019 15:26:13 -0700 Subject: [PATCH 06/24] Modified the contrib/stack/stripmapStack/Stack.py code to write out baseline files with baselines set to 0.0 instead of skipping them. --- contrib/stack/stripmapStack/Stack.py | 48 +++++++++++++++------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/contrib/stack/stripmapStack/Stack.py b/contrib/stack/stripmapStack/Stack.py index f937884..8c736b8 100755 --- a/contrib/stack/stripmapStack/Stack.py +++ b/contrib/stack/stripmapStack/Stack.py @@ -632,31 +632,38 @@ class workflow(object): ############################## -def baselinePair(baselineDir, master, slave): +def baselinePair(baselineDir, master, slave,doBaselines=True): - try: - mdb = shelve.open( os.path.join(master, 'raw'), flag='r') - sdb = shelve.open( os.path.join(slave, 'raw'), flag='r') - except: - mdb = shelve.open( os.path.join(master, 'data'), flag='r') - sdb = shelve.open( os.path.join(slave, 'data'), flag='r') + if doBaselines: # open files to calculate baselines + try: + mdb = shelve.open( os.path.join(master, 'raw'), flag='r') + sdb = shelve.open( os.path.join(slave, 'raw'), flag='r') + except: + mdb = shelve.open( os.path.join(master, 'data'), flag='r') + sdb = shelve.open( os.path.join(slave, 'data'), flag='r') - mFrame = mdb['frame'] - sFrame = sdb['frame'] + mFrame = mdb['frame'] + sFrame = sdb['frame'] - bObj = Baseline() - bObj.configure() - bObj.wireInputPort(name='masterFrame', object=mFrame) - bObj.wireInputPort(name='slaveFrame', object=sFrame) - bObj.baseline() + bObj = Baseline() + bObj.configure() + bObj.wireInputPort(name='masterFrame', object=mFrame) + bObj.wireInputPort(name='slaveFrame', object=sFrame) + bObj.baseline() # calculate baseline from orbits + pBaselineBottom = bObj.pBaselineBottom + pBaselineTop = bObj.pBaselineTop + else: # set baselines to zero if not calculated + pBaselineBottom = 0.0 + pBaselineTop = 0.0 + baselineOutName = os.path.basename(master) + "_" + os.path.basename(slave) + ".txt" f = open(os.path.join(baselineDir, baselineOutName) , 'w') - f.write("PERP_BASELINE_BOTTOM " + str(bObj.pBaselineBottom) + '\n') - f.write("PERP_BASELINE_TOP " + str(bObj.pBaselineTop) + '\n') + f.write("PERP_BASELINE_BOTTOM " + str(pBaselineBottom) + '\n') + f.write("PERP_BASELINE_TOP " + str(pBaselineTop) + '\n') f.close() - print('Baseline at top/bottom: %f %f'%(bObj.pBaselineTop,bObj.pBaselineBottom)) - return (bObj.pBaselineTop+bObj.pBaselineBottom)/2. + print('Baseline at top/bottom: %f %f'%(pBaselineTop,pBaselineBottom)) + return (pBaselineTop+pBaselineBottom)/2. def baselineStack(inps,stackMaster,acqDates,doBaselines=True): from collections import OrderedDict @@ -671,10 +678,7 @@ def baselineStack(inps,stackMaster,acqDates,doBaselines=True): for slv in acqDates: if slv != stackMaster: slave = os.path.join(inps.slcDir, slv) - if doBaselines: - baselineDict[slv]=baselinePair(baselineDir, master, slave) - else: - baselineDict[slv] = 0.0 # set slave baselines to zero if not calculated + baselineDict[slv]=baselinePair(baselineDir, master, slave, doBaselines) t = datetime.datetime.strptime(slv, datefmt) timeDict[slv] = t - t0 else: From 5d6a731753d2976739f92d6c2613cf22017d51f1 Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Thu, 30 Jan 2020 22:26:46 -0800 Subject: [PATCH 07/24] Remove internal usage of ISCE_HOME env variable ISCE_HOME was only used to get the location of the default logging config. Lots of scripts were using boilerplate to set up this config, so I added an `isce.logging` helper module which is the same as builtin python logging but already has the configuration defaults set up for isce. ISCE_HOME setup is retained in the toplevel `__init__.py` but can now be removed without affecting functionality. --- .circleci/config.yml | 2 +- __init__.py | 7 ++++++- applications/CalculatePegPoint.py | 6 +----- applications/calculateBaseline.py | 6 +----- applications/createGeneric.py | 6 +----- applications/extractHDROrbit.py | 6 +----- applications/focus.py | 6 +----- applications/insarApp.py | 8 +------- applications/isceApp.py | 7 +------ applications/make_raw.py | 10 +--------- applications/rtcApp.py | 9 +-------- applications/scansarApp.py | 11 +---------- applications/stripmapApp.py | 9 +-------- applications/topsApp.py | 9 +-------- applications/topsOffsetApp.py | 9 +-------- applications/viewMetadata.py | 9 +-------- components/isceobj/Location/Offset.py | 6 +----- components/isceobj/Scene/test/testTrack.py | 9 +-------- components/isceobj/Util/geo/__init__.py | 4 ---- components/iscesys/Component/Configurable.py | 5 +---- components/iscesys/DataRetriever/DataRetriever.py | 7 +------ components/stdproc/orbit/orbitLib/CalcSchHeightVel.py | 5 ----- contrib/demUtils/demstitcher/DemStitcher.py | 7 +------ contrib/demUtils/demstitcher/DemStitcherV3.py | 6 +----- contrib/demUtils/swbdstitcher/SWBDStitcher.py | 6 +----- contrib/demUtils/watermask/WaterMask.py | 7 +------ contrib/issi/applications/ISSI.py | 5 +---- 27 files changed, 30 insertions(+), 157 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 98bb5bd..25ba14b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,7 +72,7 @@ jobs: set -ex pwd . /opt/conda/bin/activate root - export ISCE_HOME=/root/project/install/isce + ISCE_HOME=/root/project/install/isce export PATH="$ISCE_HOME/bin:$ISCE_HOME/applications:/opt/conda/bin:$PATH" export PYTHONPATH="/root/project/install:$PYTHONPATH" export LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH" diff --git a/__init__.py b/__init__.py index a6e2c5b..f2f76a8 100755 --- a/__init__.py +++ b/__init__.py @@ -32,7 +32,12 @@ version = release_history # compatibility alias __version__ = release_version import sys, os -isce_path = os.path.split(os.path.abspath(__file__))[0] +isce_path = os.path.dirname(os.path.abspath(__file__)) + +import logging +from logging.config import fileConfig as _fc +_fc(os.path.join(isce_path, 'defaults', 'logging', 'logging.conf')) + sys.path.insert(1,isce_path) sys.path.insert(1,os.path.join(isce_path,'applications')) sys.path.insert(1,os.path.join(isce_path,'components')) diff --git a/applications/CalculatePegPoint.py b/applications/CalculatePegPoint.py index 5f52ca4..a300c0e 100755 --- a/applications/CalculatePegPoint.py +++ b/applications/CalculatePegPoint.py @@ -31,12 +31,8 @@ -import os import math -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging from iscesys.Compatibility import Compatibility Compatibility.checkPythonVersion() from isceobj.Location.Peg import Peg diff --git a/applications/calculateBaseline.py b/applications/calculateBaseline.py index 375fec3..a6eeff5 100755 --- a/applications/calculateBaseline.py +++ b/applications/calculateBaseline.py @@ -30,11 +30,7 @@ -import os -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging from iscesys.Compatibility import Compatibility Compatibility.checkPythonVersion() from iscesys.Component.FactoryInit import FactoryInit diff --git a/applications/createGeneric.py b/applications/createGeneric.py index aebb6ba..c7f706b 100755 --- a/applications/createGeneric.py +++ b/applications/createGeneric.py @@ -30,11 +30,7 @@ -import os -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging import isceobj from iscesys.Component.FactoryInit import FactoryInit diff --git a/applications/extractHDROrbit.py b/applications/extractHDROrbit.py index 421b60b..fd7278c 100755 --- a/applications/extractHDROrbit.py +++ b/applications/extractHDROrbit.py @@ -30,12 +30,8 @@ -import os import datetime -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging from iscesys.Compatibility import Compatibility Compatibility.checkPythonVersion() from iscesys.Component.FactoryInit import FactoryInit diff --git a/applications/focus.py b/applications/focus.py index 280a9d2..53f64ed 100755 --- a/applications/focus.py +++ b/applications/focus.py @@ -30,12 +30,8 @@ -import os import math -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging import isceobj from iscesys.Component.FactoryInit import FactoryInit from iscesys.DateTimeUtil.DateTimeUtil import DateTimeUtil as DTU diff --git a/applications/insarApp.py b/applications/insarApp.py index 0ab2f0a..8136d1d 100755 --- a/applications/insarApp.py +++ b/applications/insarApp.py @@ -34,8 +34,7 @@ from __future__ import print_function import time import os import sys -import logging -import logging.config +from isce import logging import isce import isceobj @@ -46,11 +45,6 @@ from iscesys.Component.Configurable import SELF import isceobj.InsarProc as InsarProc from isceobj.Scene.Frame import FrameMixin -logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', 'logging', - 'logging.conf') -) - logger = logging.getLogger('isce.insar') diff --git a/applications/isceApp.py b/applications/isceApp.py index 7aec323..f67d08d 100755 --- a/applications/isceApp.py +++ b/applications/isceApp.py @@ -41,8 +41,7 @@ import datetime import os import sys import math -import logging -import logging.config +from isce import logging import isce import isceobj @@ -1439,10 +1438,6 @@ class IsceApp(Application, FrameMixin): os.chdir(self.outputDir) ##change working directory to given output directory ##read configfile only here so that log path is in output directory - logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', 'logging', - 'logging.conf') - ) logger = logging.getLogger('isce.isceProc') logger.info(self.intromsg) self._isce.dataDirectory = self.outputDir diff --git a/applications/make_raw.py b/applications/make_raw.py index a3b1f44..3c2af20 100755 --- a/applications/make_raw.py +++ b/applications/make_raw.py @@ -27,16 +27,8 @@ # Author: Walter Szeliga #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - -import os -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) - import isce +from isce import logging from iscesys.Compatibility import Compatibility from iscesys.Component.Component import Component, Port from isceobj.Planet.Ellipsoid import Ellipsoid diff --git a/applications/rtcApp.py b/applications/rtcApp.py index 836151e..05d4575 100755 --- a/applications/rtcApp.py +++ b/applications/rtcApp.py @@ -30,10 +30,8 @@ import time -import os import sys -import logging -import logging.config +from isce import logging import isce import isceobj @@ -44,11 +42,6 @@ from iscesys.Component.Configurable import SELF from isceobj import RtcProc from isceobj.Util.decorators import use_api -logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', 'logging', - 'logging.conf') -) - logger = logging.getLogger('isce.grdsar') diff --git a/applications/scansarApp.py b/applications/scansarApp.py index 55fe196..a5338ce 100755 --- a/applications/scansarApp.py +++ b/applications/scansarApp.py @@ -27,13 +27,9 @@ # Authors: Giangi Sacco, Eric Gurrola #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - import time -import os import sys -import logging -import logging.config +from isce import logging import isce import isceobj @@ -43,11 +39,6 @@ from iscesys.Compatibility import Compatibility from iscesys.Component.Configurable import SELF from isceobj import ScansarProc -logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', 'logging', - 'logging.conf') -) - logger = logging.getLogger('isce.insar') diff --git a/applications/stripmapApp.py b/applications/stripmapApp.py index 6f76b66..3793ac6 100755 --- a/applications/stripmapApp.py +++ b/applications/stripmapApp.py @@ -35,10 +35,8 @@ from __future__ import print_function import time -import os import sys -import logging -import logging.config +from isce import logging import isce import isceobj @@ -50,11 +48,6 @@ import isceobj.StripmapProc as StripmapProc from isceobj.Scene.Frame import FrameMixin from isceobj.Util.decorators import use_api -logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', 'logging', - 'logging.conf') -) - logger = logging.getLogger('isce.insar') diff --git a/applications/topsApp.py b/applications/topsApp.py index 7bc2dc9..7f76a37 100755 --- a/applications/topsApp.py +++ b/applications/topsApp.py @@ -34,10 +34,8 @@ import time -import os import sys -import logging -import logging.config +from isce import logging import isce import isceobj @@ -47,11 +45,6 @@ from iscesys.Compatibility import Compatibility from iscesys.Component.Configurable import SELF from isceobj import TopsProc -logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', 'logging', - 'logging.conf') -) - logger = logging.getLogger('isce.insar') diff --git a/applications/topsOffsetApp.py b/applications/topsOffsetApp.py index f2049a8..52b1e3b 100755 --- a/applications/topsOffsetApp.py +++ b/applications/topsOffsetApp.py @@ -30,10 +30,8 @@ import time -import os import sys -import logging -import logging.config +from isce import logging import isce import isceobj @@ -42,11 +40,6 @@ from isce.applications.topsApp import TopsInSAR from iscesys.Component.Application import Application from isceobj.Util.decorators import use_api -logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', 'logging', - 'logging.conf') -) - logger = logging.getLogger('isce.insar') WINDOW_SIZE_WIDTH = Application.Parameter( diff --git a/applications/viewMetadata.py b/applications/viewMetadata.py index 3051193..8021ff0 100755 --- a/applications/viewMetadata.py +++ b/applications/viewMetadata.py @@ -27,14 +27,7 @@ # Author: Walter Szeliga #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - -import os -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging from iscesys.Compatibility import Compatibility Compatibility.checkPythonVersion() from iscesys.Component.FactoryInit import FactoryInit diff --git a/components/isceobj/Location/Offset.py b/components/isceobj/Location/Offset.py index 8bcff9b..96960f4 100755 --- a/components/isceobj/Location/Offset.py +++ b/components/isceobj/Location/Offset.py @@ -29,11 +29,7 @@ import math -import os -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging from isceobj.Util.decorators import type_check, force, pickled, logged import numpy as np diff --git a/components/isceobj/Scene/test/testTrack.py b/components/isceobj/Scene/test/testTrack.py index b67cc5c..51883d1 100755 --- a/components/isceobj/Scene/test/testTrack.py +++ b/components/isceobj/Scene/test/testTrack.py @@ -27,14 +27,7 @@ # Author: Walter Szeliga #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - -import os -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging from isceobj.Sensor.ERS import ERS from isceobj.Scene.Track import Track logger = logging.getLogger("testTrack") diff --git a/components/isceobj/Util/geo/__init__.py b/components/isceobj/Util/geo/__init__.py index 85f13b9..0c7ffe3 100755 --- a/components/isceobj/Util/geo/__init__.py +++ b/components/isceobj/Util/geo/__init__.py @@ -45,10 +45,6 @@ ellipsoid oblate ellipsoid of revolution (e.g, WGS84) with all the See mainpage.txt for a complete dump of geo's philosophy-- otherwise, use the docstrings. """ -import os -isce_path = os.getenv("ISCE_HOME") ## \namespace geo Vector- and Affine-spaces, on Earth __all__ = ['euclid', 'coordinates', 'ellipsoid', 'charts', 'affine', 'motion'] - - diff --git a/components/iscesys/Component/Configurable.py b/components/iscesys/Component/Configurable.py index 5e9b9ff..3cff30c 100755 --- a/components/iscesys/Component/Configurable.py +++ b/components/iscesys/Component/Configurable.py @@ -32,10 +32,7 @@ from __future__ import print_function import os import sys import operator -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging from iscesys.DictUtils.DictUtils import DictUtils as DU from iscesys.Compatibility import Compatibility Compatibility.checkPythonVersion() diff --git a/components/iscesys/DataRetriever/DataRetriever.py b/components/iscesys/DataRetriever/DataRetriever.py index 4f1a4b2..1599ff8 100755 --- a/components/iscesys/DataRetriever/DataRetriever.py +++ b/components/iscesys/DataRetriever/DataRetriever.py @@ -37,8 +37,7 @@ import isce import zipfile import os import sys -import logging -import logging.config +from isce import logging from iscesys.Component.Component import Component import shutil from urllib import request @@ -325,8 +324,4 @@ class DataRetriever(Component): # logger not defined until baseclass is called if not self.logger: - logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf') - ) self.logger = logging.getLogger('isce.iscesys.DataRetriever') diff --git a/components/stdproc/orbit/orbitLib/CalcSchHeightVel.py b/components/stdproc/orbit/orbitLib/CalcSchHeightVel.py index 684cbeb..53da593 100755 --- a/components/stdproc/orbit/orbitLib/CalcSchHeightVel.py +++ b/components/stdproc/orbit/orbitLib/CalcSchHeightVel.py @@ -29,10 +29,8 @@ -import os import logging import math -import logging.config from iscesys.Compatibility import Compatibility @@ -40,9 +38,6 @@ from isceobj.Planet import Planet from isceobj import Constants as CN from iscesys.Component.Component import Component, Port -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) - RANGE_SAMPLING_RATE = Component.Parameter('rangeSamplingRate', public_name='range sampling rate', type=float, diff --git a/contrib/demUtils/demstitcher/DemStitcher.py b/contrib/demUtils/demstitcher/DemStitcher.py index ea3d51f..ba3e1e3 100755 --- a/contrib/demUtils/demstitcher/DemStitcher.py +++ b/contrib/demUtils/demstitcher/DemStitcher.py @@ -43,8 +43,7 @@ import os import sys import math import urllib.request, urllib.parse, urllib.error -import logging -import logging.config +from isce import logging from iscesys.Component.Component import Component import xml.etree.ElementTree as ET @@ -1013,10 +1012,6 @@ class DemStitcher(Component): # logger not defined until baseclass is called if not self.logger: - logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf') - ) self.logger = logging.getLogger('isce.contrib.demUtils.DemStitcher') url = property(getUrl,setUrl) diff --git a/contrib/demUtils/demstitcher/DemStitcherV3.py b/contrib/demUtils/demstitcher/DemStitcherV3.py index c2b486d..6c5f02c 100755 --- a/contrib/demUtils/demstitcher/DemStitcherV3.py +++ b/contrib/demUtils/demstitcher/DemStitcherV3.py @@ -39,8 +39,7 @@ from ctypes import cdll import os import sys import urllib.request, urllib.error, urllib.parse -import logging -import logging.config +from isce import logging from iscesys.Component.Component import Component from contrib.demUtils.DemStitcher import DemStitcher as DS #Parameters definitions @@ -291,7 +290,4 @@ class DemStitcher(DS): #it's /srtm/version2_1/SRTM(1,3) self._remove = ['.jpg','.xml'] if not self.logger: - logging.config.fileConfig( - os.environ['ISCE_HOME'] + '/library/applications/logging.conf' - ) self.logger = logging.getLogger('isce.contrib.demUtils.DemStitcherV3') diff --git a/contrib/demUtils/swbdstitcher/SWBDStitcher.py b/contrib/demUtils/swbdstitcher/SWBDStitcher.py index cca55fb..e299e9b 100755 --- a/contrib/demUtils/swbdstitcher/SWBDStitcher.py +++ b/contrib/demUtils/swbdstitcher/SWBDStitcher.py @@ -39,9 +39,8 @@ from ctypes import cdll import numpy as np import os import sys -import logging +from isce import logging import math -import logging.config import urllib.request, urllib.parse, urllib.error from iscesys.Component.Component import Component from contrib.demUtils.DemStitcher import DemStitcher @@ -315,9 +314,6 @@ class SWBDStitcher(DemStitcher): #it's /srtm/version2_1/SRTM(1,3) self._remove = ['.jpg','.xml'] if not self.logger: - logging.config.fileConfig( - os.environ['ISCE_HOME'] + '/library/applications/logging.conf' - ) self.logger = logging.getLogger('isce.contrib.demUtils.SWBDStitcher') self.parameter_list = self.parameter_list + super(DemStitcher,self).parameter_list diff --git a/contrib/demUtils/watermask/WaterMask.py b/contrib/demUtils/watermask/WaterMask.py index 766060b..57b5603 100755 --- a/contrib/demUtils/watermask/WaterMask.py +++ b/contrib/demUtils/watermask/WaterMask.py @@ -35,8 +35,7 @@ import sys import math from html.parser import HTMLParser import urllib.request, urllib.parse, urllib.error -import logging -import logging.config +from isce import logging from iscesys.Component.Component import Component import zipfile import os @@ -979,10 +978,6 @@ class MaskStitcher(Component): # logger not defined until baseclass is called if not self.logger: - logging.config.fileConfig( - os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf') - ) self.logger = logging.getLogger('isce.contrib.demUtils.MaskStitcher') utl = property(getUrl,setUrl) diff --git a/contrib/issi/applications/ISSI.py b/contrib/issi/applications/ISSI.py index 330905b..9146035 100755 --- a/contrib/issi/applications/ISSI.py +++ b/contrib/issi/applications/ISSI.py @@ -32,10 +32,7 @@ import os import math -import logging -import logging.config -logging.config.fileConfig(os.path.join(os.environ['ISCE_HOME'], 'defaults', - 'logging', 'logging.conf')) +from isce import logging import isce from iscesys.Component.FactoryInit import FactoryInit From dbeb7f512109025173e55deda747827797a5b27e Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Tue, 4 Feb 2020 16:03:25 -0800 Subject: [PATCH 08/24] Remove old comment --- applications/isceApp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/applications/isceApp.py b/applications/isceApp.py index f67d08d..4dd6c36 100755 --- a/applications/isceApp.py +++ b/applications/isceApp.py @@ -1437,7 +1437,6 @@ class IsceApp(Application, FrameMixin): sys.exit("Could not find the output directory: %s" % self.outputDir) os.chdir(self.outputDir) ##change working directory to given output directory - ##read configfile only here so that log path is in output directory logger = logging.getLogger('isce.isceProc') logger.info(self.intromsg) self._isce.dataDirectory = self.outputDir From d7444e83ca5e4781d9ed8fb2367b0dd2a96b6491 Mon Sep 17 00:00:00 2001 From: vbrancat Date: Mon, 10 Feb 2020 15:11:45 -0800 Subject: [PATCH 09/24] Modified Ionospheric phase correction. It fixs previous version bugs and adds ionospheric phase correction facilities for grounded ice areas --- applications/stripmapApp.py | 9 +- .../isceobj/StripmapProc/runDispersive.py | 240 +++++++++--------- 2 files changed, 124 insertions(+), 125 deletions(-) diff --git a/applications/stripmapApp.py b/applications/stripmapApp.py index 938cafb..50c7de5 100755 --- a/applications/stripmapApp.py +++ b/applications/stripmapApp.py @@ -391,6 +391,13 @@ RENDERER = Application.Parameter( ) ) +DISPERSIVE_FILTER_FILLING_METHOD = Application.Parameter('dispersive_filling_method', + public_name = 'dispersive filter filling method', + default='nearest_neighbour', + type=str, + mandatory=False, + doc='method to fill the holes left by masking the ionospheric phase estimate') + DISPERSIVE_FILTER_KERNEL_XSIZE = Application.Parameter('kernel_x_size', public_name='dispersive filter kernel x-size', default=800, @@ -446,7 +453,6 @@ DISPERSIVE_FILTER_COHERENCE_THRESHOLD = Application.Parameter('dispersive_filter type=float, mandatory=False, doc='Coherence threshold to generate a mask file which gets used in the iterative filtering of the dispersive and non-disperive phase') - #Facility declarations MASTER = Application.Facility( @@ -555,6 +561,7 @@ class _RoiBase(Application, FrameMixin): PICKLE_LOAD_DIR, RENDERER, DO_DISPERSIVE, + DISPERSIVE_FILTER_FILLING_METHOD, DISPERSIVE_FILTER_KERNEL_XSIZE, DISPERSIVE_FILTER_KERNEL_YSIZE, DISPERSIVE_FILTER_KERNEL_SIGMA_X, diff --git a/components/isceobj/StripmapProc/runDispersive.py b/components/isceobj/StripmapProc/runDispersive.py index 305022b..d5389de 100644 --- a/components/isceobj/StripmapProc/runDispersive.py +++ b/components/isceobj/StripmapProc/runDispersive.py @@ -8,8 +8,11 @@ import isceobj from isceobj.Constants import SPEED_OF_LIGHT import numpy as np import gdal - +from scipy.ndimage import median_filter +from astropy.convolution import convolve from scipy import ndimage +import numpy as np + try: import cv2 except ImportError: @@ -30,19 +33,8 @@ def getValue(dataFile, band, y_ref, x_ref): ds = None return ref[0][0] -def check_consistency(lowBandIgram, highBandIgram, outputDir): - - jumpFile = os.path.join(outputDir , "jumps.bil") - cmd = 'imageMath.py -e="round((a_1-b_1)/(2.0*PI))" --a={0} --b={1} -o {2} -t float -s BIL'.format(lowBandIgram, highBandIgram, jumpFile) - print(cmd) - os.system(cmd) - - return jumpFile - - - -def dispersive_nonDispersive(lowBandIgram, highBandIgram, f0, fL, fH, outDispersive, outNonDispersive, jumpFile, y_ref=None, x_ref=None, m=None , d=None): +def dispersive_nonDispersive(lowBandIgram, highBandIgram, f0, fL, fH, outDispersive, outNonDispersive, y_ref=None, x_ref=None, m=None , d=None): if y_ref and x_ref: refL = getValue(lowBandIgram, 2, y_ref, x_ref) @@ -58,100 +50,80 @@ def dispersive_nonDispersive(lowBandIgram, highBandIgram, f0, fL, fH, outDispers if m and d: coef = (fL*fH)/(f0*(fH**2 - fL**2)) - #cmd = 'imageMath.py -e="{0}*((a_1-{8}-2*PI*c)*{1}-(b_1-{9}-2*PI*(c+f))*{2})" --a={3} --b={4} --c={5} --f={6} -o {7} -t float32 -s BIL'.format(coef,fH, fL, lowBandIgram, highBandIgram, m , d, outDispersive, refL, refH) - cmd = 'imageMath.py -e="{0}*((a_1-2*PI*c)*{1}-(b_1+(2.0*PI*g)-2*PI*(c+f))*{2})" --a={3} --b={4} --c={5} --f={6} --g={7} -o {8} -t float32 -s BIL'.format(coef,fH, fL, lowBandIgram, highBandIgram, m , d, jumpFile, outDispersive) + cmd = 'imageMath.py -e="{0}*((a_1-2*PI*c)*{1}-(b_1+(2.0*PI)-2*PI*(c+f))*{2})" --a={3} --b={4} --c={5} --f={6} -o {7} -t float32 -s BIL'.format(coef,fH, fL, lowBandIgram, highBandIgram, m , d, outDispersive) print(cmd) os.system(cmd) coefn = f0/(fH**2-fL**2) - #cmd = 'imageMath.py -e="{0}*((a_1-{8}-2*PI*c)*{1}-(b_1-{9}-2*PI*(c+f))*{2})" --a={3} --b={4} --c={5} --f={6} -o {7} -t float32 -s BIL'.format(coefn,fH, fL, highBandIgram, lowBandIgram, m , d, outNonDispersive, refH, refL) - cmd = 'imageMath.py -e="{0}*((a_1+(2.0*PI*g)-2*PI*c)*{1}-(b_1-2*PI*(c+f))*{2})" --a={3} --b={4} --c={5} --f={6} --g={7} -o {8} -t float32 -s BIL'.format(coefn,fH, fL, highBandIgram, lowBandIgram, m , d, jumpFile, outNonDispersive) + cmd = 'imageMath.py -e="{0}*((a_1+(2.0*PI)-2*PI*c)*{1}-(b_1-2*PI*(c+f))*{2})" --a={3} --b={4} --c={5} --f={6} -o {7} -t float32 -s BIL'.format(coefn,fH, fL, highBandIgram, lowBandIgram, m , d, outNonDispersive) print(cmd) os.system(cmd) else: coef = (fL*fH)/(f0*(fH**2 - fL**2)) - #cmd = 'imageMath.py -e="{0}*((a_1-{6})*{1}-(b_1-{7})*{2})" --a={3} --b={4} -o {5} -t float32 -s BIL'.format(coef,fH, fL, lowBandIgram, highBandIgram, outDispersive, refL, refH) - cmd = 'imageMath.py -e="{0}*(a_1*{1}-(b_1+2.0*PI*c)*{2})" --a={3} --b={4} --c={5} -o {6} -t float32 -s BIL'.format(coef,fH, fL, lowBandIgram, highBandIgram, jumpFile, outDispersive) + cmd = 'imageMath.py -e="{0}*(a_1*{1}-(b_1+2.0*PI)*{2})" --a={3} --b={4} -o {5} -t float32 -s BIL'.format(coef,fH, fL, lowBandIgram, highBandIgram, outDispersive) print(cmd) os.system(cmd) coefn = f0/(fH**2-fL**2) - #cmd = 'imageMath.py -e="{0}*((a_1-{6})*{1}-(b_1-{7})*{2})" --a={3} --b={4} -o {5} -t float32 -s BIL'.format(coefn,fH, fL, highBandIgram, lowBandIgram, outNonDispersive, refH, refL) - cmd = 'imageMath.py -e="{0}*((a_1+2.0*PI*c)*{1}-(b_1)*{2})" --a={3} --b={4} --c={5} -o {6} -t float32 -s BIL'.format(coefn,fH, fL, highBandIgram, lowBandIgram, jumpFile, outNonDispersive) + cmd = 'imageMath.py -e="{0}*((a_1+2.0*PI)*{1}-(b_1)*{2})" --a={3} --b={4} -o {5} -t float32 -s BIL'.format(coefn,fH, fL, highBandIgram, lowBandIgram, outNonDispersive) print(cmd) os.system(cmd) return None +def std_iono_mean_coh(f0,fL,fH,coh_mean,rgLooks,azLooks): + + # From Liao et al., Remote Sensing of Environment 2018 + + # STD sub-band at average coherence value (Eq. 8) + Nb = (rgLooks*azLooks)/3.0 + coeffA = (np.sqrt(2.0*Nb))**(-1) + coeffB = np.sqrt(1-coh_mean**2)/coh_mean + std_subbands = coeffA * coeffB + + # STD Ionosphere (Eq. 7) + coeffC = np.sqrt(1+(fL/fH)**2) + coeffD = (fH*fL*fH)/(f0*(fH**2-fL**2)) + std_iono = coeffC*coeffD*std_subbands + + return std_iono + def theoretical_variance_fromSubBands(self, f0, fL, fH, B, Sig_phi_iono, Sig_phi_nonDisp,N): - # Calculating the theoretical variance of the - # ionospheric phase based on the coherence of - # the sub-band interferograns + + # Calculating the theoretical variance of the ionospheric phase based on the coherence of the sub-band interferograns ifgDirname = os.path.join(self.insar.ifgDirname, self.insar.lowBandSlcDirname) lowBandCoherence = os.path.join(ifgDirname , self.insar.coherenceFilename) Sig_phi_L = os.path.join(ifgDirname , 'filt_' + self.insar.ifgFilename + ".sig") ifgDirname = os.path.join(self.insar.ifgDirname, self.insar.highBandSlcDirname) - #highBandIgram = os.path.join(ifgDirname , 'filt_' + self.insar.ifgFilename + ".unw") - - #ifgDirname = os.path.dirname(self.insar.lowBandIgram) - #lowBandCoherence = os.path.join(ifgDirname , self.insar.coherenceFilename) - #Sig_phi_L = os.path.join(ifgDirname , 'filt_' + self.insar.ifgFilename + ".sig") - - #ifgDirname = os.path.dirname(self.insar.highBandIgram) highBandCoherence = os.path.join(ifgDirname , self.insar.coherenceFilename) Sig_phi_H = os.path.join(ifgDirname , 'filt_' + self.insar.ifgFilename + ".sig") - - #N = self.numberAzimuthLooks*self.numberRangeLooks - #PI = np.pi - #fL,f0,fH,B = getBandFrequencies(inps) - #cL = read(inps.lowBandCoherence,bands=[1]) - #cL = cL[0,:,:] - #cL[cL==0.0]=0.001 cmd = 'imageMath.py -e="sqrt(1-a**2)/a/sqrt(2.0*{0})" --a={1} -o {2} -t float -s BIL'.format(N, lowBandCoherence, Sig_phi_L) + print(cmd) os.system(cmd) - #Sig_phi_L = np.sqrt(1-cL**2)/cL/np.sqrt(2.*N) - - #cH = read(inps.highBandCoherence,bands=[1]) - #cH = cH[0,:,:] - #cH[cH==0.0]=0.001 - cmd = 'imageMath.py -e="sqrt(1-a**2)/a/sqrt(2.0*{0})" --a={1} -o {2} -t float -s BIL'.format(N, highBandCoherence, Sig_phi_H) print(cmd) os.system(cmd) - #Sig_phi_H = np.sqrt(1-cH**2)/cH/np.sqrt(2.0*N) coef = (fL*fH)/(f0*(fH**2 - fL**2)) cmd = 'imageMath.py -e="sqrt(({0}**2)*({1}**2)*(a**2) + ({0}**2)*({2}**2)*(b**2))" --a={3} --b={4} -o {5} -t float -s BIL'.format(coef, fL, fH, Sig_phi_L, Sig_phi_H, Sig_phi_iono) os.system(cmd) - #Sig_phi_iono = np.sqrt((coef**2)*(fH**2)*Sig_phi_H**2 + (coef**2)*(fL**2)*Sig_phi_L**2) - #length, width = Sig_phi_iono.shape - - #outFileIono = os.path.join(inps.outDir, 'Sig_iono.bil') - #write(Sig_phi_iono, outFileIono, 1, 6) - #write_xml(outFileIono, length, width) - coef_non = f0/(fH**2 - fL**2) cmd = 'imageMath.py -e="sqrt(({0}**2)*({1}**2)*(a**2) + ({0}**2)*({2}**2)*(b**2))" --a={3} --b={4} -o {5} -t float -s BIL'.format(coef_non, fL, fH, Sig_phi_L, Sig_phi_H, Sig_phi_nonDisp) os.system(cmd) - #Sig_phi_non_dis = np.sqrt((coef_non**2) * (fH**2) * Sig_phi_H**2 + (coef_non**2) * (fL**2) * Sig_phi_L**2) - - #outFileNonDis = os.path.join(inps.outDir, 'Sig_nonDis.bil') - #write(Sig_phi_non_dis, outFileNonDis, 1, 6) - #write_xml(outFileNonDis, length, width) - + return None #Sig_phi_iono, Sig_phi_nonDisp -def lowPassFilter(dataFile, sigDataFile, maskFile, Sx, Sy, sig_x, sig_y, iteration=5, theta=0.0): +def lowPassFilter(self,dataFile, sigDataFile, maskFile, Sx, Sy, sig_x, sig_y, iteration=5, theta=0.0): ds = gdal.Open(dataFile + '.vrt', gdal.GA_ReadOnly) length = ds.RasterYSize width = ds.RasterXSize @@ -160,7 +132,7 @@ def lowPassFilter(dataFile, sigDataFile, maskFile, Sx, Sy, sig_x, sig_y, iterati sigData = np.memmap(sigDataFile, dtype=np.float32, mode='r', shape=(length,width)) mask = np.memmap(maskFile, dtype=np.byte, mode='r', shape=(length,width)) - dataF, sig_dataF = iterativeFilter(dataIn[:,:], mask[:,:], sigData[:,:], iteration, Sx, Sy, sig_x, sig_y, theta) + dataF, sig_dataF = iterativeFilter(self,dataIn[:,:], mask[:,:], sigData[:,:], iteration, Sx, Sy, sig_x, sig_y, theta) filtDataFile = dataFile + ".filt" sigFiltDataFile = sigDataFile + ".filt" @@ -193,7 +165,7 @@ def write_xml(fileName,width,length,bands,dataType,scheme): return None -def iterativeFilter(dataIn, mask, Sig_dataIn, iteration, Sx, Sy, sig_x, sig_y, theta=0.0): +def iterativeFilter(self,dataIn, mask, Sig_dataIn, iteration, Sx, Sy, sig_x, sig_y, theta=0.0): data = np.zeros(dataIn.shape) data[:,:] = dataIn[:,:] Sig_data = np.zeros(dataIn.shape) @@ -202,17 +174,30 @@ def iterativeFilter(dataIn, mask, Sig_dataIn, iteration, Sx, Sy, sig_x, sig_y, t print ('masking the data') data[mask==0]=np.nan Sig_data[mask==0]=np.nan - print ('Filling the holes with nearest neighbor interpolation') - dataF = fill(data) - Sig_data = fill(Sig_data) + + if self.dispersive_filling_method == "smoothed": + print('Filling the holes with smoothed values') + dataF = fill_with_smoothed(data,3) + Sig_data = fill_with_smoothed(Sig_data,3) + else: + print ('Filling the holes with nearest neighbor interpolation') + dataF = fill(data) + Sig_data = fill(Sig_data) + print ('Low pass Gaussian filtering the interpolated data') dataF, Sig_dataF = Filter(dataF, Sig_data, Sx, Sy, sig_x, sig_y, theta=0.0) for i in range(iteration): print ('iteration: ', i , ' of ',iteration) print ('masking the interpolated and filtered data') dataF[mask==0]=np.nan - print('Filling the holes with nearest neighbor interpolation of the filtered data from previous step') - dataF = fill(dataF) + + if self.dispersive_filling_method == "smoothed": + print("Fill the holes with smoothed values") + dataF = fill_with_smoothed(dataF,3) + else: + print('Filling the holes with nearest neighbor interpolation of the filtered data from previous step') + dataF = fill(dataF) + print('Replace the valid pixels with original unfiltered data') dataF[mask==1]=data[mask==1] dataF, Sig_dataF = Filter(dataF, Sig_data, Sx, Sy, sig_x, sig_y, theta=0.0) @@ -228,11 +213,6 @@ def Filter(data, Sig_data, Sx, Sy, sig_x, sig_y, theta=0.0): W1 = cv2.filter2D(1.0/Sig_data**2,-1,kernel) W2 = cv2.filter2D(1.0/Sig_data**2,-1,kernel**2) - #data = ndimage.convolve(data,kernel, mode='nearest') - #W1 = ndimage.convolve(1.0/Sig_data**2,kernel, mode='nearest') - #W2 = ndimage.convolve(1.0/Sig_data**2,kernel**2, mode='nearest') - - return data/W1, np.sqrt(W2/(W1**2)) def Gaussian_kernel(Sx, Sy, sig_x,sig_y): @@ -282,6 +262,29 @@ def rotate(k , theta): k = a*k return k +def fill_with_smoothed(off,filterSize): + + off_2filt=np.copy(off) + kernel = np.ones((filterSize,filterSize),np.float32)/(filterSize*filterSize) + loop = 0 + cnt2=1 + + while (cnt2!=0 & loop<100): + loop += 1 + idx2= np.isnan(off_2filt) + cnt2 = np.sum(np.count_nonzero(np.isnan(off_2filt))) + print(cnt2) + if cnt2 != 0: + off_filt= convolve(off_2filt,kernel,boundary='extend',nan_treatment='interpolate') + off_2filt[idx2]=off_filt[idx2] + idx3 = np.where(off_filt == 0) + off_2filt[idx3]=np.nan + off_filt=None + + return off_2filt + + + def fill(data, invalid=None): """ Replace the value of invalid 'data' cells (indicated by 'invalid') @@ -304,7 +307,7 @@ def fill(data, invalid=None): return data[tuple(ind)] -def getMask(self, maskFile): +def getMask(self, maskFile,std_iono): ifgDirname = os.path.join(self.insar.ifgDirname, self.insar.lowBandSlcDirname) lowBandIgram = os.path.join(ifgDirname , 'filt_' + self.insar.ifgFilename ) @@ -328,7 +331,7 @@ def getMask(self, maskFile): else: highBandIgram += '.unw' - if self.dispersive_filter_mask_type == "coherence": + if (self.dispersive_filter_mask_type == "coherence") and (not self.dispersive_filter_mask_type == "median_filter"): print ('generating a mask based on coherence files of sub-band interferograms with a threshold of {0}'.format(self.dispersive_filter_coherence_threshold)) cmd = 'imageMath.py -e="(a>{0})*(b>{0})" --a={1} --b={2} -t byte -s BIL -o {3}'.format(self.dispersive_filter_coherence_threshold, lowBandCor, highBandCor, maskFile) os.system(cmd) @@ -338,28 +341,31 @@ def getMask(self, maskFile): print ('generating a mask based on .conncomp files') cmd = 'imageMath.py -e="(a>0)*(b>0)" --a={0} --b={1} -t byte -s BIL -o {2}'.format(lowBandIgram + '.conncomp', highBandIgram + '.conncomp', maskFile) os.system(cmd) - #m = read(lowBandIgram + '.conncomp') - #m = m[0,:,:] - #m = thresholdConnectedComponents(m,minPixelConnComp) - #mask = np.ones_like((m)) - #mask[m==0] = 0.0 - #m = read(highBandIgram + '.conncomp') - #m = m[0,:,:] - #m = thresholdConnectedComponents(m,minPixelConnComp) - #mask[m==0] = 0.0 + elif self.dispersive_filter_mask_type == "median_filter": + print('Generating mask based on median filtering of the raw dispersive component') + + # Open raw dispersive component (non-filtered, no unwrapping-error corrected) + dispFilename = os.path.join(self.insar.ionosphereDirname,self.insar.dispersiveFilename) + sigFilename = os.path.join(self.insar.ionosphereDirname,self.insar.dispersiveFilename+'.sig') + + ds = gdal.Open(dispFilename+'.vrt',gdal.GA_ReadOnly) + disp = ds.GetRasterBand(1).ReadAsArray() + ds=None - #outName = os.path.join(inps.outDir, 'mask0.bil') - #length, width = mask.shape - #write(mask, outName, 1, 6) - #write_xml(outName, length, width) + mask = (np.abs(disp-median_filter(disp,15))<3*std_iono) + + mask = mask.astype(np.float32) + mask.tofile(maskFile) + dims=np.shape(mask) + write_xml(maskFile,dims[1],dims[0],1,"FLOAT","BIL") else: print ('generating a mask based on unwrapped files. Pixels with phase = 0 are masked out.') cmd = 'imageMath.py -e="(a_1!=0)*(b_1!=0)" --a={0} --b={1} -t byte -s BIL -o {2}'.format(lowBandIgram , highBandIgram , maskFile) os.system(cmd) -def unwrapp_error_correction(f0, B, dispFile, nonDispFile,lowBandIgram, highBandIgram, jumpsFile, y_ref=None, x_ref=None): +def unwrapp_error_correction(f0, B, dispFile, nonDispFile,lowBandIgram, highBandIgram, y_ref=None, x_ref=None): dFile = os.path.join(os.path.dirname(dispFile) , "dJumps.bil") mFile = os.path.join(os.path.dirname(dispFile) , "mJumps.bil") @@ -372,28 +378,14 @@ def unwrapp_error_correction(f0, B, dispFile, nonDispFile,lowBandIgram, highBand refL = 0.0 refH = 0.0 - #cmd = 'imageMath.py -e="round(((a_1-{7}) - (b_1-{8}) - (2.0*{0}/3.0/{1})*c + (2.0*{0}/3.0/{1})*f )/2.0/PI)" --a={2} --b={3} --c={4} --f={5} -o {6} -t float32 -s BIL'.format(B, f0, highBandIgram, lowBandIgram, nonDispFile, dispFile, dFile, refH, refL) - - cmd = 'imageMath.py -e="round(((a_1+(2.0*PI*g)) - (b_1) - (2.0*{0}/3.0/{1})*c + (2.0*{0}/3.0/{1})*f )/2.0/PI)" --a={2} --b={3} --c={4} --f={5} --g={6} -o {7} -t float32 -s BIL'.format(B, f0, highBandIgram, lowBandIgram, nonDispFile, dispFile, jumpsFile, dFile) - + cmd = 'imageMath.py -e="round(((a_1+(2.0*PI)) - (b_1) - (2.0*{0}/3.0/{1})*c + (2.0*{0}/3.0/{1})*f )/2.0/PI)" --a={2} --b={3} --c={4} --f={5} -o {6} -t float32 -s BIL'.format(B, f0, highBandIgram, lowBandIgram, nonDispFile, dispFile, dFile) print(cmd) - os.system(cmd) - #d = (phH - phL - (2.*B/3./f0)*ph_nondis + (2.*B/3./f0)*ph_iono )/2./PI - #d = np.round(d) - - #cmd = 'imageMath.py -e="round(((a_1 - {6}) + (b_1-{7}) - 2.0*c - 2.0*f )/4.0/PI - g/2)" --a={0} --b={1} --c={2} --f={3} --g={4} -o {5} -t float32 -s BIL'.format(lowBandIgram, highBandIgram, nonDispFile, dispFile, dFile, mFile, refL, refH) - - cmd = 'imageMath.py -e="round(((a_1 ) + (b_1+(2.0*PI*k)) - 2.0*c - 2.0*f )/4.0/PI - g/2)" --a={0} --b={1} --c={2} --f={3} --g={4} --k={5} -o {6} -t float32 -s BIL'.format(lowBandIgram, highBandIgram, nonDispFile, dispFile, dFile, jumpsFile, mFile) - + + cmd = 'imageMath.py -e="round(((a_1 ) + (b_1+(2.0*PI)) - 2.0*c - 2.0*f )/4.0/PI - g/2)" --a={0} --b={1} --c={2} --f={3} --g={4} -o {5} -t float32 -s BIL'.format(lowBandIgram, highBandIgram, nonDispFile, dispFile, dFile, mFile) print(cmd) - os.system(cmd) - - #m = (phL + phH - 2*ph_nondis - 2*ph_iono)/4./PI - d/2. - #m = np.round(m) - return mFile , dFile @@ -450,33 +442,33 @@ def runDispersive(self): pulseLength = masterFrame.instrument.pulseLength chirpSlope = masterFrame.instrument.chirpSlope + # Total Bandwidth B = np.abs(chirpSlope)*pulseLength - + + ###Determine looks azLooks, rgLooks = self.insar.numberOfLooks( masterFrame, self.posting, self.numberAzimuthLooks, self.numberRangeLooks) - - ######################################################### - # make sure the low-band and high-band interferograms have consistent unwrapping errors. - # For this we estimate jumps as the difference of lowBand and highBand phases divided by 2PI - # The assumprion is that bothe interferograms are flattened and the phase difference between them - # is less than 2PI. This assumprion is valid for current sensors. It needs to be evaluated for - # future sensors like NISAR. - jumpsFile = check_consistency(lowBandIgram, highBandIgram, outputDir) - - ######################################################### # estimating the dispersive and non-dispersive components - dispersive_nonDispersive(lowBandIgram, highBandIgram, f0, fL, fH, outDispersive, outNonDispersive, jumpsFile) + dispersive_nonDispersive(lowBandIgram, highBandIgram, f0, fL, fH, outDispersive, outNonDispersive) + # If median filter is selected, compute the ionosphere phase standard deviation at a mean coherence value defined by the user + if self.dispersive_filter_mask_type == "median_filter": + coh_thres = self.dispersive_filter_coherence_threshold + std_iono = std_iono_mean_coh(f0,fL,fH,coh_thres,rgLooks,azLooks) + else: + std_iono = None + # generating a mask which will help filtering the estimated dispersive and non-dispersive phase - getMask(self, maskFile) + getMask(self, maskFile,std_iono) + # Calculating the theoretical standard deviation of the estimation based on the coherence of the interferograms theoretical_variance_fromSubBands(self, f0, fL, fH, B, sigmaDispersive, sigmaNonDispersive, azLooks * rgLooks) - + # low pass filtering the dispersive phase - lowPassFilter(outDispersive, sigmaDispersive, maskFile, + lowPassFilter(self,outDispersive, sigmaDispersive, maskFile, self.kernel_x_size, self.kernel_y_size, self.kernel_sigma_x, self.kernel_sigma_y, iteration = self.dispersive_filter_iterations, @@ -484,7 +476,7 @@ def runDispersive(self): # low pass filtering the non-dispersive phase - lowPassFilter(outNonDispersive, sigmaNonDispersive, maskFile, + lowPassFilter(self,outNonDispersive, sigmaNonDispersive, maskFile, self.kernel_x_size, self.kernel_y_size, self.kernel_sigma_x, self.kernel_sigma_y, iteration = self.dispersive_filter_iterations, @@ -493,21 +485,21 @@ def runDispersive(self): # Estimating phase unwrapping errors mFile , dFile = unwrapp_error_correction(f0, B, outDispersive+".filt", outNonDispersive+".filt", - lowBandIgram, highBandIgram, jumpsFile) + lowBandIgram, highBandIgram) # re-estimate the dispersive and non-dispersive phase components by taking into account the unwrapping errors outDispersive = outDispersive + ".unwCor" outNonDispersive = outNonDispersive + ".unwCor" - dispersive_nonDispersive(lowBandIgram, highBandIgram, f0, fL, fH, outDispersive, outNonDispersive, jumpsFile, m=mFile , d=dFile) + dispersive_nonDispersive(lowBandIgram, highBandIgram, f0, fL, fH, outDispersive, outNonDispersive, m=mFile , d=dFile) # low pass filtering the new estimations - lowPassFilter(outDispersive, sigmaDispersive, maskFile, + lowPassFilter(self,outDispersive, sigmaDispersive, maskFile, self.kernel_x_size, self.kernel_y_size, self.kernel_sigma_x, self.kernel_sigma_y, iteration = self.dispersive_filter_iterations, theta = self.kernel_rotation) - lowPassFilter(outNonDispersive, sigmaNonDispersive, maskFile, + lowPassFilter(self,outNonDispersive, sigmaNonDispersive, maskFile, self.kernel_x_size, self.kernel_y_size, self.kernel_sigma_x, self.kernel_sigma_y, iteration = self.dispersive_filter_iterations, From 7c7e43c98c137738a659181bce716b5cd164dd4a Mon Sep 17 00:00:00 2001 From: vbrancat Date: Mon, 10 Feb 2020 17:53:38 -0800 Subject: [PATCH 10/24] bug fixes --- components/isceobj/StripmapProc/runDispersive.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/components/isceobj/StripmapProc/runDispersive.py b/components/isceobj/StripmapProc/runDispersive.py index 9e94148..9232d4f 100644 --- a/components/isceobj/StripmapProc/runDispersive.py +++ b/components/isceobj/StripmapProc/runDispersive.py @@ -8,13 +8,10 @@ import isceobj from isceobj.Constants import SPEED_OF_LIGHT import numpy as np import gdal -<<<<<<< HEAD from scipy.ndimage import median_filter from astropy.convolution import convolve -from scipy import ndimage import numpy as np -======= ->>>>>>> upstream/master + try: import cv2 From 9f9f7be0d701bd48e79a9e7d0c5edc9cb2b739ae Mon Sep 17 00:00:00 2001 From: vbrancat Date: Thu, 13 Feb 2020 17:27:07 -0800 Subject: [PATCH 11/24] Inserted opencv, astropy ans scipy imports within functions --- .../isceobj/StripmapProc/runDispersive.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/components/isceobj/StripmapProc/runDispersive.py b/components/isceobj/StripmapProc/runDispersive.py index 9232d4f..103ef1b 100644 --- a/components/isceobj/StripmapProc/runDispersive.py +++ b/components/isceobj/StripmapProc/runDispersive.py @@ -3,21 +3,12 @@ # # import logging -import os +import os,gdal import isceobj from isceobj.Constants import SPEED_OF_LIGHT import numpy as np -import gdal -from scipy.ndimage import median_filter -from astropy.convolution import convolve -import numpy as np -try: - import cv2 -except ImportError: - print('OpenCV2 does not appear to be installed / is not importable.') - print('OpenCV2 is needed for this step. You may experience failures ...') logger = logging.getLogger('isce.insar.runDispersive') @@ -205,6 +196,9 @@ def iterativeFilter(self,dataIn, mask, Sig_dataIn, iteration, Sx, Sy, sig_x, sig return dataF, Sig_dataF def Filter(data, Sig_data, Sx, Sy, sig_x, sig_y, theta=0.0): + + import cv2 + kernel = Gaussian_kernel(Sx, Sy, sig_x, sig_y) #(800, 800, 15.0, 100.0) kernel = rotate(kernel , theta) @@ -264,6 +258,8 @@ def rotate(k , theta): def fill_with_smoothed(off,filterSize): + from astropy.convolution import convolve + off_2filt=np.copy(off) kernel = np.ones((filterSize,filterSize),np.float32)/(filterSize*filterSize) loop = 0 @@ -286,6 +282,8 @@ def fill_with_smoothed(off,filterSize): def fill(data, invalid=None): + + from scipy import ndimage """ Replace the value of invalid 'data' cells (indicated by 'invalid') by the value of the nearest valid data cell @@ -299,8 +297,6 @@ def fill(data, invalid=None): Output: Return a filled array. """ - from scipy import ndimage - if invalid is None: invalid = np.isnan(data) ind = ndimage.distance_transform_edt(invalid, @@ -310,7 +306,9 @@ def fill(data, invalid=None): def getMask(self, maskFile,std_iono): - + + from scipy.ndimage import median_filter + ifgDirname = os.path.join(self.insar.ifgDirname, self.insar.lowBandSlcDirname) lowBandIgram = os.path.join(ifgDirname , 'filt_' + self.insar.ifgFilename ) lowBandCor = os.path.join(ifgDirname ,self.insar.coherenceFilename) From e95311da68d8feb0029f883dcde67e08d6756c3d Mon Sep 17 00:00:00 2001 From: Yujie Zheng Date: Fri, 28 Feb 2020 13:49:13 -0800 Subject: [PATCH 12/24] Added computation of non-filtered correlation --- contrib/stack/topsStack/FilterAndCoherence.py | 36 +++++++++++++++++-- contrib/stack/topsStack/Stack.py | 11 +++++- contrib/stack/topsStack/stackSentinel.py | 21 ++++++----- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/contrib/stack/topsStack/FilterAndCoherence.py b/contrib/stack/topsStack/FilterAndCoherence.py index 4e833b2..ed629b6 100755 --- a/contrib/stack/topsStack/FilterAndCoherence.py +++ b/contrib/stack/topsStack/FilterAndCoherence.py @@ -30,10 +30,12 @@ import logging -import isce -import isceobj import argparse import os + +import isce +import isceobj +from isceobj.TopsProc.runBurstIfg import computeCoherence logger = logging.getLogger('isce.tops.runFilter') def runFilter(infile, outfile, filterStrength): @@ -135,7 +137,11 @@ def createParser(): dest='cohfile') parser.add_argument('-s', '--strength', type=float, default=0.5, help='Filter strength', dest='filterstrength') - + parser.add_argument('--slc1', type=str, help="SLC 1", dest='slc1') + parser.add_argument('--slc2', type=str, help="SLC 2", dest='slc2') + parser.add_argument('--cc','--complex_coh',type=str, default='fine.cori.full',help='complex coherence file',dest='cpx_cohfile') + parser.add_argument('-r','--range_looks',type=int, default=9, help= 'range looks', dest='numberRangelooks') + parser.add_argument('-z','--azimuth_looks',type=int, default=3, help= 'azimuth looks', dest='numberAzlooks') return parser def cmdLineParse(iargs=None): @@ -152,7 +158,31 @@ def main(iargs=None): runFilter(inps.infile, inps.filtfile, inps.filterstrength) estCoherence(inps.filtfile, inps.cohfile) + if inps.slc1 and inps.slc2: + computeCoherence(inps.slc1,inps.slc2,inps.cpx_cohfile) + from mroipac.looks.Looks import Looks + print('Multilooking {0} ...'.format(inps.cpx_cohfile)) + + infile=inps.cpx_cohfile + inimg = isceobj.createImage() + inimg.load(infile + '.xml') + + alks=inps.numberAzlooks + rlks=inps.numberRangelooks + + spl = os.path.splitext(inimg.filename) + #ext = '.{0}alks_{1}rlks'.format(alks, rlks) + #outname = spl[0] + ext + spl[1] + outname=spl[0] + lkObj = Looks() + lkObj.setDownLooks(alks) + lkObj.setAcrossLooks(rlks) + lkObj.setInputImage(inimg) + lkObj.setOutputFilename(outname) + lkObj.looks() + fullfilename=inps.cpx_cohfile + ret=os.system('rm '+fullfilename) if __name__ == '__main__': main() diff --git a/contrib/stack/topsStack/Stack.py b/contrib/stack/topsStack/Stack.py index 84965e5..a9e5238 100644 --- a/contrib/stack/topsStack/Stack.py +++ b/contrib/stack/topsStack/Stack.py @@ -205,7 +205,12 @@ class config(object): self.f.write('filt : ' + self.filtName + '\n') self.f.write('coh : ' + self.cohName + '\n') self.f.write('strength : ' + self.filtStrength + '\n') - + self.f.write('slc1 : ' + self.slc1 + '\n') + self.f.write('slc2 : ' + self.slc2 + '\n') + self.f.write('complex_coh : '+ self.cpxcor + '\n') + self.f.write('range_looks : ' + self.rangeLooks + '\n') + self.f.write('azimuth_looks : ' + self.azimuthLooks + '\n') + def unwrap(self, function): self.f.write('###################################'+'\n') self.f.write(function + '\n') @@ -629,12 +634,16 @@ class run(object): master = pair[0] slave = pair[1] mergedDir = os.path.join(self.work_dir, 'merged/interferograms/' + master + '_' + slave) + mergedSLCDir = os.path.join(self.work_dir, 'merged/SLC') configName = os.path.join(self.config_path ,'config_igram_filt_coh_' + master + '_' + slave) configObj = config(configName) configObj.configure(self) configObj.input = os.path.join(mergedDir,'fine.int') configObj.filtName = os.path.join(mergedDir,'filt_fine.int') configObj.cohName = os.path.join(mergedDir,'filt_fine.cor') + configObj.slc1=os.path.join(mergedSLCDir, '{}/{}.slc.full'.format(master, master)) + configObj.slc2=os.path.join(mergedSLCDir, '{}/{}.slc.full'.format(slave, slave)) + configObj.cpxcor=os.path.join(mergedDir,'fine.cor.full') #configObj.filtStrength = str(self.filtStrength) configObj.FilterAndCoherence('[Function-1]') configObj.finalize() diff --git a/contrib/stack/topsStack/stackSentinel.py b/contrib/stack/topsStack/stackSentinel.py index 05a9ce5..8e24db9 100755 --- a/contrib/stack/topsStack/stackSentinel.py +++ b/contrib/stack/topsStack/stackSentinel.py @@ -505,12 +505,14 @@ def slcStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, upd def correlationStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, pairs, updateStack): ############################# - i = slcStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, updateStack) + i = slcStack(inps, acquisitionDates,stackMasterDate, slaveDates, safe_dict, updateStack) + i+=1 runObj = run() - runObj.configure(inps, 'run_' + str(i) + "_merge_master") - runObj.mergeMaster(stackMasterDate, virtual = 'False') + runObj.configure(inps, 'run_' + str(i) + "_merge_master_slave_slc") + runObj.mergeMaster(stackMasterDate, virtual = 'True') + runObj.mergeSlaveSLC(slaveDates, virtual = 'True') runObj.finalize() i+=1 @@ -530,6 +532,13 @@ def interferogramStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe i = slcStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, updateStack) + i+=1 + runObj = run() + runObj.configure(inps, 'run_' + str(i) + "_merge_master_slave_slc") + runObj.mergeMaster(stackMasterDate, virtual = 'True') + runObj.mergeSlaveSLC(slaveDates, virtual = 'True') + runObj.finalize() + i+=1 runObj = run() runObj.configure(inps, 'run_' + str(i) + "_merge_burst_igram") @@ -548,12 +557,6 @@ def interferogramStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe runObj.unwrap(pairs) runObj.finalize() - i+=1 - runObj = run() - runObj.configure(inps, 'run_' + str(i) + "_merge_master_slave_slc") - runObj.mergeMaster(stackMasterDate, virtual = 'True') - runObj.mergeSlaveSLC(slaveDates, virtual = 'True') - runObj.finalize() def offsetStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, pairs, updateStack): From 58b032943d9361d54833701953b59b8a2fb93c40 Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Tue, 17 Dec 2019 09:36:50 -0800 Subject: [PATCH 13/24] Use command_argument_count in mdx_main I was having trouble compiling using macports gfortran-mp-9 due to being unable to link to symbol iargc. I think [1] implies this is a gfortran 77 extension, so maybe it isn't available in the mac library? It looks like there is a replacement [2] which is standard fortran as of '03, and using this function fixes this bug for me. [1] https://gcc.gnu.org/onlinedocs/gfortran/IARGC.html [2] https://gcc.gnu.org/onlinedocs/gfortran/COMMAND_005fARGUMENT_005fCOUNT.html --- contrib/mdx/src/SConscript | 2 +- contrib/mdx/src/mdx_main.F | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/contrib/mdx/src/SConscript b/contrib/mdx/src/SConscript index 9f0bd88..51ea02b 100644 --- a/contrib/mdx/src/SConscript +++ b/contrib/mdx/src/SConscript @@ -35,7 +35,7 @@ import os Import('envmdx') envmdx.Append( CCFLAGS=['-DSUN','-DIO64','-I'+envmdx['MOTIFINCPATH'],'-I'+envmdx['X11INCPATH']] ) envmdx.PrependUnique( LIBPATH=[envmdx['MOTIFLIBPATH'],envmdx['X11LIBPATH']] ) -envmdx.Append( FORTRANFLAGS=['-DSUN','-DIO64','-DGFORTRAN'] ) +envmdx.Append( FORTRANFLAGS=['-DSUN','-DIO64'] ) listFiles = ['graphx_mdx.c','rdf_reader_subs.f','mdx_subs.F'] build = envmdx['PRJ_LIB_DIR'] diff --git a/contrib/mdx/src/mdx_main.F b/contrib/mdx/src/mdx_main.F index 042b73c..ce77460 100644 --- a/contrib/mdx/src/mdx_main.F +++ b/contrib/mdx/src/mdx_main.F @@ -74,12 +74,6 @@ c***************************************************************** integer i_arg integer i_inarg - integer iargc -#ifdef GFORTRAN -c external iargc -#else - external iargc -#endif integer rdflen external rdflen @@ -110,7 +104,7 @@ c external iargc equivalence(b_data,r_data) a_cmd = '-V' - i_inarg = iargc() + i_inarg = command_argument_count() if (i_inarg .eq. 0) then write(6,*) ' ' write(6,'(1x,a,a18,a)' ) ' << mdx Version ',version_mdx(), ' >> ' From 9bee1a326b06e6501eb135078fe9d314af108918 Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Fri, 20 Dec 2019 19:45:16 -0800 Subject: [PATCH 14/24] Fix complex cast for gcc9 Fixes #57 --- components/mroipac/looks/bindings/looksmodule.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mroipac/looks/bindings/looksmodule.cpp b/components/mroipac/looks/bindings/looksmodule.cpp index 27d40f1..122e52b 100644 --- a/components/mroipac/looks/bindings/looksmodule.cpp +++ b/components/mroipac/looks/bindings/looksmodule.cpp @@ -250,7 +250,7 @@ int takeLookscpx(DataAccessor *IAIn, DataAccessor* IAout, int ld, int la) for(int j = 0; j < nfull; j++) { - bdbl[j] += ain[j]; + bdbl[j] += complex(ain[j].real(), ain[j].imag()); } } From 655c46cc4add275879167b750a5e91f6d00f168e Mon Sep 17 00:00:00 2001 From: shitong01 Date: Fri, 13 Mar 2020 10:46:49 +0800 Subject: [PATCH 15/24] Remove overwrite assignment when redundant bursts encountered across scenes. --- components/isceobj/Sensor/TOPS/Sentinel1.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/components/isceobj/Sensor/TOPS/Sentinel1.py b/components/isceobj/Sensor/TOPS/Sentinel1.py index fc8b640..21fa447 100755 --- a/components/isceobj/Sensor/TOPS/Sentinel1.py +++ b/components/isceobj/Sensor/TOPS/Sentinel1.py @@ -493,15 +493,9 @@ class Sentinel1(Component): offset = np.int(np.rint((aslice.product.bursts[0].burstStartUTC - t0).total_seconds() / burstStartInterval.total_seconds())) for kk in range(aslice.product.numberOfBursts): - #####Overwrite previous copy if one exists + #####Skip appending if burst also exists from previous scene if (offset+kk) < len(self.product.bursts): - self.product.bursts[offset+kk] = aslice.product.bursts[kk] - - ####Keep track of tiff src files - if len(self.tiff): - self._tiffSrc[offset+kk] = aslice.tiff[0] - - self._elevationAngleVsTau[offset+kk] = aslice._elevationAngleVsTau[kk] + continue elif (offset+kk) == len(self.product.bursts): self.product.bursts.append(aslice.product.bursts[kk]) From 84946b6624a07dcd6cd5ba418b960deb64e57210 Mon Sep 17 00:00:00 2001 From: Sara Mirzaee Date: Sat, 14 Mar 2020 12:00:25 -0400 Subject: [PATCH 16/24] Separate master topo job from run_1_unpack --- contrib/stack/topsStack/stackSentinel.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/contrib/stack/topsStack/stackSentinel.py b/contrib/stack/topsStack/stackSentinel.py index 8e24db9..0103500 100755 --- a/contrib/stack/topsStack/stackSentinel.py +++ b/contrib/stack/topsStack/stackSentinel.py @@ -430,11 +430,16 @@ def slcStack(inps, acquisitionDates, stackMasterDate, slaveDates, safe_dict, upd ############################# i=0 + if not updateStack: + i += 1 + runObj = run() + runObj.configure(inps, 'run_' + str(i) + "_unpack_topo_master") + runObj.unpackStackMasterSLC(safe_dict) + runObj.finalize() + i+=1 runObj = run() - runObj.configure(inps, 'run_' + str(i) + "_unpack_slc_topo_master") - if not updateStack: - runObj.unpackStackMasterSLC(safe_dict) + runObj.configure(inps, 'run_' + str(i) + "_unpack_slave_slc") runObj.unpackSlavesSLC(stackMasterDate, slaveDates, safe_dict) runObj.finalize() From 1329a0b9612cfe7a525e9e6c31f0853b559760c3 Mon Sep 17 00:00:00 2001 From: Sara Mirzaee Date: Sat, 14 Mar 2020 13:55:13 -0400 Subject: [PATCH 17/24] parallel processing bursts --- contrib/stack/topsStack/topo.py | 91 +++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 37 deletions(-) mode change 100644 => 100755 contrib/stack/topsStack/topo.py diff --git a/contrib/stack/topsStack/topo.py b/contrib/stack/topsStack/topo.py old mode 100644 new mode 100755 index 7b54fe8..3b4148d --- a/contrib/stack/topsStack/topo.py +++ b/contrib/stack/topsStack/topo.py @@ -10,6 +10,8 @@ import sys import s1a_isce_utils as ut from isceobj.Planet.Planet import Planet from zerodop.topozero import createTopozero +import multiprocessing as mp + def createParser(): parser = argparse.ArgumentParser( description='Generates lat/lon/h and los for each pixel') @@ -30,6 +32,49 @@ def cmdLineParse(iargs=None): return parser.parse_args(args=iargs) +def call_topo(input): + + (dirname, demImage, master, ind) = input + + burst = master.bursts[ind] + latname = os.path.join(dirname, 'lat_%02d.rdr' % (ind + 1)) + lonname = os.path.join(dirname, 'lon_%02d.rdr' % (ind + 1)) + hgtname = os.path.join(dirname, 'hgt_%02d.rdr' % (ind + 1)) + losname = os.path.join(dirname, 'los_%02d.rdr' % (ind + 1)) + maskname = os.path.join(dirname, 'shadowMask_%02d.rdr' % (ind + 1)) + incname = os.path.join(dirname, 'incLocal_%02d.rdr' % (ind + 1)) + #####Run Topo + planet = Planet(pname='Earth') + topo = createTopozero() + topo.slantRangePixelSpacing = burst.rangePixelSize + topo.prf = 1.0 / burst.azimuthTimeInterval + topo.radarWavelength = burst.radarWavelength + topo.orbit = burst.orbit + topo.width = burst.numberOfSamples + topo.length = burst.numberOfLines + topo.wireInputPort(name='dem', object=demImage) + topo.wireInputPort(name='planet', object=planet) + topo.numberRangeLooks = 1 + topo.numberAzimuthLooks = 1 + topo.lookSide = -1 + topo.sensingStart = burst.sensingStart + topo.rangeFirstSample = burst.startingRange + topo.demInterpolationMethod = 'BIQUINTIC' + topo.latFilename = latname + topo.lonFilename = lonname + topo.heightFilename = hgtname + topo.losFilename = losname + topo.maskFilename = maskname + topo.incFilename = incname + topo.topo() + + bbox = [topo.minimumLatitude, topo.maximumLatitude, topo.minimumLongitude, topo.maximumLongitude] + + topo = None + + return bbox + + def main(iargs=None): inps = cmdLineParse(iargs) @@ -40,6 +85,8 @@ def main(iargs=None): demImage.load(inps.dem + '.xml') boxes = [] + inputs = [] + for swath in swathList: master = ut.loadProduct(os.path.join(inps.master , 'IW{0}.xml'.format(swath))) @@ -51,50 +98,20 @@ def main(iargs=None): else: os.makedirs(dirname) - - ###For each burst for ind in range(master.numberOfBursts): - burst = master.bursts[ind] + inputs.append((dirname, demImage, master, ind)) - latname = os.path.join(dirname, 'lat_%02d.rdr'%(ind+1)) - lonname = os.path.join(dirname, 'lon_%02d.rdr'%(ind+1)) - hgtname = os.path.join(dirname, 'hgt_%02d.rdr'%(ind+1)) - losname = os.path.join(dirname, 'los_%02d.rdr'%(ind+1)) - maskname = os.path.join(dirname, 'shadowMask_%02d.rdr'%(ind+1)) - incname = os.path.join(dirname, 'incLocal_%02d.rdr'%(ind+1)) - #####Run Topo - planet = Planet(pname='Earth') - topo = createTopozero() - topo.slantRangePixelSpacing = burst.rangePixelSize - topo.prf = 1.0/burst.azimuthTimeInterval - topo.radarWavelength = burst.radarWavelength - topo.orbit = burst.orbit - topo.width = burst.numberOfSamples - topo.length = burst.numberOfLines - topo.wireInputPort(name='dem', object=demImage) - topo.wireInputPort(name='planet', object=planet) - topo.numberRangeLooks = 1 - topo.numberAzimuthLooks = 1 - topo.lookSide = -1 - topo.sensingStart = burst.sensingStart - topo.rangeFirstSample = burst.startingRange - topo.demInterpolationMethod='BIQUINTIC' - topo.latFilename = latname - topo.lonFilename = lonname - topo.heightFilename = hgtname - topo.losFilename = losname - topo.maskFilename = maskname - topo.incFilename = incname - topo.topo() + pool = mp.Pool(mp.cpu_count()) + results = pool.map(call_topo, inputs) + pool.close() - bbox = [topo.minimumLatitude, topo.maximumLatitude, topo.minimumLongitude, topo.maximumLongitude] - boxes.append(bbox) - - topo = None + for bbox in results: + boxes.append(bbox) boxes = np.array(boxes) bbox = [np.min(boxes[:,0]), np.max(boxes[:,1]), np.min(boxes[:,2]), np.max(boxes[:,3])] print ('bbox : ',bbox) + if __name__ == '__main__': From ea24d2320b3fd8647e38194a7ab07463f6db62a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20A=C3=AFv=C3=A1zis?= Date: Wed, 18 Mar 2020 11:26:00 -0700 Subject: [PATCH 18/24] iscesys/DictUtils: made a copy of the configuration dict before iterating over it --- components/iscesys/DictUtils/DictUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/iscesys/DictUtils/DictUtils.py b/components/iscesys/DictUtils/DictUtils.py index 7391e24..a78dacf 100755 --- a/components/iscesys/DictUtils/DictUtils.py +++ b/components/iscesys/DictUtils/DictUtils.py @@ -81,7 +81,7 @@ class DictUtils: spare = [] # dict1 is the one to update - for k2,v2 in dict2.items(): + for k2,v2 in dict(dict2).items(): if DictUtils.keyIsIn(k2,dict1): if isinstance(v2,dict):#if is a dict keep going down the node DictUtils.updateDictionary(dict1[k2],v2,replace,spare) From 82b46df7414c202244feae15877006cff67d69ee Mon Sep 17 00:00:00 2001 From: Zhang Yunjun Date: Thu, 19 Mar 2020 00:20:56 -0700 Subject: [PATCH 19/24] dem.py: use abspath filename in xml after correction + dem.py: replace filename in xml file with abspath after geoid to ellipsoid correction + update README.md files in contrib/stack/*Stack, as fixImageXml.py is not needed anymore + fixImageXml: use native argparse mutually exclusive group for --full/--base options to replace checking with own code --- applications/dem.py | 8 ++++++- applications/fixImageXml.py | 30 ++++++++++++--------------- contrib/stack/stripmapStack/README.md | 1 - contrib/stack/topsStack/README.md | 1 - 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/applications/dem.py b/applications/dem.py index d538821..a4f5fc6 100755 --- a/applications/dem.py +++ b/applications/dem.py @@ -34,6 +34,8 @@ import sys import os import argparse from contrib.demUtils import createDemStitcher + + def main(): #if not argument provided force the --help flag if(len(sys.argv) == 1): @@ -132,8 +134,12 @@ def main(): print('Could not create a stitched DEM. Some tiles are missing') else: if(args.correct): - ds.correct() #ds.correct(args.output,args.source,width,min(lat[0],lat[1]),min(lon[0],lon[1])) + demImg = ds.correct() + # replace filename with full path including dir in which file is located + demImg.filename = os.path.abspath(os.path.join(args.dir, demImg.filename)) + demImg.setAccessMode('READ') + demImg.renderHdr() else: print('Error. The --bbox (or -b) option must be specified when --action stitch is used') raise ValueError diff --git a/applications/fixImageXml.py b/applications/fixImageXml.py index d858b63..afe859c 100755 --- a/applications/fixImageXml.py +++ b/applications/fixImageXml.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 + +import os +import argparse import isce import isceobj -import argparse -import os +from isceobj.Util.ImageUtil import ImageLib as IML + def cmdLineParse(): ''' @@ -13,19 +16,14 @@ def cmdLineParse(): parser = argparse.ArgumentParser(description='Fixes pathnames in ISCE image XML files. Can be used to do more things in the future.') parser.add_argument('-i', '--input', type=str, required=True, dest='infile', help = 'Input image for which the XML file needs to be fixed.') - parser.add_argument('-f', '--full', action='store_true', default=False, dest='full', + + fname = parser.add_mutually_exclusive_group(required=True) + fname.add_argument('-f', '--full', action='store_false', help = 'Replace filename with full path including dir in which file is located') - parser.add_argument('-b', '--base', action='store_true', default=False, dest='base', + fname.add_argument('-b', '--base', action='store_true', help = 'Replace filename with basename to use in current directory') inps = parser.parse_args() - - if (inps.full and inps.base): - raise Exception('User requested to use both full path and basename') - - if (not inps.full) and (not inps.base): - raise Exception('User did not request any change') - return inps @@ -33,8 +31,6 @@ if __name__ == '__main__': ''' Main driver. ''' - from imageMath import IML - inps = cmdLineParse() if inps.infile.endswith('.xml'): @@ -42,15 +38,15 @@ if __name__ == '__main__': dirname = os.path.dirname(inps.infile) - img, dataname, metaName = IML.loadImage(inps.infile) + img = IML.loadImage(inps.infile)[0] if inps.full: fname = os.path.abspath( os.path.join(dirname, os.path.basename(inps.infile))) - elif inps.base: - fname = os.path.basename( os.path.basename(inps.infile)) else: - raise Exception('Unknown state in {0}'.format(os.path.basename(__file__))) + fname = os.path.basename( os.path.basename(inps.infile)) img.filename = fname img.setAccessMode('READ') img.renderHdr() + + diff --git a/contrib/stack/stripmapStack/README.md b/contrib/stack/stripmapStack/README.md index 4bf9260..b857712 100644 --- a/contrib/stack/stripmapStack/README.md +++ b/contrib/stack/stripmapStack/README.md @@ -29,7 +29,6 @@ d) fix the path of the file in the xml file of the DEM by using fixImageXml.py. mkdir DEM; cd DEM dem.py -a stitch -b -37 -31 -72 -69 -r -s 1 -c rm demLat*.dem demLat*.dem.xml demLat*.dem.vrt -fixImageXml.py -f -i demLat*.dem.wgs84 cd .. ``` diff --git a/contrib/stack/topsStack/README.md b/contrib/stack/topsStack/README.md index 67b6ecc..f0ebfaa 100644 --- a/contrib/stack/topsStack/README.md +++ b/contrib/stack/topsStack/README.md @@ -57,7 +57,6 @@ Download of DEM (need to use wgs84 version) using the ISCE DEM download script. mkdir DEM; cd DEM dem.py -a stitch -b 18 20 -100 -97 -r -s 1 –c rm demLat*.dem demLat*.dem.xml demLat*.dem.vrt -fixImageXml.py -f -i demLat*.dem.wgs84 #Updating DEM’s wgs84 xml to include full path to the DEM cd .. ``` From 7644bf8c938988102c6549cf7e143f51831ccebc Mon Sep 17 00:00:00 2001 From: Zhang Yunjun Date: Mon, 23 Mar 2020 17:04:08 -0700 Subject: [PATCH 20/24] MaskAndFilter: more ploting options + plot the evolution of offset after each step to better check the changes by masking, filling and filtering operations. + plot both azimuth and range offset in one figure and move the plotting to the end after file writing + update file path in xml file so that script can work even if the files are moved. + convert SNR nan value to zero before used as the mask, to avoid warning message + add the following options to customize the plot - add -v and --v-snr option to change the display range for offset and SNR - add --figsize and --save option + suppress the DEBUG message from matplotlib + remove obsolete getShape() and resampleOffset() + update IML module import + adjust indentation --- contrib/stack/stripmapStack/MaskAndFilter.py | 281 ++++++++++--------- 1 file changed, 143 insertions(+), 138 deletions(-) diff --git a/contrib/stack/stripmapStack/MaskAndFilter.py b/contrib/stack/stripmapStack/MaskAndFilter.py index c8cd8d6..d60be21 100755 --- a/contrib/stack/stripmapStack/MaskAndFilter.py +++ b/contrib/stack/stripmapStack/MaskAndFilter.py @@ -4,52 +4,80 @@ # Copyright 2016 # -import numpy as np -import argparse import os -import isce -import isceobj +import argparse +import numpy as np +from scipy import ndimage +import matplotlib.pyplot as plt import gdal from gdalconst import GA_ReadOnly -from scipy import ndimage + +# suppress the DEBUG message +import logging +mpl_logger = logging.getLogger('matplotlib') +mpl_logger.setLevel(logging.WARNING) + +import isce +import isceobj +from isceobj.Util.ImageUtil import ImageLib as IML GDAL2NUMPY_DATATYPE = { - -1 : np.uint8, -2 : np.uint16, -3 : np.int16, -4 : np.uint32, -5 : np.int32, -6 : np.float32, -7 : np.float64, -10: np.complex64, -11: np.complex128, - + 1 : np.uint8, + 2 : np.uint16, + 3 : np.int16, + 4 : np.uint32, + 5 : np.int32, + 6 : np.float32, + 7 : np.float64, + 10: np.complex64, + 11: np.complex128, } + +EXAMPLE = '''example: + MaskAndFilter.py -d offset.bip -s offset_snr.bip + MaskAndFilter.py -d offset.bip -s offset_snr.bip --plot +''' + + def createParser(): ''' Command line parser. ''' - parser = argparse.ArgumentParser( description='filters the densOffset, oversamples it and adds back to the geometry offset') + parser = argparse.ArgumentParser(description='Mask and filter the densOffset', + formatter_class=argparse.RawTextHelpFormatter, + epilog=EXAMPLE) parser.add_argument('-d', '--dense_offset', dest='denseOffset', type=str, required=True, help='The dense offsets file obtained from cross correlation or any other approach') parser.add_argument('-s', '--snr', dest='snr', type=str, required=True, help='The SNR of the dense offsets obtained from cross correlation or any other approach') parser.add_argument('-n', '--filter_size', dest='filterSize', type=int, default=8, - help='The size of the median filter') + help='Size of the median filter (default: %(default)s).') parser.add_argument('-t', '--snr_threshold', dest='snrThreshold', type=float, default=5, - help='The snr threshold used to mask the offset') + help='Min SNR used in the offset (default: %(default)s).') + + # output parser.add_argument('-A', '--output_azimuth_offset', dest='outAzimuth', type=str, default='filtAzimuth.off', - help='The azimuth offsets after rubber sheeting') + help='File name of the azimuth offsets after rubber sheeting (default: %(default)s).') parser.add_argument('-R', '--output_range_offset', dest='outRange', type=str, default='filtRange.off', - help='The range offsets after rubber sheeting') + help='File name of the range offsets after rubber sheeting (default: %(default)s).') parser.add_argument('-o', '--output_directory', dest='outDir', type=str, default='./', - help='Output directory') - parser.add_argument('-p', '--plot', dest='plot', action='store_true', default=False, - help='plot the offsets before and after masking and filtering') + help='Output directory (default: %(default)s).') + + # plot + plot = parser.add_argument_group('plot') + plot.add_argument('-p', '--plot', dest='plot', action='store_true', default=False, + help='plot the offsets before and after masking and filtering') + plot.add_argument('-v', dest='vlim', nargs=2, type=float, default=(-0.05, 0.05), + help='display range for offset (default: %(default)s).') + plot.add_argument('--v-snr', dest='vlim_snr', nargs=2, type=float, default=(0, 100), + help='display range for offset SNR (default: %(default)s).') + plot.add_argument('--figsize', dest='figsize', nargs=2, type=float, default=(18, 5), + help='figure size in inch (default: %(default)s).') + plot.add_argument('--save', dest='fig_name', type=str, default=None, + help='save figure as file') return parser @@ -58,7 +86,7 @@ def cmdLineParse(iargs = None): return parser.parse_args(args=iargs) -def read(file, processor='ISCE' , bands=None , dataType=None): +def read(file, processor='ISCE', bands=None, dataType=None): ''' raeder based on GDAL. Args: @@ -73,10 +101,13 @@ def read(file, processor='ISCE' , bands=None , dataType=None): Returns: * data : A numpy array with dimensions : number_of_bands * length * width ''' - + # generate ENVI hdr file and fix the file path in xml + file = os.path.abspath(file) if processor == 'ISCE': - cmd = 'isce2gis.py envi -i ' + file - os.system(cmd) + img, dataname, metaname = IML.loadImage(file) + img.filename = file + img.setAccessMode('READ') + img.renderHdr() dataset = gdal.Open(file,GA_ReadOnly) @@ -97,23 +128,22 @@ def read(file, processor='ISCE' , bands=None , dataType=None): # Fill the array with the Raster bands idx=0 for i in bands: - band=dataset.GetRasterBand(i) - data[idx,:,:] = band.ReadAsArray() - idx+=1 + band=dataset.GetRasterBand(i) + data[idx,:,:] = band.ReadAsArray() + idx+=1 dataset = None return data def write(raster, fileName, nbands, bandType): - - ############ # Create the file driver = gdal.GetDriverByName( 'ENVI' ) dst_ds = driver.Create(fileName, raster.shape[1], raster.shape[0], nbands, bandType ) dst_ds.GetRasterBand(1).WriteArray( raster, 0 ,0 ) - dst_ds = None + return + def fill(data, invalid=None): """ @@ -132,40 +162,46 @@ def fill(data, invalid=None): if invalid is None: invalid = np.isnan(data) ind = ndimage.distance_transform_edt(invalid, - return_distances=False, - return_indices=True) + return_distances=False, + return_indices=True) return data[tuple(ind)] -def mask_filter(inps, band, outName, plot=False): - #masking and Filtering - Offset = read(inps.denseOffset, bands=band) - Offset = Offset[0,:,:] +def mask_filter(inps, band, outName): + """masking and Filtering""" + + # read offset + offset = read(inps.denseOffset, bands=band) + offset = offset[0,:,:] + + # read SNR snr = read(inps.snr, bands=[1]) snr = snr[0,:,:] + snr[np.isnan(snr)] = 0 - # Masking the dense offsets based on SNR - print ('masking the dense offsets with SNR threshold: ', inps.snrThreshold) - Offset[snr