From 01d95c4b60c0b2d7b87f31a89c5403c8af4be5df Mon Sep 17 00:00:00 2001 From: Gerald Manipon Date: Mon, 22 Jul 2019 08:44:35 -0700 Subject: [PATCH 1/5] add circleci job to build and deploy a release Deployed docker image should be able to be pulled via e.g. `docker pull isce2:v2.3.2`. Doing a `docker pull isce2:latest` still pulls the deployed docker image from the most recent of these three circleci jobs: PR merge, weekly build, or release. --- .circleci/config.yml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 984216f..0adf2fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -106,6 +106,32 @@ jobs: root: images paths: - "*" + build-release: + docker: + - image: docker:stable-git + steps: + - checkout + - setup_remote_docker + - run: + name: Install dependencies + command: | + apk add --no-cache \ + python-dev py-pip bash pigz build-base libffi-dev openssl-dev + pip install \ + docker-compose awscli + - run: + name: Build docker image + command: | + mkdir images + echo "export TAG=$CIRCLE_TAG" >> images/env.sh + source images/env.sh + docker build --rm --force-rm -t isce/isce2:$TAG -f docker/Dockerfile . + cd images + docker save isce/isce2:$TAG > isce2.tar + - persist_to_workspace: + root: images + paths: + - "*" build-periodically: docker: - image: docker:stable-git @@ -172,6 +198,22 @@ workflows: filters: branches: only: master + build-deploy-release: + jobs: + - build-release: + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ + - deploy: + requires: + - build + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ weekly: triggers: - schedule: From 702a78eea20a679a0fc6ac3abf8f7ceb75e18c08 Mon Sep 17 00:00:00 2001 From: Gerald Manipon Date: Mon, 22 Jul 2019 08:53:18 -0700 Subject: [PATCH 2/5] set the correct required job --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0adf2fa..a3aded6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,7 +208,7 @@ workflows: ignore: /.*/ - deploy: requires: - - build + - build-release filters: tags: only: /^v.*/ From b8f8199001469c894453304fec4ca12eb0c7bbdf Mon Sep 17 00:00:00 2001 From: shiroma Date: Mon, 29 Jul 2019 09:21:28 -0700 Subject: [PATCH 3/5] - fixing issues with reference UTC (when it includes microseconds); - fixing the cast from complex32 to complex64 (some versions require complex64 casting through a "sink" - with statement)' --- components/isceobj/Sensor/UAVSAR_HDF5_SLC.py | 34 +++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/components/isceobj/Sensor/UAVSAR_HDF5_SLC.py b/components/isceobj/Sensor/UAVSAR_HDF5_SLC.py index 6323f5b..d29488f 100755 --- a/components/isceobj/Sensor/UAVSAR_HDF5_SLC.py +++ b/components/isceobj/Sensor/UAVSAR_HDF5_SLC.py @@ -80,6 +80,16 @@ POLARIZATION = Component.Parameter( doc='polarization channel of the UAVSAR slc file to be processed' ) +class DummySink(object): + def write(self, data): + pass + + def __enter__(self): + return self + + def __exit__(*x): + pass + from .Sensor import Sensor class UAVSAR_HDF5_SLC(Sensor): """ @@ -188,7 +198,10 @@ class UAVSAR_HDF5_SLC(Sensor): referenceUTC = file['/science/LSAR/SLC/swaths/zeroDopplerTime'].attrs['units'].decode('utf-8') referenceUTC = referenceUTC.replace('seconds since ','') - referenceUTC = datetime.datetime.strptime(referenceUTC,'%Y-%m-%d %H:%M:%S') + format_str = '%Y-%m-%d %H:%M:%S' + if '.' in referenceUTC: + format_str += '.%f' + referenceUTC = datetime.datetime.strptime(referenceUTC, format_str) relStart = file['/science/LSAR/SLC/swaths/zeroDopplerTime'][0] relEnd = file['/science/LSAR/SLC/swaths/zeroDopplerTime'][-1] @@ -222,7 +235,10 @@ class UAVSAR_HDF5_SLC(Sensor): referenceUTC = file['/science/LSAR/SLC/swaths/zeroDopplerTime'].attrs['units'].decode('utf-8') referenceUTC = referenceUTC.replace('seconds since ','') - t0 = datetime.datetime.strptime(referenceUTC,'%Y-%m-%d %H:%M:%S') + format_str = '%Y-%m-%d %H:%M:%S' + if '.' in referenceUTC: + format_str += '.%f' + t0 = datetime.datetime.strptime(referenceUTC, format_str) t = file['/science/LSAR/SLC/metadata/orbit/time'] position = file['/science/LSAR/SLC/metadata/orbit/position'] velocity = file['/science/LSAR/SLC/metadata/orbit/velocity'] @@ -247,9 +263,17 @@ class UAVSAR_HDF5_SLC(Sensor): ds = fid['/science/LSAR/SLC/swaths/' + self.frequency + '/' + self.polarization] nLines = ds.shape[0] - with open(self.output, 'wb') as fout: - for ii in range(nLines): - ds[ii,:].astype(np.complex64).tofile(fout) + # if TypeError is raised (e.g. complex32), force casting to complex64 + try: + _ = ds.dtype + sink = DummySink() + except TypeError: + sink = ds.astype(np.complex64) + + with sink: + with open(self.output, 'wb') as fout: + for ii in range(nLines): + ds[ii, :].tofile(fout) fid.close() From 20eecc127d4f17f443684ce982151d76980d8e47 Mon Sep 17 00:00:00 2001 From: shiroma Date: Mon, 29 Jul 2019 14:46:51 -0700 Subject: [PATCH 4/5] Updates the sensor: UAVSAR_HDF5_SLC - fixes issues with reference UTC (when it includes microseconds); - updates cast from complex32 to complex64 (some versions require complex64 casting through a "sink" - with statement). --- components/isceobj/Sensor/UAVSAR_HDF5_SLC.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/components/isceobj/Sensor/UAVSAR_HDF5_SLC.py b/components/isceobj/Sensor/UAVSAR_HDF5_SLC.py index d29488f..ffac857 100755 --- a/components/isceobj/Sensor/UAVSAR_HDF5_SLC.py +++ b/components/isceobj/Sensor/UAVSAR_HDF5_SLC.py @@ -80,16 +80,6 @@ POLARIZATION = Component.Parameter( doc='polarization channel of the UAVSAR slc file to be processed' ) -class DummySink(object): - def write(self, data): - pass - - def __enter__(self): - return self - - def __exit__(*x): - pass - from .Sensor import Sensor class UAVSAR_HDF5_SLC(Sensor): """ @@ -263,14 +253,8 @@ class UAVSAR_HDF5_SLC(Sensor): ds = fid['/science/LSAR/SLC/swaths/' + self.frequency + '/' + self.polarization] nLines = ds.shape[0] - # if TypeError is raised (e.g. complex32), force casting to complex64 - try: - _ = ds.dtype - sink = DummySink() - except TypeError: - sink = ds.astype(np.complex64) - - with sink: + # force casting to complex64 + with ds.astype(np.complex64): with open(self.output, 'wb') as fout: for ii in range(nLines): ds[ii, :].tofile(fout) From c9a6ba9fe0c938e24d433386dc683f69b9223019 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Wed, 31 Jul 2019 10:08:41 -0800 Subject: [PATCH 5/5] Restore snaphu_mcf unwrapping option --- components/isceobj/TopsProc/Factories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/isceobj/TopsProc/Factories.py b/components/isceobj/TopsProc/Factories.py index 744bc91..2e1b22a 100644 --- a/components/isceobj/TopsProc/Factories.py +++ b/components/isceobj/TopsProc/Factories.py @@ -50,7 +50,7 @@ def createUnwrapper(other, do_unwrap = None, unwrapperName = None, elif unwrapperName.lower() == 'snaphu': from .runUnwrapSnaphu import runUnwrap elif unwrapperName.lower() == 'snaphu_mcf': - from .runUnwrapSnaphu import runUnwrap + from .runUnwrapSnaphu import runUnwrapMcf as runUnwrap elif unwrapperName.lower() == 'downsample_snaphu': from .run_downsample_unwrapper import runUnwrap elif unwrapperName.lower() == 'icu':