Merge branch 'master' of https://github.com/isce-framework/isce2 into UAVSAR

.circleci/config.yml

@@ -1,6 +1,6 @@
-version: 2
+version: 2.1
 jobs:
-  build:
+  test:
     docker:
         - image: hysds/pge-base:latest
           user: root
@@ -79,5 +79,115 @@ jobs:
               topsApp.py --help --steps
               stripmapApp.py --help --steps
               python3 -c "import isce"
-              
+  build:
+    docker:
+      - image: docker:stable-git
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Install dependencies
+          command: |
+            apk add --no-cache \
+              python-dev py-pip bash pigz build-base libffi-dev openssl-dev
+            pip install \
+              docker-compose awscli
+      - run:
+          name: Build docker image
+          command: |
+            mkdir images
+            SHA1=$(echo $CIRCLE_SHA1 | cut -c1-7)
+            echo "export TAG=$SHA1" >> images/env.sh
+            source images/env.sh
+            docker build --rm --force-rm -t isce/isce2:$TAG -f docker/Dockerfile .
+            cd images
+            docker save isce/isce2:$TAG > isce2.tar
+      - persist_to_workspace:
+          root: images
+          paths:
+            - "*"
+  build-periodically:
+    docker:
+      - image: docker:stable-git
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Install dependencies
+          command: |
+            apk add --no-cache \
+              python-dev py-pip bash pigz build-base libffi-dev openssl-dev
+            pip install \
+              docker-compose awscli
+      - run:
+          name: Build docker image
+          command: |
+            mkdir images
+            echo 'export TAG=$(date -u +%Y%m%d)' >> images/env.sh
+            source images/env.sh
+            docker build --rm --force-rm -t isce/isce2:$TAG -f docker/Dockerfile .
+            cd images
+            docker save isce/isce2:$TAG > isce2.tar
+      - persist_to_workspace:
+          root: images
+          paths:
+            - "*"
+  deploy:
+    docker:
+      - image: docker:stable-git
+    steps:
+      - setup_remote_docker
+      - run:
+          name: Install dependencies
+          command: |
+            apk add --no-cache \
+              curl file
+      - attach_workspace:
+          at: images
+      - run:
+          name: Deploy
+          command: |
+            cd images
+            source env.sh
+            docker load -i isce2.tar
+            docker tag isce/isce2:$TAG isce/isce2:latest
+            docker login -u $DOCKER_USER -p $DOCKER_PASS
+            docker push isce/isce2:$TAG
+            docker push isce/isce2:latest
 
+workflows:
+  version: 2
+  test:
+    jobs:
+      - test
+  build-deploy:
+    jobs:
+      - build:
+          filters:
+            branches:
+              only: master
+      - deploy:
+          requires:
+            - build
+          filters:
+            branches:
+              only: master
+  weekly:
+    triggers:
+      - schedule:
+          cron: "0 7 * * 0"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - build-periodically:
+          filters:
+            branches:
+              only: master
+      - deploy:
+          requires:
+            - build-periodically
+          filters:
+            branches:
+              only: master

README.md

@@ -1,5 +1,7 @@
 # ISCE2
 
+[![CircleCI](https://circleci.com/gh/isce-framework/isce2.svg?style=svg)](https://circleci.com/gh/isce-framework/isce2)
+
 This is the Interferometric synthetic aperture radar Scientific Computing
 Environment (ISCE).  Its initial development was funded by NASA's Earth Science
 Technology Office (ESTO) under the Advanced Information Systems Technology

contrib/stack/stripmapStack/prepRawALOS.py

@@ -195,14 +195,14 @@ def main(iargs=None):
     dateDirs = sorted(glob.glob(os.path.join(inps.inputDir,'2*')))
     if inps.outputDir is not None:
         f = open(run_unPack,'w')
-        for dataDir in dateDirs:
+        for dateDir in dateDirs:
-            AlosFiles = glob.glob(os.path.join(dataDir, 'ALP*'))
+            AlosFiles = glob.glob(os.path.join(dateDir, 'ALP*'))
             if len(AlosFiles)>0:
-                acquisitionDate = os.path.basename(dataDir)
+                acquisitionDate = os.path.basename(dateDir)
                 slcDir = os.path.join(inps.outputDir, acquisitionDate)
                 if not os.path.exists(slcDir):
                     os.makedirs(slcDir)     
-                cmd = 'unpackFrame_ALOS_raw.py -i ' + os.path.abspath(dataDir) + ' -o ' + slcDir      
+                cmd = 'unpackFrame_ALOS_raw.py -i ' + os.path.abspath(dateDir) + ' -o ' + slcDir      
                 IMG_files = glob.glob(os.path.join(AlosFiles[0],'IMG*'))
                 if inps.fbd2fbs:
                     #recommended for regular interferometry to use all FBS bandwidth

contrib/stack/stripmapStack/prepRawCSK.py

@@ -171,26 +171,26 @@ def main(iargs=None):
     dateDirs = glob.glob(os.path.join(inputDir,'2*'))
     if outputDir is not None:
         f = open(run_unPack,'w')
-        for dataDir in dateDirs:
+        for dateDir in dateDirs:
-            CSKFiles = glob.glob(os.path.join(dataDir, 'CSK*.h5'))
+            CSKFiles = glob.glob(os.path.join(dateDir, 'CSK*.h5'))
             if len(CSKFiles)>0:
-                acquisitionDate = os.path.basename(dataDir)
+                acquisitionDate = os.path.basename(dateDir)
                 slcDir = os.path.join(outputDir, acquisitionDate)
                 if not os.path.exists(slcDir):
                     os.makedirs(slcDir)     
-                cmd = 'unpackFrame_CSK_raw.py -i ' + os.path.abspath(dataDir) + ' -o ' + slcDir      
+                cmd = 'unpackFrame_CSK_raw.py -i ' + os.path.abspath(dateDir) + ' -o ' + slcDir      
                 print (cmd)
                 f.write(inps.text_cmd + cmd+'\n')
     
             """ 
             ##### FOR now lets ptu all scences in single folder
-            CSKFiles = glob.glob(os.path.join(dataDir, 'EL*'))
+            CSKFiles = glob.glob(os.path.join(dateDir, 'EL*'))
             if len(CSKFiles)>0:
-               acquisitionDate = os.path.basename(dataDir)
+               acquisitionDate = os.path.basename(dateDir)
                slcDir = os.path.join(outputDir, acquisitionDate)
                if not os.path.exists(slcDir):
                   os.makedirs(slcDir)     
-               cmd = 'unpackFrame_CSK_raw.py -i ' + os.path.abspath(dataDir) + ' -o ' + slcDir      
+               cmd = 'unpackFrame_CSK_raw.py -i ' + os.path.abspath(dateDir) + ' -o ' + slcDir      
 
                if len(CSKFiles) > 1:
                   cmd = cmd + ' -m' 

contrib/stack/stripmapStack/prepSlcALOS2.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+# Author: David Bekaert
+# Zhang Yunjun, adopted from prepRawALOS.py for ALOS2 SM SLC
+
+
+import os
+import glob
+import argparse
+import shutil
+import tarfile
+import zipfile
+from uncompressFile import uncompressfile
+
+
+def createParser():
+    '''
+    Create command line parser.
+    '''
+
+    parser = argparse.ArgumentParser(description='Prepare ALOS2 slc for processing (unzip/untar files, '
+                                     'organize in date folders, generate script to unpack into isce formats).')
+    parser.add_argument('-i', '--input', dest='inputDir', type=str, required=True,
+            help='directory with the downloaded SLC data')
+    parser.add_argument('-rmfile', '--rmfile', dest='rmfile',action='store_true', default=False,
+            help='Optional: remove zip/tar/compressed files after unpacking into date structure '
+                 '(default is to keep in archive fo  lder)')
+    parser.add_argument('-o', '--output', dest='outputDir', type=str, required=False,
+            help='output directory where data needs to be unpacked into isce format (for script generation).')
+    parser.add_argument('-t', '--text_cmd', dest='text_cmd', type=str, default='source ~/.bash_profile;',
+            help='text command to be added to the beginning of each line of the run files. Default: source ~/.bash_profile;')
+    return parser
+
+
+def cmdLineParse(iargs=None):
+    '''
+    Command line parser.
+    '''
+
+    parser = createParser()
+    inps = parser.parse_args(args = iargs)
+
+    # parsing required inputs
+    inps.inputDir = os.path.abspath(inps.inputDir)
+
+    # parsing optional inputs
+    if inps.outputDir:
+        inps.outputDir = os.path.abspath(inps.outputDir)
+    return inps
+
+
+def get_Date(ALOS_folder):
+    """Grab acquisition date"""
+    # will search for different version of workreport to be compatible with ASf, WInSAR etc
+    workreport_files = ('*workreport','summary.txt')
+    for workreport_file in workreport_files:
+        workreports = glob.glob(os.path.join(ALOS_folder,workreport_file))
+
+        # if nothing is found return a failure
+        if len(workreports) > 0:
+            for workreport in workreports:
+                template_dict = {}
+                with open(workreport) as openfile:
+                    for line in openfile:
+                        c = line.split("=")
+                        template_dict[c[0].strip()] = c[1].strip()
+                acquisitionDate = (str(template_dict['Img_SceneCenterDateTime'][1:9]))
+                if acquisitionDate:
+                    successflag = True
+                    return successflag, acquisitionDate
+
+    # if it reached here it could not find the acqusiitionDate
+    successflag = False                                                                                                                                                    
+    acquisitionDate = 'FAIL'
+    return successflag, acquisitionDate
+
+
+def get_ALOS2_name(infile):
+    """Get the ALOS2210402970 name from compress file in various format."""
+    outname = None
+    fbase = os.path.basename(infile)
+    if 'ALOS2' in fbase:
+        fbase = fbase.replace('_','-')
+        outname = [i for i in fbase.split('-') if 'ALOS2' in i][0]
+    else:
+        fext = os.path.splitext(infile)[1]
+        if fext in ['.tar', '.gz']:
+            with tarfile.open(infile, 'r') as tar:
+                file_list = tar.getnames()
+        elif fext in ['.zip']:
+            with zipfile.ZipFile(infile, 'r') as z:
+                file_list = z.namelist()
+        else:
+            raise ValueError('unrecognized file extension: {}'.format(fext))
+        led_file = [i for i in file_list if 'LED' in i][0]
+        led_file = os.path.basename(led_file)
+        outname = [i for i in led_file.split('-') if 'ALOS2' in i][0]
+    return outname
+
+
+def main(iargs=None):
+    '''
+    The main driver.
+    '''
+
+    inps = cmdLineParse(iargs)
+
+    # filename of the runfile
+    run_unPack = 'run_unPackALOS2'
+
+    # loop over the different folder of ALOS2 zip/tar files and unzip them, make the names consistent
+    file_exts = (os.path.join(inps.inputDir, '*.zip'),
+                 os.path.join(inps.inputDir, '*.tar'),
+                 os.path.join(inps.inputDir, '*.gz'))
+    for file_ext in file_exts:
+        # loop over zip/tar files
+        for fname in sorted(glob.glob(file_ext)):
+            ## the path to the folder/zip
+            workdir = os.path.dirname(fname)
+
+            ## get the output name folder without any extensions
+            dir_unzip = get_ALOS2_name(fname)
+            dir_unzip = os.path.join(workdir, dir_unzip)
+
+            # loop over two cases (either file or folder): 
+            # if this is a file, try to unzip/untar it
+            if os.path.isfile(fname):
+                # unzip the file in the outfolder
+                successflag_unzip = uncompressfile(fname, dir_unzip)
+
+                # put failed files in a seperate directory
+                if not successflag_unzip:
+                    dir_failed = os.path.join(workdir,'FAILED_FILES')
+                    if not os.path.isdir(dir_failed):
+                        os.makedirs(dir_failed)
+                    cmd = 'mv {} {}'.format(fname, dir_failed)
+                    os.system(cmd)
+                else:
+                    # check if file needs to be removed or put in archive folder
+                    if inps.rmfile:
+                        os.remove(fname)
+                        print('Deleting: ' + fname)
+                    else:
+                        dir_archive = os.path.join(workdir,'ARCHIVED_FILES')
+                        if not os.path.isdir(dir_archive):
+                            os.makedirs(dir_archive)
+                        cmd = 'mv {} {}'.format(fname, dir_archive)
+                        os.system(cmd)
+
+
+        # loop over the different ALOS folders and make sure the folder names are consistent.
+        # this step is not needed unless the user has manually unzipped data before.
+        ALOS_folders = glob.glob(os.path.join(inps.inputDir, 'ALOS2*'))
+        for ALOS_folder in ALOS_folders:
+            # in case the user has already unzipped some files
+            # make sure they are unzipped similar like the uncompressfile code
+            temp = os.path.basename(ALOS_folder)
+            parts = temp.split(".")
+            parts = parts[0].split('-')
+            ALOS_outfolder_temp = parts[0]
+            ALOS_outfolder_temp = os.path.join(os.path.dirname(ALOS_folder),ALOS_outfolder_temp)
+            # check if the folder (ALOS_folder) has a different filename as generated from uncompressFile (ALOS_outfolder_temp)
+            if not (ALOS_outfolder_temp == ALOS_folder):
+                # it is different, check if the ALOS_outfolder_temp already exists, if yes, delete the current folder
+                if os.path.isdir(ALOS_outfolder_temp):
+                    print('Remove ' + ALOS_folder + ' as ' + ALOS_outfolder_temp + ' exists...')
+                    # check if this folder already exist, if so overwrite it
+                    shutil.rmtree(ALOS_folder)
+
+
+    # loop over the different ALOS folders and organize in date folders
+    ALOS_folders = glob.glob(os.path.join(inps.inputDir, 'ALOS2*'))                        
+    for ALOS_folder in ALOS_folders:
+        # get the date
+        successflag, imgDate = get_Date(ALOS_folder)       
+
+        workdir = os.path.dirname(ALOS_folder)
+        if successflag:
+            # move the file into the date folder
+            SLC_dir = os.path.join(workdir,imgDate,'')
+            if not os.path.isdir(SLC_dir):
+                os.makedirs(SLC_dir)
+                
+            # check if the folder already exist in that case overwrite it
+            ALOS_folder_out = os.path.join(SLC_dir,os.path.basename(ALOS_folder))
+            if os.path.isdir(ALOS_folder_out):
+                shutil.rmtree(ALOS_folder_out)
+            # move the ALOS acqusition folder in the date folder
+            cmd  = 'mv ' + ALOS_folder + ' ' + SLC_dir + '.' 
+            os.system(cmd)
+
+            print ('Succes: ' + imgDate)
+        else:
+            print('Failed: ' + ALOS_folder)
+        
+
+    # now generate the unpacking script for all the date dirs
+    dateDirs = sorted(glob.glob(os.path.join(inps.inputDir,'2*')))
+    if inps.outputDir is not None:
+        f = open(run_unPack,'w')
+        for dateDir in dateDirs:
+            AlosFiles = glob.glob(os.path.join(dateDir, 'ALOS2*'))
+            # if there is at least one frame
+            if len(AlosFiles)>0:
+                acquisitionDate = os.path.basename(dateDir)
+                slcDir = os.path.join(inps.outputDir, acquisitionDate)
+                if not os.path.exists(slcDir):
+                    os.makedirs(slcDir)     
+                cmd = 'unpackFrame_ALOS2.py -i ' + os.path.abspath(dateDir) + ' -o ' + slcDir      
+                print (cmd)
+                f.write(inps.text_cmd + cmd+'\n')
+        f.close()
+    return
+
+
+if __name__ == '__main__':
+
+    main()

contrib/stack/stripmapStack/prepSlcRSAT2.py

@@ -161,14 +161,14 @@ def main(iargs=None):
     dateDirs = glob.glob(os.path.join(inputDir,'2*'))
     if outputDir is not None:
         f = open(run_unPack,'w')
-        for dataDir in dateDirs:
+        for dateDir in dateDirs:
-            RSAT2Files = glob.glob(os.path.join(dataDir, 'imagery_HH.tif'))
+            RSAT2Files = glob.glob(os.path.join(dateDir, 'imagery_HH.tif'))
             if len(RSAT2Files)>0:
-                acquisitionDate = os.path.basename(dataDir)
+                acquisitionDate = os.path.basename(dateDir)
                 slcDir = os.path.join(outputDir, acquisitionDate)
                 if not os.path.exists(slcDir):
                     os.makedirs(slcDir)     
-                cmd = 'unpackFrame_RSAT2.py -i ' + os.path.abspath(dataDir) + ' -o ' + slcDir      
+                cmd = 'unpackFrame_RSAT2.py -i ' + os.path.abspath(dateDir) + ' -o ' + slcDir      
                 print (cmd)
                 f.write(inps.text_cmd + cmd+'\n')
         f.close()

contrib/stack/stripmapStack/unpackFrame_ALOS2.py

@@ -14,24 +14,25 @@ def cmdLineParse():
     Command line parser.
     '''
 
-    parser = argparse.ArgumentParser(description='Unpack CSK SLC data and store metadata in pickle file.')
+    parser = argparse.ArgumentParser(description='Unpack ALOS2 SLC data and store metadata in pickle file.')
     parser.add_argument('-i','--input', dest='h5dir', type=str,
-            required=True, help='Input CSK directory')
+            required=True, help='Input ALOS2 directory')
     parser.add_argument('-o', '--output', dest='slcdir', type=str,
             required=True, help='Output SLC directory')
     parser.add_argument('-d', '--deskew', dest='deskew', action='store_true',
             default=False, help='To read in for deskewing data later')
-
+    parser.add_argument('-p', '--polarization', dest='polarization', type=str,
+            default='HH', help='polarization in case if quad or full pol data exists. Deafult: HH')
     return parser.parse_args()
 
 
-def unpack(hdf5, slcname, deskew=False):
+def unpack(hdf5, slcname, deskew=False, polarization='HH'):
     '''
     Unpack HDF5 to binary SLC file.
     '''
 
-    imgname = glob.glob(os.path.join(hdf5,'IMG*'))[0]
+    imgname = glob.glob(os.path.join(hdf5, '*/IMG-{}*'.format(polarization)))[0]
-    ldrname = glob.glob(os.path.join(hdf5, 'LED*'))[0]
+    ldrname = glob.glob(os.path.join(hdf5, '*/LED*'))[0]
     if not os.path.isdir(slcname):
         os.mkdir(slcname)
 
@@ -94,4 +95,7 @@ if __name__ == '__main__':
     if inps.h5dir.endswith('/'):
         inps.h5dir = inps.h5dir[:-1]
 
-    obj = unpack(inps.h5dir, inps.slcdir, deskew=inps.deskew)
+    obj = unpack(inps.h5dir, inps.slcdir,
+                 deskew=inps.deskew,
+                 polarization=inps.polarization)
+