KJ135
/
ISCE_INSAR
mirror of http://172.16.0.12:5000/WBJY/ISCE_INSAR.git
4
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #19 from yunjunz/PIXEL_ALOS 

stripmapStack: support ALOS filename that are not started with ALP
LT1AB
piyushrpt 2019-03-11 09:45:12 -07:00 committed by GitHub
parent bfbc555f41 85df2bce15
commit 7a4ba86f68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 83 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,6 +1,7 @@
*.pyc
*~
*.swp
*.DS_Store
__pycache__
.sconf_temp
.sconsign.dblite

94
contrib/stack/stripmapStack/prepRawALOS.py
View File

@ -1,25 +1,32 @@
#!/usr/bin/env python3
# David Bekaert
import os
import glob
import argparse
from uncompressFile import uncompressfile
import shutil
import tarfile
import zipfile
from uncompressFile import uncompressfile
def createParser():
'''
Create command line parser.
'''
parser = argparse.ArgumentParser(description='Prepare ALOS raw processing (unzip/untar files, organize in date folders, generate script to unpack into isce formats).')
parser.add_argument('-i', '--input', dest='input', type=str, required=True,
parser = argparse.ArgumentParser(description='Prepare ALOS raw processing (unzip/untar files, '
'organize in date folders, generate script to unpack into isce formats).')
parser.add_argument('-i', '--input', dest='inputDir', type=str, required=True,
help='directory with the raw data')
parser.add_argument('-rmfile', '--rmfile', dest='rmfile',action='store_true', default=False,
help='Optional: remove zip/tar/compressed files after unpacking into date structure (default is to keep in archive fo lder)')
parser.add_argument('-o', '--output', dest='output', type=str, required=False,
help='Optional: remove zip/tar/compressed files after unpacking into date structure '
'(default is to keep in archive fo lder)')
parser.add_argument('-o', '--output', dest='outputDir', type=str, required=False,
help='output directory where data needs to be unpacked into isce format (for script generation).')
parser.add_argument('-t', '--text_cmd', dest='text_cmd', type=str, default='source ~/.bash_profile;'
, help='text command to be added to the beginning of each line of the run files. Default: source ~/.bash_profile;')
parser.add_argument('-t', '--text_cmd', dest='text_cmd', type=str, default='source ~/.bash_profile;',
help='text command to be added to the beginning of each line of the run files. Default: source ~/.bash_profile;')
return parser
@ -29,7 +36,15 @@ def cmdLineParse(iargs=None):
'''
parser = createParser()
return parser.parse_args(args = iargs)
inps = parser.parse_args(args = iargs)
# parsing required inputs
inps.inputDir = os.path.abspath(inps.inputDir)
# parsing optional inputs
if inps.outputDir:
inps.outputDir = os.path.abspath(inps.outputDir)
return inps
def get_Date(ALOSfolder):
@ -56,38 +71,52 @@ def get_Date(ALOSfolder):
acquisitionDate = 'FAIL'
return successflag, acquisitionDate
def get_ALOS_ALP_name(infile):
"""Get the ALPSRP075780620 name from compress file in various format."""
outname = None
fbase = os.path.basename(infile)
if fbase.startswith("ALP"):
outname = fbase.split("-")[0]
else:
fext = os.path.splitext(infile)[1]
if fext in ['.tar', '.gz']:
with tarfile.open(infile, 'r') as tar:
file_list = tar.getnames()
elif fext in ['.zip']:
with zipfile.ZipFile(infile, 'r') as z:
file_list = z.namelist()
else:
raise ValueError('unrecognized file extension: {}'.format(fext))
led_file = [i for i in file_list if 'LED' in i][0]
led_file = os.path.basename(led_file)
outname = [i for i in led_file.split("-") if 'ALP' in i][0]
return outname
def main(iargs=None):
'''
The main driver.
'''
inps = cmdLineParse(iargs)
# parsing required inputs
inputDir = os.path.abspath(inps.input)
# parsing optional inputs
if inps.output:
outputDir = os.path.abspath(inps.output)
else:
outputDir = None
rmfile = inps.rmfile
# filename of the runfile
run_unPack = 'run_unPackALOS'
# loop over the different folder, ALOS zip/tar files and unzip them, make the names consistent
ALOS_extensions = (os.path.join(inputDir, 'ALP*.zip'),os.path.join(inputDir, 'ALP*.tar'),os.path.join(inputDir, 'ALP*.gz'))
ALOS_extensions = (os.path.join(inps.inputDir, '*.zip'),
os.path.join(inps.inputDir, '*.tar'),
os.path.join(inps.inputDir, '*.gz'))
for ALOS_extension in ALOS_extensions:
ALOS_filesfolders = glob.glob(ALOS_extension)
# loop over zip/tar files
ALOS_filesfolders = sorted(glob.glob(ALOS_extension))
for ALOS_infilefolder in ALOS_filesfolders:
## the path to the folder/zip
workdir = os.path.dirname(ALOS_infilefolder)
## get the output name folder without any extensions
temp = os.path.basename(ALOS_infilefolder)
# trim the extensions and keep only very first part
parts = temp.split(".")
parts = parts[0].split('-')
ALOS_outfolder = parts[0]
ALOS_outfolder = get_ALOS_ALP_name(ALOS_infilefolder)
# add the path back in
ALOS_outfolder = os.path.join(workdir, ALOS_outfolder)
@ -104,7 +133,7 @@ def main(iargs=None):
os.rename(ALOS_infilefolder,os.path.join(workdir,'FAILED_FILES','.'))
else:
# check if file needs to be removed or put in archive folder
if rmfile:
if inps.rmfile:
os.remove(ALOS_infilefolder)
print('Deleting: ' + ALOS_infilefolder)
else:
@ -116,9 +145,10 @@ def main(iargs=None):
# loop over the different ALOS folders and make sure the folder names are consistent.
# this step is not needed unless the user has manually unzipped data before.
ALOS_folders = glob.glob(os.path.join(inputDir, 'ALP*'))
ALOS_folders = glob.glob(os.path.join(inps.inputDir, 'ALP*'))
for ALOS_folder in ALOS_folders:
# in case the user has already unzipped some files, make sure they are unzipped similar like the uncompressfile code
# in case the user has already unzipped some files
# make sure they are unzipped similar like the uncompressfile code
temp = os.path.basename(ALOS_folder)
parts = temp.split(".")
parts = parts[0].split('-')
@ -134,7 +164,7 @@ def main(iargs=None):
# loop over the different ALOS folders and organize in date folders
ALOS_folders = glob.glob(os.path.join(inputDir, 'ALP*'))
ALOS_folders = glob.glob(os.path.join(inps.inputDir, 'ALP*'))
for ALOS_folder in ALOS_folders:
# get the date
successflag, imgDate = get_Date(ALOS_folder)
@ -160,14 +190,14 @@ def main(iargs=None):
# now generate the unpacking script for all the date dirs
dateDirs = glob.glob(os.path.join(inputDir,'2*'))
if outputDir is not None:
dateDirs = glob.glob(os.path.join(inps.inputDir,'2*'))
if inps.outputDir is not None:
f = open(run_unPack,'w')
for dataDir in dateDirs:
AlosFiles = glob.glob(os.path.join(dataDir, 'ALP*'))
if len(AlosFiles)>0:
acquisitionDate = os.path.basename(dataDir)
slcDir = os.path.join(outputDir, acquisitionDate)
slcDir = os.path.join(inps.outputDir, acquisitionDate)
if not os.path.exists(slcDir):
os.makedirs(slcDir)
cmd = 'unpackFrame_ALOS_raw.py -i ' + os.path.abspath(dataDir) + ' -o ' + slcDir
@ -179,9 +209,9 @@ def main(iargs=None):
print (cmd)
f.write(inps.text_cmd + cmd+'\n')
f.close()
return
if __name__ == '__main__':
main()

17
contrib/stack/stripmapStack/uncompressFile.py
View File

@ -2,12 +2,14 @@
# David Bekaert
import zipfile
import os
import glob
import argparse
import tarfile
import shutil
import tarfile
import zipfile
def createParser():
'''
@ -50,6 +52,8 @@ def main(iargs=None):
print('Done')
elif completeFlag == False:
print('Failed')
return
def uncompressfile(inputFile,outputDir):
@ -105,7 +109,7 @@ def uncompressfile(inputFile,outputDir):
# Check if the data is unpacked in its own folder
folderfiles = glob.glob(os.path.join(outputDir,'*'))
if len(folderfiles)==1:
while len(folderfiles)==1:
# get the sub-folder name only
tempdir = os.path.basename(folderfiles[0])
if os.path.isdir(folderfiles[0]):
@ -114,7 +118,9 @@ def uncompressfile(inputFile,outputDir):
os.rename(folderfiles[0],tempdir2)
os.rmdir(outputDir)
os.rename(tempdir2,outputDir)
folderfiles = glob.glob(os.path.join(outputDir,'*'))
return completeFlag
elif extension == '.tar' or extension == '.gz':
TAR = tarfile.open(inputFile)
@ -124,9 +130,9 @@ def uncompressfile(inputFile,outputDir):
TAR.close()
completeFlag = True
# Check if the data is unpacked in its own folder
# Check if the data is unpacked in its own folder or its sub-folders
folderfiles = glob.glob(os.path.join(outputDir,'*'))
if len(folderfiles)==1:
while len(folderfiles) == 1:
# get the sub-folder name only
tempdir = os.path.basename(folderfiles[0])
if os.path.isdir(folderfiles[0]):
@ -135,6 +141,7 @@ def uncompressfile(inputFile,outputDir):
os.rename(folderfiles[0],tempdir2)
os.rmdir(outputDir)
os.rename(tempdir2,outputDir)
folderfiles = glob.glob(os.path.join(outputDir,'*'))
return completeFlag
except:
print('Tar file seems to be corrupted, abord...')