stage ASF data virtually for TOPS processing (#319)
* stage ASF data virtually for TOPS processing * obsolete fetchCookies class removedLT1AB
ehavazli
GitHub
parent
10e03b6e01
commit
385ed6b7d8
|
|
@ -0,0 +1,288 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Authors: Piyush Agram, Emre Havazli
|
||||
# Copyright 2021
|
||||
|
||||
import os
|
||||
import netrc
|
||||
import base64
|
||||
import zipfile
|
||||
import logging
|
||||
import argparse
|
||||
from osgeo import gdal
|
||||
|
||||
from urllib.request import build_opener, install_opener, Request, urlopen
|
||||
from urllib.request import HTTPHandler, HTTPSHandler, HTTPCookieProcessor
|
||||
from urllib.error import HTTPError, URLError
|
||||
from http.cookiejar import MozillaCookieJar
|
||||
|
||||
|
||||
class SentinelVRT:
|
||||
"""
|
||||
Class for virtual download of S1 products.
|
||||
"""
|
||||
def __init__(self, url, dest):
|
||||
"""
|
||||
Constructor with URL.
|
||||
"""
|
||||
# URL
|
||||
self.url = url
|
||||
|
||||
# Destination folder
|
||||
self.dest = os.path.join(dest, os.path.basename(url))
|
||||
|
||||
# Product Type
|
||||
if "IW_GRD" in self.url:
|
||||
self.productType = "GRD"
|
||||
elif "IW_SLC" in self.url:
|
||||
self.productType = "SLC"
|
||||
else:
|
||||
raise Exception("Product type could not be determined for: "
|
||||
"{0}".format(self.url))
|
||||
|
||||
# Write dummy zip file to test output can be written
|
||||
if os.path.exists(self.dest):
|
||||
print("Destination zip file already exists. "
|
||||
"Will be overwritten ....")
|
||||
os.remove(self.dest)
|
||||
self.createZip()
|
||||
|
||||
# Fetch manifest
|
||||
self.IPF = None # TODO: Get calibration XML for IPF 2.36-low priority
|
||||
self.fetchManifest()
|
||||
|
||||
# Fetch annotation
|
||||
self.fetchAnnotation()
|
||||
|
||||
# Fetch images - TODO: GRD support
|
||||
if self.productType == "SLC":
|
||||
self.fetchSLCImagery()
|
||||
|
||||
def createZip(self):
|
||||
"""
|
||||
Create local zip file to populate.
|
||||
"""
|
||||
try:
|
||||
with zipfile.ZipFile(self.dest, mode='w') as myzip:
|
||||
with myzip.open('download.log', 'w') as myfile:
|
||||
myfile.write('Downloaded with ISCE2\n'.encode('utf-8'))
|
||||
except:
|
||||
raise Exception('Could not create zipfile: {0}'.format(self.dest))
|
||||
|
||||
def fetchManifest(self):
|
||||
"""
|
||||
Fetch manifest.safe
|
||||
"""
|
||||
try:
|
||||
res = gdal.ReadDir(self.srcsafe)
|
||||
if 'manifest.safe' not in res:
|
||||
raise Exception("Manifest file not found in "
|
||||
"{0}".format(self.srcsafe))
|
||||
except:
|
||||
raise Exception("Could not fetch manifest from "
|
||||
"{0}".format(self.srcsafe))
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(self.dest, mode='a') as myzip:
|
||||
with myzip.open(os.path.join(self.zip2safe,'manifest.safe'),
|
||||
'w') as myfile:
|
||||
logging.info('Fetching manifest.safe')
|
||||
self.downloadFile(os.path.join(self.srcsafe,
|
||||
'manifest.safe'), myfile)
|
||||
|
||||
except:
|
||||
raise Exception("Could not download manifest.safe from "
|
||||
"{0} to {1}".format(self.url, self.dest))
|
||||
|
||||
def fetchAnnotation(self):
|
||||
"""
|
||||
Fetch annotation files.
|
||||
"""
|
||||
dirname = os.path.join(self.srcsafe, 'annotation')
|
||||
res = gdal.ReadDir(dirname)
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(self.dest, mode='a') as myzip:
|
||||
for ii in res:
|
||||
if ii.endswith('.xml'):
|
||||
srcname = os.path.join(dirname, ii)
|
||||
destname = os.path.join(self.zip2safe,
|
||||
'annotation', ii)
|
||||
logging.info('Fetching {0}'.format(srcname))
|
||||
with myzip.open(destname, 'w') as myfile:
|
||||
self.downloadFile(srcname, myfile)
|
||||
except:
|
||||
raise Exception("Could not download {0} from {1} to "
|
||||
"{2}".format(ii, self.url, self.dest))
|
||||
|
||||
def fetchSLCImagery(self):
|
||||
"""
|
||||
Create VRTs for TIFF files.
|
||||
"""
|
||||
import isce
|
||||
from isceobj.Sensor.TOPS.Sentinel1 import Sentinel1
|
||||
|
||||
dirname = os.path.join(self.srcsafe, 'measurement')
|
||||
res = gdal.ReadDir(dirname)
|
||||
|
||||
# If more were known about the tiff, this can be improved
|
||||
vrt_template = """<VRTDataset rasterXSize="{samples}" rasterYSize="{lines}">
|
||||
<VRTRasterBand dataType="CInt16" band="1">
|
||||
<NoDataValue>0.0</NoDataValue>
|
||||
<SimpleSource>
|
||||
<SourceFilename relativeToVRT="0">{tiffname}</SourceFilename>
|
||||
<SourceBand>1</SourceBand>
|
||||
<SourceProperties RasterXSize="{samples}" RasterYSize="{lines}" DataType="CInt16" BlockXSize="{samples}" BlockYSize="1"/>
|
||||
<SrcRect xOff="0" yOff="0" xSize="{samples}" ySize="{lines}"/>
|
||||
<DstRect xOff="0" yOff="0" xSize="{samples}" ySize="{lines}"/>
|
||||
</SimpleSource>
|
||||
</VRTRasterBand>
|
||||
</VRTDataset>"""
|
||||
|
||||
# Parse annotation files to have it ready with information
|
||||
for ii in res:
|
||||
parts = ii.split('-')
|
||||
swath = int(parts[1][-1])
|
||||
pol = parts[3]
|
||||
|
||||
# Read and parse metadata for swath
|
||||
xmlname = ii.replace('.tiff', '.xml')
|
||||
|
||||
try:
|
||||
reader = Sentinel1()
|
||||
reader.configure()
|
||||
reader.xml = [os.path.join("/vsizip", self.dest,
|
||||
self.zip2safe, 'annotation',
|
||||
xmlname)]
|
||||
reader.manifest = [os.path.join("/vsizip", self.dest,
|
||||
self.zip2safe,
|
||||
'manifest.safe')]
|
||||
reader.swathNumber = swath
|
||||
reader.polarization = pol
|
||||
reader.parse()
|
||||
|
||||
vrtstr = vrt_template.format(
|
||||
samples=reader.product.bursts[0].numberOfSamples,
|
||||
lines=(reader.product.bursts[0].numberOfLines *
|
||||
len(reader.product.bursts)),
|
||||
tiffname=os.path.join(self.srcsafe, 'measurement', ii))
|
||||
|
||||
#Write the VRT to zip file
|
||||
with zipfile.ZipFile(self.dest, mode='a') as myzip:
|
||||
destname = os.path.join(self.zip2safe, 'measurement',
|
||||
ii)
|
||||
with myzip.open(destname, 'w') as myfile:
|
||||
myfile.write(vrtstr.encode('utf-8'))
|
||||
except:
|
||||
raise Exception("Could not create vrt for {0} at {1} in "
|
||||
"{2}".format(ii, self.url, self.dest))
|
||||
|
||||
@property
|
||||
def vsi(self):
|
||||
return os.path.join('/vsizip/vsicurl', self.url)
|
||||
|
||||
@property
|
||||
def srcsafe(self):
|
||||
return os.path.join(self.vsi, self.zip2safe)
|
||||
|
||||
@property
|
||||
def zip2safe(self):
|
||||
"""
|
||||
Get safe directory path from zip name.
|
||||
"""
|
||||
return os.path.basename(self.url).replace('.zip', '.SAFE')
|
||||
|
||||
@staticmethod
|
||||
def downloadFile(inname, destid):
|
||||
|
||||
# Get file size
|
||||
stats = gdal.VSIStatL(inname)
|
||||
if stats is None:
|
||||
raise Exception('Could not get stats for {0}'.format(inname))
|
||||
|
||||
# Copy file to local folder
|
||||
success = False
|
||||
while not success:
|
||||
try:
|
||||
vfid = gdal.VSIFOpenL(inname, 'rb')
|
||||
data = gdal.VSIFReadL(1, stats.size, vfid)
|
||||
gdal.VSIFCloseL(vfid)
|
||||
success = True
|
||||
except AttributeError as errmsg:
|
||||
if errmsg.endswith('307'):
|
||||
print('Redirected on {0}. Retrying ... '.format(inname))
|
||||
except Exception as err:
|
||||
print(err)
|
||||
raise Exception('Could not download file: {0}'.format(inname))
|
||||
|
||||
# Write to destination id
|
||||
destid.write(data)
|
||||
|
||||
|
||||
def cmdLineParse():
|
||||
"""
|
||||
Command line parser.
|
||||
"""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Download S1 annotation files with VRT pointing to '
|
||||
'tiff files')
|
||||
parser.add_argument('-i', '--input', dest='inlist', type=str,
|
||||
required=True, help='Text file with URLs to fetch')
|
||||
parser.add_argument('-o', '--output', dest='outdir', type=str,
|
||||
default='.', help='Output folder to store the data in')
|
||||
parser.add_argument('-c', '--cookies', dest='cookies', type=str,
|
||||
default='asfcookies.txt', help='Path to cookies file')
|
||||
parser.add_argument('-d', '--debug', dest='debug', action='store_true',
|
||||
default=False, help='Set to CPL_DEBUG to ON')
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main(inps=None):
|
||||
"""
|
||||
Main driver.
|
||||
"""
|
||||
|
||||
# check if output directory exists
|
||||
if os.path.isdir(inps.outdir):
|
||||
print('Output directory {0} exists'.format(inps.outdir))
|
||||
else:
|
||||
print('Creating output directory {0}'.format(inps.outdir))
|
||||
os.mkdir(inps.outdir)
|
||||
|
||||
# Setup GDAL with cookies
|
||||
gdal.UseExceptions()
|
||||
|
||||
gdal.SetConfigOption('GDAL_HTTP_COOKIEFILE', inps.cookies)
|
||||
gdal.SetConfigOption('GDAL_HTTP_COOKIEJAR', inps.cookies)
|
||||
gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'TRUE')
|
||||
if inps.debug:
|
||||
gdal.SetConfigOption('CPL_DEBUG', 'ON')
|
||||
gdal.SetConfigOption('CPL_CURL_VERBOSE', 'YES')
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
else:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
# Read in URLs into a list
|
||||
urlList = []
|
||||
try:
|
||||
with open(inps.inlist, 'r') as fid:
|
||||
for cnt, line in enumerate(fid):
|
||||
urlList.append(line.strip())
|
||||
|
||||
except:
|
||||
raise Exception('Could not parse input file "{0}" as a list of line '
|
||||
'separated URLs'.format(inps.inlist))
|
||||
|
||||
for url in urlList:
|
||||
logging.info('Downloading: {0}'.format(url))
|
||||
downloader = SentinelVRT(url, inps.outdir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Parse command line
|
||||
inps = cmdLineParse()
|
||||
|
||||
# Process
|
||||
main(inps)
|
||||
Loading…
Reference in New Issue