KJ135
/
ISCE_INSAR
mirror of http://172.16.0.12:5000/WBJY/ISCE_INSAR.git
4
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #254 from mirzaees/aux_url 

Fix url change for downloading orbits
LT1AB
Ryan Burns 2021-03-09 12:16:43 -08:00 committed by GitHub
parent 8ffb2f281a 1a86bca5c4
commit 9eed870041
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 99 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
contrib/stack/topsStack/README.md
View File

@ -37,9 +37,7 @@ The following calibration auxliary (AUX_CAL) file is used for **antenna pattern
Run the command below to download the AUX_CAL file once and store it somewhere (_i.e._ ~/aux/aux_cal) so that you can use it all the time, for `stackSentinel.py -a` or `auxiliary data directory` in `topsApp.py`.
```
wget https://qc.sentinel1.eo.esa.int/product/S1A/AUX_CAL/20140908T000000/S1A_AUX_CAL_V20140908T000000_G20190626T100201.SAFE.TGZ
tar zxvf S1A_AUX_CAL_V20140908T000000_G20190626T100201.SAFE.TGZ
rm S1A_AUX_CAL_V20140908T000000_G20190626T100201.SAFE.TGZ
wget https://aux.sentinel1.eo.esa.int/AUX_CAL/2014/09/08/S1A_AUX_CAL_V20140908T000000_G20190626T100201.SAFE/ --no-check-certificate --recursive --level=1 --cut-dirs=4 -nH
```
#### 1. Create your project folder somewhere ####

122
contrib/stack/topsStack/dloadOrbits.py
View File

@ -8,35 +8,33 @@ import requests
from html.parser import HTMLParser
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
fmt = '%Y%m%d'
today = datetime.datetime.now().strftime(fmt)
today = datetime.datetime.now().strftime(fmt)
server = 'https://qc.sentinel1.eo.esa.int/'
server = 'https://aux.sentinel1.eo.esa.int/'
queryfmt = '%Y-%m-%d'
datefmt = '%Y%m%dT%H%M%S'
S1Astart = '20140901'
S1Astart_dt = datetime.datetime.strptime(S1Astart, '%Y%m%d')
S1Bstart = '20160501'
S1Bstart_dt = datetime.datetime.strptime(S1Bstart, '%Y%m%d')
def cmdLineParse():
'''
Automated download of orbits.
'''
parser = argparse.ArgumentParser('S1A orbit downloader')
parser.add_argument('--start','-b', dest='start', type=str, default=S1Astart, help='Start date')
parser.add_argument('--end','-e', dest='end',
type=str, default=today, help='Stop date')
parser.add_argument('--dir', '-d', dest='dirname',
type=str, default='.', help='Directory with precise orbits')
return parser.parse_args()
'''
Automated download of orbits.
'''
parser = argparse.ArgumentParser('S1A orbit downloader')
parser.add_argument('--start', '-b', dest='start', type=str, default=S1Astart, help='Start date')
parser.add_argument('--end', '-e', dest='end', type=str, default=today, help='Stop date')
parser.add_argument('--dir', '-d', dest='dirname', type=str, default='.', help='Directory with precise orbits')
return parser.parse_args()
def fileToRange(fname):
'''
@ -50,13 +48,14 @@ def fileToRange(fname):
return (start, stop, mission)
def gatherExistingOrbits(dirname):
'''
Gather existing orbits.
'''
fnames = glob.glob(os.path.join(dirname, 'S1?_OPER_AUX_POEORB*'))
rangeList=[]
rangeList = []
for name in fnames:
rangeList.append(fileToRange(name))
@ -84,10 +83,11 @@ def ifAlreadyExists(indate, mission, rangeList):
return found
def validS1BDate(indate):
if indate < S1Bstart_dt:
return False
else:
else:
return True
@ -110,7 +110,7 @@ def download_file(url, outdir='.', session=None):
success = False
if success:
with open(path,'wb') as f:
with open(path, 'wb') as f:
for chunk in request.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
@ -121,13 +121,13 @@ def download_file(url, outdir='.', session=None):
class MyHTMLParser(HTMLParser):
def __init__(self):
def __init__(self, url):
HTMLParser.__init__(self)
self.fileList = []
self.in_td = False
self.in_a = False
self.in_table = False
self._url = None
self._url = url
def handle_starttag(self, tag, attrs):
if tag == 'td':
@ -135,14 +135,16 @@ class MyHTMLParser(HTMLParser):
elif tag == 'a':
self.in_a = True
for name, val in attrs:
if name== "href":
if name == "href":
if val.startswith("http"):
self._url = val.strip()
def handle_data(self,data):
def handle_data(self, data):
if self.in_td and self.in_a:
if ('S1A_OPER' in data) or ('S1B_OPER' in data):
# print(data.strip())
self.fileList.append((self._url, data.strip()))
print(self._url, data.strip())
def handle_tag(self, tag):
if tag == 'td':
@ -150,51 +152,7 @@ class MyHTMLParser(HTMLParser):
self.in_a = False
elif tag == 'a':
self.in_a = False
self._url=None
def query(indate, mission, session):
'''
Query the system for a given date.
'''
if mission == 'S1B':
if not validS1BDate(indate):
return
delta = datetime.timedelta(days=2)
timebef = (indate - delta).strftime(queryfmt)
timeaft = (indate + delta).strftime(queryfmt)
url = server + 'aux_poeorb'
query = url + '/?validity_start={0}..{1}&sentinel1__mission={2}'.format(timebef, timeaft,mission)
success = False
match = None
try:
r = session.get(query, verify=False)
r.raise_for_status()
parser = MyHTMLParser()
parser.feed(r.text)
for resulturl, result in parser.fileList:
tbef, taft, mission = fileToRange(os.path.basename(result))
if (tbef <= indate) and (taft >= indate):
#match = os.path.join(url, result)
match = resulturl
break
if match is not None:
success = True
except:
pass
if match is None:
print('Failed to find {1} orbits for tref {0}'.format(indate, mission))
return
return match
self._url = None
if __name__ == '__main__':
@ -202,7 +160,7 @@ if __name__ == '__main__':
Main driver.
'''
#Parse command line
# Parse command line
inps = cmdLineParse()
###Compute interval
@ -212,20 +170,40 @@ if __name__ == '__main__':
days = (tend - tstart).days
print('Number of days to check: ', days)
####Gather existing orbits
ranges = gatherExistingOrbits(inps.dirname)
session = requests.session()
for dd in range(days):
indate = tstart + datetime.timedelta(days=dd, hours=12)
print('Searching for {0}'.format(indate))
url = server + 'POEORB/' + str(indate.year).zfill(2) + '/' + str(indate.month).zfill(2) + '/' + str(
indate.day).zfill(2) + '/'
session = requests.session()
match = None
for mission in ['S1A', 'S1B']:
if not ifAlreadyExists(indate, mission, ranges):
match = query(indate, mission, session)
try:
r = session.get(url, verify=False)
r.raise_for_status()
parser = MyHTMLParser(url)
parser.feed(r.text)
for resulturl, result in parser.fileList:
match = os.path.join(resulturl, result)
if match is not None:
success = True
except:
pass
if match is not None:
download_file(match, inps.dirname, session)
pass
else:
print('Failed to find {1} orbits for tref {0}'.format(indate, mission))
else:
print('Already exists: ', mission, indate)
print('Exit dloadOrbits Successfully')

147
contrib/stack/topsStack/fetchOrbit.py
View File

@ -8,15 +8,15 @@ import argparse
import datetime
from html.parser import HTMLParser
server = 'https://qc.sentinel1.eo.esa.int/'
server2 = 'http://aux.sentinel1.eo.esa.int/'
server = 'http://aux.sentinel1.eo.esa.int/'
orbitMap = [('precise','aux_poeorb'),
('restituted','aux_resorb')]
orbitMap = [('precise', 'POEORB/'),
('restituted', 'RESORB/')]
datefmt = "%Y%m%dT%H%M%S"
queryfmt = "%Y-%m-%d"
queryfmt2= "%Y/%m/%d/"
queryfmt2 = "%Y/%m/%d/"
def cmdLineParse():
'''
@ -25,9 +25,9 @@ def cmdLineParse():
parser = argparse.ArgumentParser(description='Fetch orbits corresponding to given SAFE package')
parser.add_argument('-i', '--input', dest='input', type=str, required=True,
help='Path to SAFE package of interest')
help='Path to SAFE package of interest')
parser.add_argument('-o', '--output', dest='outdir', type=str, default='.',
help='Path to output directory')
help='Path to output directory')
return parser.parse_args()
@ -43,9 +43,9 @@ def FileToTimeStamp(safename):
try:
tstamp = datetime.datetime.strptime(fields[-4], datefmt)
sstamp = datetime.datetime.strptime(fields[-5], datefmt)
except:
p = re.compile(r'(?<=_)\d{8}')
dt2 = p.search(safename).group()
except:
p = re.compile(r'(?<=_)\d{8}')
dt2 = p.search(safename).group()
tstamp = datetime.datetime.strptime(dt2, '%Y%m%d')
satName = fields[0]
@ -55,47 +55,37 @@ def FileToTimeStamp(safename):
class MyHTMLParser(HTMLParser):
def __init__(self, satName):
def __init__(self, satName, url):
HTMLParser.__init__(self)
self.fileList = []
self.pages = 0
self.in_td = False
self.in_a = False
self.in_ul = False
self.in_table = False
self._url = url
self.satName = satName
def handle_starttag(self, tag, attrs):
if tag == 'td':
self.in_td = True
elif tag == 'a' and self.in_td:
elif tag == 'a':
self.in_a = True
elif tag == 'ul':
for k,v in attrs:
if k == 'class' and v.startswith('pagination'):
self.in_ul = True
elif tag == 'li' and self.in_ul:
self.pages += 1
for name, val in attrs:
if name == "href":
if val.startswith("http"):
self._url = val.strip()
def handle_data(self,data):
def handle_data(self, data):
if self.in_td and self.in_a:
#if 'S1A_OPER' in data:
#if 'S1B_OPER' in data:
if satName in data:
self.fileList.append(data.strip())
if self.satName in data:
self.fileList.append((self._url, data.strip()))
def handle_endtag(self, tag):
def handle_tag(self, tag):
if tag == 'td':
self.in_td = False
self.in_a = False
elif tag == 'a' and self.in_td:
elif tag == 'a':
self.in_a = False
elif tag == 'ul' and self.in_ul:
self.in_ul = False
elif tag == 'html':
if self.pages == 0:
self.pages = 1
else:
# decrement page back and page forward list items
self.pages -= 2
self._url = None
def download_file(url, outdir='.', session=None):
@ -117,7 +107,7 @@ def download_file(url, outdir='.', session=None):
success = False
if success:
with open(path,'wb') as f:
with open(path, 'wb') as f:
for chunk in request.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
@ -149,82 +139,35 @@ if __name__ == '__main__':
fileTS, satName, fileTSStart = FileToTimeStamp(inps.input)
print('Reference time: ', fileTS)
print('Satellite name: ', satName)
match = None
session = requests.Session()
for spec in orbitMap:
oType = spec[0]
if oType == 'precise':
delta =datetime.timedelta(days=2)
elif oType == 'restituted':
delta = datetime.timedelta(days=1)
url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \
'/' + str(fileTS.day).zfill(2) + '/'
timebef = (fileTS - delta).strftime(queryfmt)
timeaft = (fileTS + delta).strftime(queryfmt)
url = server + spec[1]
query = (url +
'/?validity_start={0}..{1}&sentinel1__mission={2}'
.format(timebef, timeaft,satName))
print(query)
success = False
match = None
try:
print('Querying for {0} orbits'.format(oType))
r = session.get(query, verify=False)
r = session.get(url, verify=False)
r.raise_for_status()
parser = MyHTMLParser(satName)
parser = MyHTMLParser(satName, url)
parser.feed(r.text)
print("Found {} pages".format(parser.pages))
# get results from first page
results = parser.fileList
# page through and get more results
for page in range(2, parser.pages + 1):
page_parser = MyHTMLParser(satName)
page_query = "{}&page={}".format(query, page)
print(page_query)
r = session.get(page_query, verify=False)
r.raise_for_status()
page_parser.feed(r.text)
results.extend(page_parser.fileList)
# run through all results and pull the orbit files
if results:
for result in results:
tbef, taft, mission = fileToRange(os.path.basename(result))
if (tbef <= fileTSStart) and (taft >= fileTS):
datestr2 = FileToTimeStamp(result)[0].strftime(queryfmt2)
match = (server2 + spec[1].replace('aux_', '').upper() +
'/' +datestr2+ result + '.EOF')
break
if match is not None:
success = True
except Exception as e:
print('Exception - something went wrong with the web scraper:')
print('Exception: {}'.format(e))
print('Continuing process')
for resulturl, result in parser.fileList:
match = os.path.join(resulturl, result)
if match is not None:
success = True
except:
pass
if match is None:
print('Failed to find {0} orbits for Time {1}'.format(oType, fileTS))
if success:
break
if match:
res = download_file(match, inps.outdir, session=session)
if res is False:
print('Failed to download URL: ', match)
session.close()
if match is not None:
res = download_file(match, inps.outdir, session)
if res is False:
print('Failed to download URL: ', match)
else:
print('Failed to find {1} orbits for tref {0}'.format(fileTS, satName))

6
examples/input_files/reference_TOPS_SENTINEL1.xml
View File

@ -24,7 +24,7 @@ All file paths in the input files should either be absolute paths or relative to
For more details on what these corrections are and where to get the aux files, see: https://sentinel.esa.int/documents/247904/1653440/Sentinel-1-IPF_EAP_Phase_correction
Aux data can be accessed here:
https://qc.sentinel1.eo.esa.int/aux_cal/?instrument_configuration_id=3
https://aux.sentinel1.eo.esa.int/AUX_CAL/
Note 3: Precise orbits
@ -33,10 +33,10 @@ All file paths in the input files should either be absolute paths or relative to
Use of precise / restituted orbits are highly recommend for precise processing of Sentinel-1A interferograms. You can dump all the orbits in a common folder and ISCE will automatically identify the right orbit file to use with your data.
Precise orbit data can be accessed here and are typically available 3 weeks after the SLCs are available:
https://qc.sentinel1.eo.esa.int/aux_poeorb/
https://aux.sentinel1.eo.esa.int/POEORB/
Restituted orbits can be accessed here and are available at the same time as the SLCs:
https://qc.sentinel1.eo.esa.int/aux_resorb/
https://aux.sentinel1.eo.esa.int/RESORB/
Note 3: Multiple slices