From 500b9e2de2b3575dddb93627749e0d885041762f Mon Sep 17 00:00:00 2001 From: Sara Mirzaee Date: Tue, 9 Mar 2021 08:24:02 -0600 Subject: [PATCH 1/3] dloadOrbits.py: fix url change --- contrib/stack/topsStack/dloadOrbits.py | 122 ++++++++++--------------- 1 file changed, 50 insertions(+), 72 deletions(-) diff --git a/contrib/stack/topsStack/dloadOrbits.py b/contrib/stack/topsStack/dloadOrbits.py index 9bac621..c58a48b 100755 --- a/contrib/stack/topsStack/dloadOrbits.py +++ b/contrib/stack/topsStack/dloadOrbits.py @@ -8,35 +8,33 @@ import requests from html.parser import HTMLParser from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - fmt = '%Y%m%d' -today = datetime.datetime.now().strftime(fmt) +today = datetime.datetime.now().strftime(fmt) -server = 'https://qc.sentinel1.eo.esa.int/' +server = 'https://aux.sentinel1.eo.esa.int/' queryfmt = '%Y-%m-%d' datefmt = '%Y%m%dT%H%M%S' S1Astart = '20140901' S1Astart_dt = datetime.datetime.strptime(S1Astart, '%Y%m%d') - S1Bstart = '20160501' S1Bstart_dt = datetime.datetime.strptime(S1Bstart, '%Y%m%d') def cmdLineParse(): - ''' - Automated download of orbits. - ''' - parser = argparse.ArgumentParser('S1A orbit downloader') - parser.add_argument('--start','-b', dest='start', type=str, default=S1Astart, help='Start date') - parser.add_argument('--end','-e', dest='end', -type=str, default=today, help='Stop date') - parser.add_argument('--dir', '-d', dest='dirname', -type=str, default='.', help='Directory with precise orbits') - return parser.parse_args() + ''' + Automated download of orbits. + ''' + parser = argparse.ArgumentParser('S1A orbit downloader') + parser.add_argument('--start', '-b', dest='start', type=str, default=S1Astart, help='Start date') + parser.add_argument('--end', '-e', dest='end', type=str, default=today, help='Stop date') + parser.add_argument('--dir', '-d', dest='dirname', type=str, default='.', help='Directory with precise orbits') + return parser.parse_args() + def fileToRange(fname): ''' @@ -50,13 +48,14 @@ def fileToRange(fname): return (start, stop, mission) + def gatherExistingOrbits(dirname): ''' Gather existing orbits. ''' fnames = glob.glob(os.path.join(dirname, 'S1?_OPER_AUX_POEORB*')) - rangeList=[] + rangeList = [] for name in fnames: rangeList.append(fileToRange(name)) @@ -84,10 +83,11 @@ def ifAlreadyExists(indate, mission, rangeList): return found + def validS1BDate(indate): if indate < S1Bstart_dt: return False - else: + else: return True @@ -110,7 +110,7 @@ def download_file(url, outdir='.', session=None): success = False if success: - with open(path,'wb') as f: + with open(path, 'wb') as f: for chunk in request.iter_content(chunk_size=1024): if chunk: f.write(chunk) @@ -121,13 +121,13 @@ def download_file(url, outdir='.', session=None): class MyHTMLParser(HTMLParser): - def __init__(self): + def __init__(self, url): HTMLParser.__init__(self) self.fileList = [] self.in_td = False self.in_a = False self.in_table = False - self._url = None + self._url = url def handle_starttag(self, tag, attrs): if tag == 'td': @@ -135,14 +135,16 @@ class MyHTMLParser(HTMLParser): elif tag == 'a': self.in_a = True for name, val in attrs: - if name== "href": + if name == "href": if val.startswith("http"): self._url = val.strip() - def handle_data(self,data): + def handle_data(self, data): if self.in_td and self.in_a: if ('S1A_OPER' in data) or ('S1B_OPER' in data): + # print(data.strip()) self.fileList.append((self._url, data.strip())) + print(self._url, data.strip()) def handle_tag(self, tag): if tag == 'td': @@ -150,51 +152,7 @@ class MyHTMLParser(HTMLParser): self.in_a = False elif tag == 'a': self.in_a = False - self._url=None - -def query(indate, mission, session): - ''' - Query the system for a given date. - ''' - - if mission == 'S1B': - if not validS1BDate(indate): - return - - delta = datetime.timedelta(days=2) - timebef = (indate - delta).strftime(queryfmt) - timeaft = (indate + delta).strftime(queryfmt) - - url = server + 'aux_poeorb' - query = url + '/?validity_start={0}..{1}&sentinel1__mission={2}'.format(timebef, timeaft,mission) - success = False - match = None - try: - r = session.get(query, verify=False) - r.raise_for_status() - - parser = MyHTMLParser() - parser.feed(r.text) - - for resulturl, result in parser.fileList: - tbef, taft, mission = fileToRange(os.path.basename(result)) - - - if (tbef <= indate) and (taft >= indate): - #match = os.path.join(url, result) - match = resulturl - break - - if match is not None: - success = True - except: - pass - - if match is None: - print('Failed to find {1} orbits for tref {0}'.format(indate, mission)) - return - - return match + self._url = None if __name__ == '__main__': @@ -202,7 +160,7 @@ if __name__ == '__main__': Main driver. ''' - #Parse command line + # Parse command line inps = cmdLineParse() ###Compute interval @@ -212,20 +170,40 @@ if __name__ == '__main__': days = (tend - tstart).days print('Number of days to check: ', days) - ####Gather existing orbits ranges = gatherExistingOrbits(inps.dirname) - session = requests.session() for dd in range(days): indate = tstart + datetime.timedelta(days=dd, hours=12) - print('Searching for {0}'.format(indate)) + + url = server + 'POEORB/' + str(indate.year).zfill(2) + '/' + str(indate.month).zfill(2) + '/' + str( + indate.day).zfill(2) + '/' + + session = requests.session() + match = None for mission in ['S1A', 'S1B']: if not ifAlreadyExists(indate, mission, ranges): - match = query(indate, mission, session) + + try: + r = session.get(url, verify=False) + r.raise_for_status() + + parser = MyHTMLParser(url) + parser.feed(r.text) + + for resulturl, result in parser.fileList: + match = os.path.join(resulturl, result) + if match is not None: + success = True + except: + pass if match is not None: download_file(match, inps.dirname, session) - pass + else: + print('Failed to find {1} orbits for tref {0}'.format(indate, mission)) + else: print('Already exists: ', mission, indate) + + print('Exit dloadOrbits Successfully') From e40f70593f879974c48e7b57bc7bca3044abfc85 Mon Sep 17 00:00:00 2001 From: Sara Mirzaee <37273875+mirzaees@users.noreply.github.com> Date: Tue, 9 Mar 2021 09:57:24 -0500 Subject: [PATCH 2/3] Update README.md qc.sentinel1.eo.esa.int -> aux.sentinel1.eo.esa.int --- contrib/stack/topsStack/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/contrib/stack/topsStack/README.md b/contrib/stack/topsStack/README.md index a2fc2d3..09066f9 100644 --- a/contrib/stack/topsStack/README.md +++ b/contrib/stack/topsStack/README.md @@ -37,9 +37,7 @@ The following calibration auxliary (AUX_CAL) file is used for **antenna pattern Run the command below to download the AUX_CAL file once and store it somewhere (_i.e._ ~/aux/aux_cal) so that you can use it all the time, for `stackSentinel.py -a` or `auxiliary data directory` in `topsApp.py`. ``` -wget https://qc.sentinel1.eo.esa.int/product/S1A/AUX_CAL/20140908T000000/S1A_AUX_CAL_V20140908T000000_G20190626T100201.SAFE.TGZ -tar zxvf S1A_AUX_CAL_V20140908T000000_G20190626T100201.SAFE.TGZ -rm S1A_AUX_CAL_V20140908T000000_G20190626T100201.SAFE.TGZ +wget https://aux.sentinel1.eo.esa.int/AUX_CAL/2014/09/08/S1A_AUX_CAL_V20140908T000000_G20190626T100201.SAFE/ --no-check-certificate --recursive --level=1 --cut-dirs=4 -nH ``` #### 1. Create your project folder somewhere #### From 1a86bca5c406d4dee94af473fb3a8fbc5e376553 Mon Sep 17 00:00:00 2001 From: Sara Mirzaee Date: Tue, 9 Mar 2021 13:43:13 -0600 Subject: [PATCH 3/3] fix url change --- contrib/stack/topsStack/fetchOrbit.py | 147 ++++++------------ .../input_files/reference_TOPS_SENTINEL1.xml | 6 +- 2 files changed, 48 insertions(+), 105 deletions(-) diff --git a/contrib/stack/topsStack/fetchOrbit.py b/contrib/stack/topsStack/fetchOrbit.py index cfa43e4..9a11fa6 100755 --- a/contrib/stack/topsStack/fetchOrbit.py +++ b/contrib/stack/topsStack/fetchOrbit.py @@ -8,15 +8,15 @@ import argparse import datetime from html.parser import HTMLParser -server = 'https://qc.sentinel1.eo.esa.int/' -server2 = 'http://aux.sentinel1.eo.esa.int/' +server = 'http://aux.sentinel1.eo.esa.int/' -orbitMap = [('precise','aux_poeorb'), - ('restituted','aux_resorb')] +orbitMap = [('precise', 'POEORB/'), + ('restituted', 'RESORB/')] datefmt = "%Y%m%dT%H%M%S" queryfmt = "%Y-%m-%d" -queryfmt2= "%Y/%m/%d/" +queryfmt2 = "%Y/%m/%d/" + def cmdLineParse(): ''' @@ -25,9 +25,9 @@ def cmdLineParse(): parser = argparse.ArgumentParser(description='Fetch orbits corresponding to given SAFE package') parser.add_argument('-i', '--input', dest='input', type=str, required=True, - help='Path to SAFE package of interest') + help='Path to SAFE package of interest') parser.add_argument('-o', '--output', dest='outdir', type=str, default='.', - help='Path to output directory') + help='Path to output directory') return parser.parse_args() @@ -43,9 +43,9 @@ def FileToTimeStamp(safename): try: tstamp = datetime.datetime.strptime(fields[-4], datefmt) sstamp = datetime.datetime.strptime(fields[-5], datefmt) - except: - p = re.compile(r'(?<=_)\d{8}') - dt2 = p.search(safename).group() + except: + p = re.compile(r'(?<=_)\d{8}') + dt2 = p.search(safename).group() tstamp = datetime.datetime.strptime(dt2, '%Y%m%d') satName = fields[0] @@ -55,47 +55,37 @@ def FileToTimeStamp(safename): class MyHTMLParser(HTMLParser): - def __init__(self, satName): + def __init__(self, satName, url): HTMLParser.__init__(self) self.fileList = [] - self.pages = 0 self.in_td = False self.in_a = False - self.in_ul = False + self.in_table = False + self._url = url self.satName = satName + def handle_starttag(self, tag, attrs): if tag == 'td': self.in_td = True - elif tag == 'a' and self.in_td: + elif tag == 'a': self.in_a = True - elif tag == 'ul': - for k,v in attrs: - if k == 'class' and v.startswith('pagination'): - self.in_ul = True - elif tag == 'li' and self.in_ul: - self.pages += 1 + for name, val in attrs: + if name == "href": + if val.startswith("http"): + self._url = val.strip() - def handle_data(self,data): + def handle_data(self, data): if self.in_td and self.in_a: - #if 'S1A_OPER' in data: - #if 'S1B_OPER' in data: - if satName in data: - self.fileList.append(data.strip()) + if self.satName in data: + self.fileList.append((self._url, data.strip())) - def handle_endtag(self, tag): + def handle_tag(self, tag): if tag == 'td': self.in_td = False self.in_a = False - elif tag == 'a' and self.in_td: + elif tag == 'a': self.in_a = False - elif tag == 'ul' and self.in_ul: - self.in_ul = False - elif tag == 'html': - if self.pages == 0: - self.pages = 1 - else: - # decrement page back and page forward list items - self.pages -= 2 + self._url = None def download_file(url, outdir='.', session=None): @@ -117,7 +107,7 @@ def download_file(url, outdir='.', session=None): success = False if success: - with open(path,'wb') as f: + with open(path, 'wb') as f: for chunk in request.iter_content(chunk_size=1024): if chunk: f.write(chunk) @@ -149,82 +139,35 @@ if __name__ == '__main__': fileTS, satName, fileTSStart = FileToTimeStamp(inps.input) print('Reference time: ', fileTS) print('Satellite name: ', satName) - + match = None session = requests.Session() - + for spec in orbitMap: oType = spec[0] - if oType == 'precise': - delta =datetime.timedelta(days=2) - elif oType == 'restituted': - delta = datetime.timedelta(days=1) + url = server + spec[1] + str(fileTS.year).zfill(2) + '/' + str(fileTS.month).zfill(2) + \ + '/' + str(fileTS.day).zfill(2) + '/' - timebef = (fileTS - delta).strftime(queryfmt) - timeaft = (fileTS + delta).strftime(queryfmt) - - url = server + spec[1] - - query = (url + - '/?validity_start={0}..{1}&sentinel1__mission={2}' - .format(timebef, timeaft,satName)) - - print(query) success = False match = None + try: - print('Querying for {0} orbits'.format(oType)) - r = session.get(query, verify=False) + r = session.get(url, verify=False) r.raise_for_status() - parser = MyHTMLParser(satName) + + parser = MyHTMLParser(satName, url) parser.feed(r.text) - print("Found {} pages".format(parser.pages)) - - # get results from first page - results = parser.fileList - - # page through and get more results - for page in range(2, parser.pages + 1): - page_parser = MyHTMLParser(satName) - page_query = "{}&page={}".format(query, page) - print(page_query) - r = session.get(page_query, verify=False) - r.raise_for_status() - - page_parser.feed(r.text) - results.extend(page_parser.fileList) - - # run through all results and pull the orbit files - if results: - for result in results: - tbef, taft, mission = fileToRange(os.path.basename(result)) - - if (tbef <= fileTSStart) and (taft >= fileTS): - datestr2 = FileToTimeStamp(result)[0].strftime(queryfmt2) - match = (server2 + spec[1].replace('aux_', '').upper() + - '/' +datestr2+ result + '.EOF') - break - - if match is not None: - success = True - except Exception as e: - print('Exception - something went wrong with the web scraper:') - print('Exception: {}'.format(e)) - print('Continuing process') + for resulturl, result in parser.fileList: + match = os.path.join(resulturl, result) + if match is not None: + success = True + except: pass - if match is None: - print('Failed to find {0} orbits for Time {1}'.format(oType, fileTS)) - - if success: - break - - if match: - res = download_file(match, inps.outdir, session=session) - - if res is False: - print('Failed to download URL: ', match) - - session.close() - + if match is not None: + res = download_file(match, inps.outdir, session) + if res is False: + print('Failed to download URL: ', match) + else: + print('Failed to find {1} orbits for tref {0}'.format(fileTS, satName)) diff --git a/examples/input_files/reference_TOPS_SENTINEL1.xml b/examples/input_files/reference_TOPS_SENTINEL1.xml index 5483920..378fb2a 100644 --- a/examples/input_files/reference_TOPS_SENTINEL1.xml +++ b/examples/input_files/reference_TOPS_SENTINEL1.xml @@ -24,7 +24,7 @@ All file paths in the input files should either be absolute paths or relative to For more details on what these corrections are and where to get the aux files, see: https://sentinel.esa.int/documents/247904/1653440/Sentinel-1-IPF_EAP_Phase_correction Aux data can be accessed here: - https://qc.sentinel1.eo.esa.int/aux_cal/?instrument_configuration_id=3 + https://aux.sentinel1.eo.esa.int/AUX_CAL/ Note 3: Precise orbits @@ -33,10 +33,10 @@ All file paths in the input files should either be absolute paths or relative to Use of precise / restituted orbits are highly recommend for precise processing of Sentinel-1A interferograms. You can dump all the orbits in a common folder and ISCE will automatically identify the right orbit file to use with your data. Precise orbit data can be accessed here and are typically available 3 weeks after the SLCs are available: - https://qc.sentinel1.eo.esa.int/aux_poeorb/ + https://aux.sentinel1.eo.esa.int/POEORB/ Restituted orbits can be accessed here and are available at the same time as the SLCs: - https://qc.sentinel1.eo.esa.int/aux_resorb/ + https://aux.sentinel1.eo.esa.int/RESORB/ Note 3: Multiple slices