KJ135
/
ISCE_INSAR
mirror of http://172.16.0.12:5000/WBJY/ISCE_INSAR.git
4
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Proposed changes to the dloadOrbits.py to reflect the new orbit file download links (#300) 

I've edited the dloadOrbits.py to reflect the new orbit file download links. This may also address issue # https://github.com/isce-framework/isce2/issues/299.
LT1AB
Bryan Marfito 2021-07-22 23:50:46 +08:00 committed by GitHub
parent 3928c3bdb7
commit 5297f385fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 46 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
contrib/stack/topsStack/dloadOrbits.py
View File

@ -7,17 +7,16 @@ import glob
import requests import requests
from html.parser import HTMLParser from html.parser import HTMLParser
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
fmt = '%Y%m%d' fmt = '%Y%m%d'
today = datetime.datetime.now().strftime(fmt) today = datetime.datetime.now().strftime(fmt)
server = 'https://aux.sentinel1.eo.esa.int/' server = 'https://scihub.copernicus.eu/gnss/'
queryfmt = '%Y-%m-%d' queryfmt = '%Y-%m-%d'
datefmt = '%Y%m%dT%H%M%S' datefmt = '%Y%m%dT%H%M%S'
#Generic credentials to query and download orbit files
credentials = ('gnssguest', 'gnssguest')
S1Astart = '20140901' S1Astart = '20140901'
S1Astart_dt = datetime.datetime.strptime(S1Astart, '%Y%m%d') S1Astart_dt = datetime.datetime.strptime(S1Astart, '%Y%m%d')
@ -29,7 +28,7 @@ def cmdLineParse():
''' '''
Automated download of orbits. Automated download of orbits.
''' '''
parser = argparse.ArgumentParser('S1A orbit downloader') parser = argparse.ArgumentParser('S1A and 1B AUX_POEORB precise orbit downloader')
parser.add_argument('--start', '-b', dest='start', type=str, default=S1Astart, help='Start date') parser.add_argument('--start', '-b', dest='start', type=str, default=S1Astart, help='Start date')
parser.add_argument('--end', '-e', dest='end', type=str, default=today, help='Stop date') parser.add_argument('--end', '-e', dest='end', type=str, default=today, help='Stop date')
parser.add_argument('--dir', '-d', dest='dirname', type=str, default='.', help='Directory with precise orbits') parser.add_argument('--dir', '-d', dest='dirname', type=str, default='.', help='Directory with precise orbits')
@ -99,9 +98,9 @@ def download_file(url, outdir='.', session=None):
if session is None: if session is None:
session = requests.session() session = requests.session()
path = os.path.join(outdir, os.path.basename(url)) path = outdir
print('Downloading URL: ', url) print('Downloading URL: ', url)
request = session.get(url, stream=True, verify=False) request = session.get(url, stream=True, verify=True, auth=credentials)
try: try:
request.raise_for_status() request.raise_for_status()
@ -121,38 +120,25 @@ def download_file(url, outdir='.', session=None):
class MyHTMLParser(HTMLParser): class MyHTMLParser(HTMLParser):
def __init__(self, url): def __init__(self,url):
HTMLParser.__init__(self) HTMLParser.__init__(self)
self.fileList = [] self.fileList = []
self.in_td = False
self.in_a = False
self.in_table = False
self._url = url self._url = url
def handle_starttag(self, tag, attrs): def handle_starttag(self, tag, attrs):
if tag == 'td': for name, val in attrs:
self.in_td = True if name == 'href':
elif tag == 'a': if val.startswith("https://scihub.copernicus.eu/gnss/odata") and val.endswith(")/"):
self.in_a = True pass
for name, val in attrs: else:
if name == "href": downloadLink = val.strip()
if val.startswith("http"): downloadLink = downloadLink.split("/Products('Quicklook')")
self._url = val.strip() downloadLink = downloadLink[0] + downloadLink[-1]
self._url = downloadLink
def handle_data(self, data): def handle_data(self, data):
if self.in_td and self.in_a: if data.startswith("S1") and data.endswith(".EOF"):
if ('S1A_OPER' in data) or ('S1B_OPER' in data): self.fileList.append((self._url, data.strip()))
# print(data.strip())
self.fileList.append((self._url, data.strip()))
print(self._url, data.strip())
def handle_tag(self, tag):
if tag == 'td':
self.in_td = False
self.in_a = False
elif tag == 'a':
self.in_a = False
self._url = None
if __name__ == '__main__': if __name__ == '__main__':
@ -174,36 +160,44 @@ if __name__ == '__main__':
for dd in range(days): for dd in range(days):
indate = tstart + datetime.timedelta(days=dd, hours=12) indate = tstart + datetime.timedelta(days=dd, hours=12)
timebef = indate - datetime.timedelta(days=1)
url = server + 'POEORB/' + str(indate.year).zfill(2) + '/' + str(indate.month).zfill(2) + '/' + str( timeaft = indate + datetime.timedelta(days=1)
indate.day).zfill(2) + '/' timebef=str(timebef.strftime('%Y-%m-%d'))
timeaft = str(timeaft.strftime('%Y-%m-%d'))
url = server + 'search?q= ( beginPosition:[{0}T00:00:00.000Z TO {1}T23:59:59.999Z] AND endPosition:[{0}T00:00:00.000Z TO {1}T23:59:59.999Z] ) AND ( (platformname:Sentinel-1 AND producttype:AUX_POEORB))'.format(timebef, timeaft)
session = requests.session() session = requests.session()
match = None match = None
success = False
for mission in ['S1A', 'S1B']:
if not ifAlreadyExists(indate, mission, ranges): for selectMission in ['S1A', 'S1B']:
if not ifAlreadyExists(indate, selectMission, ranges):
try: try:
r = session.get(url, verify=False) r = session.get(url, verify=True, auth=credentials)
r.raise_for_status() r.raise_for_status()
parser = MyHTMLParser(url) parser = MyHTMLParser(url)
parser.feed(r.text) parser.feed(r.text)
for resulturl, result in parser.fileList: for resulturl, result in parser.fileList:
match = os.path.join(resulturl, result) tbef, taft, mission = fileToRange(os.path.basename(result))
if match is not None: if selectMission==mission:
success = True matchFileName = result
match = resulturl
if match is not None:
success = True
except: except:
pass pass
if match is not None: if match is not None:
download_file(match, inps.dirname, session)
output = os.path.join(inps.dirname, matchFileName)
print(output)
res = download_file(match, output, session)
else: else:
print('Failed to find {1} orbits for tref {0}'.format(indate, mission)) print('Failed to find {1} orbits for tref {0}'.format(indate, selectMission))
else: else:
print('Already exists: ', mission, indate) print('Already exists: ', selectMission, indate)
print('Exit dloadOrbits Successfully') print('Exit dloadOrbits Successfully')