dloadOrbits.py: fix url change
Sara Mirzaee
parent
8ffb2f281a
commit
500b9e2de2
|
|
@ -8,20 +8,19 @@ import requests
|
|||
from html.parser import HTMLParser
|
||||
|
||||
from requests.packages.urllib3.exceptions import InsecureRequestWarning
|
||||
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
||||
|
||||
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
||||
|
||||
fmt = '%Y%m%d'
|
||||
today = datetime.datetime.now().strftime(fmt)
|
||||
|
||||
server = 'https://qc.sentinel1.eo.esa.int/'
|
||||
server = 'https://aux.sentinel1.eo.esa.int/'
|
||||
queryfmt = '%Y-%m-%d'
|
||||
datefmt = '%Y%m%dT%H%M%S'
|
||||
|
||||
S1Astart = '20140901'
|
||||
S1Astart_dt = datetime.datetime.strptime(S1Astart, '%Y%m%d')
|
||||
|
||||
|
||||
S1Bstart = '20160501'
|
||||
S1Bstart_dt = datetime.datetime.strptime(S1Bstart, '%Y%m%d')
|
||||
|
||||
|
|
@ -32,12 +31,11 @@ def cmdLineParse():
|
|||
'''
|
||||
parser = argparse.ArgumentParser('S1A orbit downloader')
|
||||
parser.add_argument('--start', '-b', dest='start', type=str, default=S1Astart, help='Start date')
|
||||
parser.add_argument('--end','-e', dest='end',
|
||||
type=str, default=today, help='Stop date')
|
||||
parser.add_argument('--dir', '-d', dest='dirname',
|
||||
type=str, default='.', help='Directory with precise orbits')
|
||||
parser.add_argument('--end', '-e', dest='end', type=str, default=today, help='Stop date')
|
||||
parser.add_argument('--dir', '-d', dest='dirname', type=str, default='.', help='Directory with precise orbits')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def fileToRange(fname):
|
||||
'''
|
||||
Derive datetime range from orbit file name.
|
||||
|
|
@ -50,6 +48,7 @@ def fileToRange(fname):
|
|||
|
||||
return (start, stop, mission)
|
||||
|
||||
|
||||
def gatherExistingOrbits(dirname):
|
||||
'''
|
||||
Gather existing orbits.
|
||||
|
|
@ -84,6 +83,7 @@ def ifAlreadyExists(indate, mission, rangeList):
|
|||
|
||||
return found
|
||||
|
||||
|
||||
def validS1BDate(indate):
|
||||
if indate < S1Bstart_dt:
|
||||
return False
|
||||
|
|
@ -121,13 +121,13 @@ def download_file(url, outdir='.', session=None):
|
|||
|
||||
class MyHTMLParser(HTMLParser):
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, url):
|
||||
HTMLParser.__init__(self)
|
||||
self.fileList = []
|
||||
self.in_td = False
|
||||
self.in_a = False
|
||||
self.in_table = False
|
||||
self._url = None
|
||||
self._url = url
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == 'td':
|
||||
|
|
@ -142,7 +142,9 @@ class MyHTMLParser(HTMLParser):
|
|||
def handle_data(self, data):
|
||||
if self.in_td and self.in_a:
|
||||
if ('S1A_OPER' in data) or ('S1B_OPER' in data):
|
||||
# print(data.strip())
|
||||
self.fileList.append((self._url, data.strip()))
|
||||
print(self._url, data.strip())
|
||||
|
||||
def handle_tag(self, tag):
|
||||
if tag == 'td':
|
||||
|
|
@ -152,50 +154,6 @@ class MyHTMLParser(HTMLParser):
|
|||
self.in_a = False
|
||||
self._url = None
|
||||
|
||||
def query(indate, mission, session):
|
||||
'''
|
||||
Query the system for a given date.
|
||||
'''
|
||||
|
||||
if mission == 'S1B':
|
||||
if not validS1BDate(indate):
|
||||
return
|
||||
|
||||
delta = datetime.timedelta(days=2)
|
||||
timebef = (indate - delta).strftime(queryfmt)
|
||||
timeaft = (indate + delta).strftime(queryfmt)
|
||||
|
||||
url = server + 'aux_poeorb'
|
||||
query = url + '/?validity_start={0}..{1}&sentinel1__mission={2}'.format(timebef, timeaft,mission)
|
||||
success = False
|
||||
match = None
|
||||
try:
|
||||
r = session.get(query, verify=False)
|
||||
r.raise_for_status()
|
||||
|
||||
parser = MyHTMLParser()
|
||||
parser.feed(r.text)
|
||||
|
||||
for resulturl, result in parser.fileList:
|
||||
tbef, taft, mission = fileToRange(os.path.basename(result))
|
||||
|
||||
|
||||
if (tbef <= indate) and (taft >= indate):
|
||||
#match = os.path.join(url, result)
|
||||
match = resulturl
|
||||
break
|
||||
|
||||
if match is not None:
|
||||
success = True
|
||||
except:
|
||||
pass
|
||||
|
||||
if match is None:
|
||||
print('Failed to find {1} orbits for tref {0}'.format(indate, mission))
|
||||
return
|
||||
|
||||
return match
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
'''
|
||||
|
|
@ -212,20 +170,40 @@ if __name__ == '__main__':
|
|||
days = (tend - tstart).days
|
||||
print('Number of days to check: ', days)
|
||||
|
||||
####Gather existing orbits
|
||||
ranges = gatherExistingOrbits(inps.dirname)
|
||||
session = requests.session()
|
||||
|
||||
for dd in range(days):
|
||||
indate = tstart + datetime.timedelta(days=dd, hours=12)
|
||||
print('Searching for {0}'.format(indate))
|
||||
|
||||
url = server + 'POEORB/' + str(indate.year).zfill(2) + '/' + str(indate.month).zfill(2) + '/' + str(
|
||||
indate.day).zfill(2) + '/'
|
||||
|
||||
session = requests.session()
|
||||
match = None
|
||||
|
||||
for mission in ['S1A', 'S1B']:
|
||||
if not ifAlreadyExists(indate, mission, ranges):
|
||||
match = query(indate, mission, session)
|
||||
|
||||
try:
|
||||
r = session.get(url, verify=False)
|
||||
r.raise_for_status()
|
||||
|
||||
parser = MyHTMLParser(url)
|
||||
parser.feed(r.text)
|
||||
|
||||
for resulturl, result in parser.fileList:
|
||||
match = os.path.join(resulturl, result)
|
||||
if match is not None:
|
||||
success = True
|
||||
except:
|
||||
pass
|
||||
|
||||
if match is not None:
|
||||
download_file(match, inps.dirname, session)
|
||||
pass
|
||||
else:
|
||||
print('Failed to find {1} orbits for tref {0}'.format(indate, mission))
|
||||
|
||||
else:
|
||||
print('Already exists: ', mission, indate)
|
||||
|
||||
print('Exit dloadOrbits Successfully')
|
||||
|
|
|
|||
Loading…
Reference in New Issue