283 lines
11 KiB
Python
283 lines
11 KiB
Python
|
# coding:utf-8
|
||
|
|
# coding=gbk
|
||
|
|
|
||
|
|
import os, sys
|
||
|
|
# import multiprocessing
|
||
|
|
# import subprocess
|
||
|
|
#import chardet
|
||
|
|
import concurrent
|
||
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
|
import argparse
|
||
|
|
import logging
|
||
|
|
from logHandler import LogHandler
|
||
|
|
LogHandler.init_log_handler('run_log\\' + 'ISCE_LOG')
|
||
|
|
logger = logging.getLogger("mylog")
|
||
|
|
|
||
|
|
|
||
|
|
def excteCMD(cmd_txt):
|
||
|
|
logger.info("\twaitline cmdline:\t{} \t ".format(cmd_txt))
|
||
|
|
result=os.system(cmd_txt)
|
||
|
|
cmdlines = ""
|
||
|
|
if result!=0:
|
||
|
|
with os.popen("{} 2>&1".format(cmd_txt)) as p:
|
||
|
|
cmdlines = p.read()
|
||
|
|
logger.info("\t state:\t{}\t cmdline:\t{} \t{} \n {}\n{}".format(result ,cmd_txt,"="*10,cmdlines,"*"*40))
|
||
|
|
return result
|
||
|
|
"""
|
||
|
|
logger.info("waitline cmdline:\t{} \t++++++++:".format(cmd_txt))
|
||
|
|
cmd_p = subprocess.Popen(cmd_txt, shell=True,stdout=subprocess.PIPE,cwd=os.getcwd(),bufsize=1,universal_newlines=True,encoding="utf-8")
|
||
|
|
cmdlines=""
|
||
|
|
while subprocess.Popen.poll(cmd_p) is None:
|
||
|
|
line = cmd_p.stdout.readline()
|
||
|
|
charls = chardet.detect(line)
|
||
|
|
try:
|
||
|
|
cmdlines="{}{}\n".format(cmdlines,line.decode(charls['encoding']))
|
||
|
|
print(line.decode(charls['encoding']))
|
||
|
|
except:
|
||
|
|
continue
|
||
|
|
cmd_p.wait()
|
||
|
|
result=cmd_p.returncode
|
||
|
|
if result==0:
|
||
|
|
cmdlines=""
|
||
|
|
logger.info("\n{}\t state:\t{}\t cmdline:\t{} \t {} \n {}\n{}".format("="*10,result ,cmd_txt,"="*10,cmdlines,"="*40))
|
||
|
|
return result
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
def buildCmdTxt(cmd_line):
|
||
|
|
cmdStr = cmd_line.replace("\n","")
|
||
|
|
if runAsExeFlag == False:
|
||
|
|
cmd_txt = "python ./{0}/{1}".format(py_folder_name, cmdStr)
|
||
|
|
return cmdStr
|
||
|
|
pyFileName=cmdStr.split(' ')[0]
|
||
|
|
if len(pyFileName) == 0:
|
||
|
|
return ''
|
||
|
|
exeName=''
|
||
|
|
if cygwinFlag:
|
||
|
|
exeName = "./{0}.exe".format(pyFileName.split('.')[0])
|
||
|
|
else:
|
||
|
|
exeName = "{0}.exe".format(pyFileName.split('.')[0])
|
||
|
|
|
||
|
|
newCmdLine = os.path.join(py_folder_name, cmdStr.replace(pyFileName, exeName))
|
||
|
|
return newCmdLine
|
||
|
|
pass
|
||
|
|
|
||
|
|
def sigProcess(cmd_lines,log_path):
|
||
|
|
#print("*********sigProcess log_path:{0}".format(log_path))
|
||
|
|
|
||
|
|
for cmd_line in cmd_lines:
|
||
|
|
#cmd_txt="python ./{0}/{1}".format(py_folder_name,cmd_line.replace("\n",""))
|
||
|
|
cmd_txt= buildCmdTxt(cmd_line)
|
||
|
|
|
||
|
|
#print("cmd_line:{0}".format(cmd_line))
|
||
|
|
print("cmd_txt:{0}".format(cmd_txt))
|
||
|
|
if len(cmd_txt) == 0:
|
||
|
|
print("cmd_line{0} cmd_txt is null".format(cmd_line))
|
||
|
|
continue
|
||
|
|
res = excteCMD(cmd_txt)
|
||
|
|
if res != 0:
|
||
|
|
print("=================================发生了错误=======================")
|
||
|
|
sys.exit(0)
|
||
|
|
pass
|
||
|
|
|
||
|
|
def muilProcess(cmd_lines,log_path,process_num=4):
|
||
|
|
print("*********muilProcess log_path:{0}".format(log_path))
|
||
|
|
with ThreadPoolExecutor(max_workers=process_num) as pool:
|
||
|
|
pl = []
|
||
|
|
for cmd_line in cmd_lines:
|
||
|
|
#cmd_txt="python {0}/{1}".format(py_folder_name,cmd_line.replace("\n",""))
|
||
|
|
cmd_txt = buildCmdTxt(cmd_line)
|
||
|
|
if len(cmd_txt) == 0:
|
||
|
|
print("cmd_line{0} cmd_txt is null".format(cmd_line))
|
||
|
|
continue
|
||
|
|
pl.append(pool.submit(excteCMD,cmd_txt))
|
||
|
|
for future in as_completed(pl):
|
||
|
|
data = future.result()
|
||
|
|
if data != 0:
|
||
|
|
print("=================================发生了错误=======================")
|
||
|
|
sys.exit(0)
|
||
|
|
pass
|
||
|
|
|
||
|
|
def createParser():
|
||
|
|
parser = argparse.ArgumentParser( description='get out path')
|
||
|
|
parser.add_argument('-e', '--execpath', type=str, dest='execpath',default='',
|
||
|
|
help='Directory with executable path')
|
||
|
|
parser.add_argument('-o', '--outpath', type=str, dest='outpath',default='',
|
||
|
|
help='Directory with the out path')
|
||
|
|
parser.add_argument('-s', '--step', type=int, dest='step',default=-1,
|
||
|
|
help='Directory with the out path')
|
||
|
|
parser.add_argument('-cygwinFlag', '--cygwinFlag', dest='cygwinFlag', default=False,
|
||
|
|
help='cygwin:True,windows:False')
|
||
|
|
parser.add_argument('-runAsExe', '--runAsExe', dest='runAsExe', default=True,
|
||
|
|
help='控制run_files里命令的运行类型,True:执行exe,False执行py文件')
|
||
|
|
return parser
|
||
|
|
|
||
|
|
def cmdLineParse(iargs=None):
|
||
|
|
'''
|
||
|
|
Command line parser.
|
||
|
|
'''
|
||
|
|
parser = createParser()
|
||
|
|
return parser.parse_args(args=iargs)
|
||
|
|
|
||
|
|
def remove_ISCEApp_from_xml(xmlPath):
|
||
|
|
"""
|
||
|
|
修改配置文件的错误路径
|
||
|
|
"""
|
||
|
|
if '.xml' in xmlPath:
|
||
|
|
import xml.dom.minidom as xmldom
|
||
|
|
domobj = xmldom.parse(xmlPath)
|
||
|
|
rootNode = domobj.documentElement
|
||
|
|
# print("elementobj:{0}".format(rootNode.nodeName))
|
||
|
|
pathInxml = ''
|
||
|
|
# 获得子标签
|
||
|
|
propertyElementObj = rootNode.getElementsByTagName("property")
|
||
|
|
for property in propertyElementObj:
|
||
|
|
if property.hasAttribute("name"):
|
||
|
|
if property.getAttribute("name") == "file_name" or property.getAttribute("name") == "metadata_location":
|
||
|
|
pathNode = property.getElementsByTagName("value")[0]
|
||
|
|
pathInxml = pathNode.childNodes[0].data
|
||
|
|
print('pathInxml:', pathInxml)
|
||
|
|
if '/ISCEApp/' in pathInxml:
|
||
|
|
pathInxml = pathInxml[pathInxml.find('/ISCEApp/')+9:]
|
||
|
|
print('Modify PathInxml:', pathInxml)
|
||
|
|
if pathInxml[0]=='/':
|
||
|
|
pathInxml = pathInxml[1:]
|
||
|
|
print('Modify PathInxml:', pathInxml)
|
||
|
|
|
||
|
|
pathNode.childNodes[0].data = pathInxml
|
||
|
|
|
||
|
|
with open(xmlPath, 'w') as f:
|
||
|
|
# 缩进换行编码
|
||
|
|
domobj.writexml(f, addindent=' ', encoding='utf-8')
|
||
|
|
elif 'slc.vrt'in xmlPath:
|
||
|
|
from xml.etree.ElementTree import ElementTree
|
||
|
|
tree = ElementTree()
|
||
|
|
tree.parse(xmlPath)
|
||
|
|
root = tree.getroot()
|
||
|
|
pathInxml = root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text
|
||
|
|
print('pathInxml:', pathInxml)
|
||
|
|
if '/ISCEApp/' in pathInxml:
|
||
|
|
pathInxml_new = pathInxml[0:7] + pathInxml[pathInxml.find('/ISCEApp/') + 8:]
|
||
|
|
root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text = pathInxml_new
|
||
|
|
print('Modify pathInxml:', pathInxml_new)
|
||
|
|
pathInxml = root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text
|
||
|
|
if '//' in pathInxml:
|
||
|
|
root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text = pathInxml.replace('//', '/')
|
||
|
|
print('Modify pathInxml:', root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text)
|
||
|
|
tree.write(xmlPath, encoding="utf-8", xml_declaration=True)
|
||
|
|
|
||
|
|
# xml_result = []
|
||
|
|
def get_xml(cwd,step):
|
||
|
|
get_dir = os.listdir(cwd)
|
||
|
|
for i in get_dir:
|
||
|
|
sub_dir = os.path.join(cwd, i)
|
||
|
|
if os.path.isdir(sub_dir):
|
||
|
|
get_xml(sub_dir, step)
|
||
|
|
else:
|
||
|
|
if (step == '01' or step == '07') and i == 'IW1_top.xml' or i == 'IW1_bottom.xml' or i == 'IW2_top.xml' or i == 'IW2_bottom.xml':
|
||
|
|
xml_result.append(sub_dir)
|
||
|
|
elif (step == '01' or step == '02') and 'burst' in i and 'slc.vrt' in i and 'bot' not in i and 'top' not in i:
|
||
|
|
xml_result.append(sub_dir)
|
||
|
|
return xml_result
|
||
|
|
|
||
|
|
def modify_error_xml(workspace,step):
|
||
|
|
global xml_result
|
||
|
|
xml_result = []
|
||
|
|
if step == '01':
|
||
|
|
search_dir = os.path.join(workspace, 'reference')
|
||
|
|
path_list = get_xml(search_dir,step)
|
||
|
|
for xml_path in path_list:
|
||
|
|
print('xml_path:',xml_path)
|
||
|
|
remove_ISCEApp_from_xml(xml_path)
|
||
|
|
if step == '02':
|
||
|
|
search_dir = os.path.join(workspace, 'secondarys')
|
||
|
|
path_list = get_xml(search_dir,step)
|
||
|
|
for xml_path in path_list:
|
||
|
|
print('xml_path:', xml_path)
|
||
|
|
remove_ISCEApp_from_xml(xml_path)
|
||
|
|
if step == '07':
|
||
|
|
search_dir = os.path.join(workspace, 'coarse_interferograms')
|
||
|
|
path_list = get_xml(search_dir,step)
|
||
|
|
for xml_path in path_list:
|
||
|
|
print('xml_path:', xml_path)
|
||
|
|
remove_ISCEApp_from_xml(xml_path)
|
||
|
|
if step == '07':
|
||
|
|
search_dir = os.path.join(workspace, 'coreg_secondarys')
|
||
|
|
path_list = get_xml(search_dir,step)
|
||
|
|
for xml_path in path_list:
|
||
|
|
print('xml_path:', xml_path)
|
||
|
|
remove_ISCEApp_from_xml(xml_path)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def auto_run_main(iargs=None):
|
||
|
|
global root_path ,py_folder_name,log_path,runAsExeFlag,cygwinFlag
|
||
|
|
print('auto_run_main iargs:{}'.format(sys.argv))
|
||
|
|
|
||
|
|
runAsExeFlag = True
|
||
|
|
|
||
|
|
inputArg = cmdLineParse(iargs)
|
||
|
|
if len(inputArg.outpath) > 0:
|
||
|
|
root_path = inputArg.outpath
|
||
|
|
else:
|
||
|
|
root_path = os.getcwd()
|
||
|
|
cygwinFlag = inputArg.cygwinFlag
|
||
|
|
if len(inputArg.execpath) > 0:
|
||
|
|
py_folder_name = inputArg.execpath
|
||
|
|
else:
|
||
|
|
py_folder_name = os.getcwd()
|
||
|
|
print("root_path:{0}".format(root_path))
|
||
|
|
log_path = os.path.join(root_path, "log.txt")
|
||
|
|
|
||
|
|
run_files_path = os.path.join(root_path, "run_files")
|
||
|
|
if not os.path.exists(run_files_path):
|
||
|
|
os.makedirs(run_files_path)
|
||
|
|
|
||
|
|
cmd_text_filename = os.listdir(run_files_path)
|
||
|
|
sorted(cmd_text_filename, key=lambda filename:float(filename.split("_")[1]))
|
||
|
|
|
||
|
|
for filename in cmd_text_filename:
|
||
|
|
logger.info(log_path + "=Excetding data=================")
|
||
|
|
|
||
|
|
step = filename.split("_")[1]
|
||
|
|
if inputArg.step!=-1 and int(step)!=inputArg.step:
|
||
|
|
#print("step:{0} inputArg.step:{0}".format(int(step),inputArg.step))
|
||
|
|
continue
|
||
|
|
'''
|
||
|
|
else:
|
||
|
|
print("step:{0} inputArg.step:{0}".format(int(step),inputArg.step))
|
||
|
|
continue
|
||
|
|
'''
|
||
|
|
logger.info(log_path + "=command text:{}=========".format(filename))
|
||
|
|
cmd_lines = []
|
||
|
|
with open(os.path.join(run_files_path, filename), 'r', encoding='utf-8') as fp:
|
||
|
|
cmd_lines = fp.readlines()
|
||
|
|
|
||
|
|
# if step in ['01']:#,'01',"02","03","04",'05','06','07','08','09','10',11','12','13','14','15','16']: # 已经执行过的指令
|
||
|
|
# continue
|
||
|
|
|
||
|
|
msg = "=" * 10 + 'running step:' + step + "=" * 10
|
||
|
|
logger.info(msg)
|
||
|
|
print(msg)
|
||
|
|
# modify_error_xml(root_path, step)
|
||
|
|
|
||
|
|
if step in ['04', '05', '08', '09', '11','12']:
|
||
|
|
sigProcess(cmd_lines, log_path)
|
||
|
|
elif step in ["01", '02', "03", '06', '07', '13', '14', '15']:
|
||
|
|
muilProcess(cmd_lines, log_path, process_num=4)
|
||
|
|
elif step in ['10']:
|
||
|
|
muilProcess(cmd_lines, log_path, process_num=4)
|
||
|
|
elif step in ['09']:
|
||
|
|
muilProcess(cmd_lines, log_path, process_num=2)
|
||
|
|
elif step in ['16']:
|
||
|
|
muilProcess(cmd_lines, log_path, process_num=4)
|
||
|
|
modify_error_xml(root_path, step)
|
||
|
|
logger.info( "\n====================================exceted=========================\n")
|
||
|
|
|
||
|
|
# if __name__ == '__main__':
|
||
|
|
# sys.argv.extend([r'-oI:/MicroWorkspace/C-SAR/Deformation/Temporary/processing/isce_workspace','-eI:/onestar/deformation-sentiral/ISCEApp'])
|
||
|
|
# sys.argv.extend([r'-oI:/MicroWorkspace/C-SAR/Deformation/Temporary/processing/isce_workspace','-eI:/onestar/deformation-sentiral/ISCEProgram'])
|
||
|
|
# auto_run_main(sys.argv[1:])
|
||
|
|
# -o I:/MicroWorkspace/C-SAR/Deformation/Temporary/processing/isce_workspace -e I:/onestar/deformation-sentiral/ISCEApp
|