# coding:utf-8 # coding=gbk import os, sys # import multiprocessing # import subprocess # import chardet from concurrent.futures import ThreadPoolExecutor,as_completed import argparse import logging from tool.logs.logHandler import LogHandler # from logHandler import LogHandler LogHandler.init_log_handler('run_log\\' + 'ISCE_LOG') logger = logging.getLogger("mylog") def excteCMD(cmd_txt): logger.info("\twaitline cmdline:\t{} \t ".format(cmd_txt)) result=os.system(cmd_txt) cmdlines = "" if result!=0: with os.popen("{} 2>&1".format(cmd_txt, encoding='utf-8')) as p: cmdlines = p.read() logger.info("\t state:\t{}\t cmdline:\t{} \t{} \n {}\n{}".format(result ,cmd_txt,"="*10,cmdlines,"*"*40)) return result """ logger.info("waitline cmdline:\t{} \t++++++++:".format(cmd_txt)) cmd_p = subprocess.Popen(cmd_txt, shell=True,stdout=subprocess.PIPE,cwd=os.getcwd(),bufsize=1,universal_newlines=True,encoding="utf-8") cmdlines="" while subprocess.Popen.poll(cmd_p) is None: line = cmd_p.stdout.readline() charls = chardet.detect(line) try: cmdlines="{}{}\n".format(cmdlines,line.decode(charls['encoding'])) print(line.decode(charls['encoding'])) except: continue cmd_p.wait() result=cmd_p.returncode if result==0: cmdlines="" logger.info("\n{}\t state:\t{}\t cmdline:\t{} \t {} \n {}\n{}".format("="*10,result ,cmd_txt,"="*10,cmdlines,"="*40)) return result """ def buildCmdTxt(cmd_line): cmdStr = cmd_line.replace("\n","") if runAsExeFlag == False: cmd_txt = "python ./{0}/{1}".format(py_folder_name, cmdStr) return cmdStr pyFileName=cmdStr.split(' ')[0] if len(pyFileName) == 0: return '' exeName='' if cygwinFlag: exeName = "./{0}.exe".format(pyFileName.split('.')[0]) else: exeName = "{0}.exe".format(pyFileName.split('.')[0]) newCmdLine = os.path.join(py_folder_name, cmdStr.replace(pyFileName, exeName)) return newCmdLine pass def sigProcess(cmd_lines,log_path): #print("*********sigProcess log_path:{0}".format(log_path)) for cmd_line in cmd_lines: #cmd_txt="python ./{0}/{1}".format(py_folder_name,cmd_line.replace("\n","")) cmd_txt= buildCmdTxt(cmd_line) #print("cmd_line:{0}".format(cmd_line)) print("cmd_txt:{0}".format(cmd_txt)) if len(cmd_txt) == 0: print("cmd_line{0} cmd_txt is null".format(cmd_line)) continue res = excteCMD(cmd_txt) if res != 0: print("=================================发生了错误=======================") sys.exit(0) pass def muilProcess(cmd_lines,log_path,process_num=4): print("*********muilProcess log_path:{0}".format(log_path)) with ThreadPoolExecutor(max_workers=process_num) as pool: pl = [] for cmd_line in cmd_lines: #cmd_txt="python {0}/{1}".format(py_folder_name,cmd_line.replace("\n","")) cmd_txt = buildCmdTxt(cmd_line) if len(cmd_txt) == 0: print("cmd_line{0} cmd_txt is null".format(cmd_line)) continue pl.append(pool.submit(excteCMD,cmd_txt)) for future in as_completed(pl): data = future.result() if data != 0: print("=================================发生了错误=======================") sys.exit(0) pass def createParser(): parser = argparse.ArgumentParser( description='get out path') parser.add_argument('-e', '--execpath', type=str, dest='execpath',default='', help='Directory with executable path') parser.add_argument('-o', '--outpath', type=str, dest='outpath',default='', help='Directory with the out path') parser.add_argument('-s', '--step', type=int, dest='step',default=-1, help='Directory with the out path') parser.add_argument('-cygwinFlag', '--cygwinFlag', dest='cygwinFlag', default=False, help='cygwin:True,windows:False') parser.add_argument('-runAsExe', '--runAsExe', dest='runAsExe', default=True, help='控制run_files里命令的运行类型,True:执行exe,False执行py文件') return parser def cmdLineParse(iargs=None): ''' Command line parser. ''' parser = createParser() return parser.parse_args(args=iargs) def remove_ISCEApp_from_xml(xmlPath): """ 修改配置文件的错误路径 """ if '.xml' in xmlPath: import xml.dom.minidom as xmldom domobj = xmldom.parse(xmlPath) rootNode = domobj.documentElement # print("elementobj:{0}".format(rootNode.nodeName)) pathInxml = '' # 获得子标签 propertyElementObj = rootNode.getElementsByTagName("property") for property in propertyElementObj: if property.hasAttribute("name"): if property.getAttribute("name") == "file_name" or property.getAttribute("name") == "metadata_location": pathNode = property.getElementsByTagName("value")[0] pathInxml = pathNode.childNodes[0].data print('pathInxml:', pathInxml) if '/ISCEApp/' in pathInxml: pathInxml = pathInxml[pathInxml.find('/ISCEApp/')+9:] print('Modify PathInxml:', pathInxml) if pathInxml[0]=='/': pathInxml = pathInxml[1:] print('Modify PathInxml:', pathInxml) pathNode.childNodes[0].data = pathInxml with open(xmlPath, 'w') as f: # 缩进换行编码 domobj.writexml(f, addindent=' ', encoding='utf-8') elif 'slc.vrt'in xmlPath: from xml.etree.ElementTree import ElementTree tree = ElementTree() tree.parse(xmlPath) root = tree.getroot() pathInxml = root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text print('pathInxml:', pathInxml) if '/ISCEApp/' in pathInxml: pathInxml_new = pathInxml[0:7] + pathInxml[pathInxml.find('/ISCEApp/') + 8:] root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text = pathInxml_new print('Modify pathInxml:', pathInxml_new) pathInxml = root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text if '//' in pathInxml: root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text = pathInxml.replace('//', '/') print('Modify pathInxml:', root.find('VRTRasterBand').find('SimpleSource').find('SourceFilename').text) tree.write(xmlPath, encoding="utf-8", xml_declaration=True) # xml_result = [] def get_xml(cwd,step): get_dir = os.listdir(cwd) for i in get_dir: sub_dir = os.path.join(cwd, i) if os.path.isdir(sub_dir): get_xml(sub_dir, step) else: if (step == '01' or step == '07') and i == 'IW1_top.xml' or i == 'IW1_bottom.xml' or i == 'IW2_top.xml' or i == 'IW2_bottom.xml': xml_result.append(sub_dir) elif (step == '01' or step == '02') and 'burst' in i and 'slc.vrt' in i and 'bot' not in i and 'top' not in i: xml_result.append(sub_dir) return xml_result def modify_error_xml(workspace,step): global xml_result xml_result = [] if step == '01': search_dir = os.path.join(workspace, 'reference') path_list = get_xml(search_dir,step) for xml_path in path_list: print('xml_path:',xml_path) remove_ISCEApp_from_xml(xml_path) if step == '02': search_dir = os.path.join(workspace, 'secondarys') path_list = get_xml(search_dir,step) for xml_path in path_list: print('xml_path:', xml_path) remove_ISCEApp_from_xml(xml_path) if step == '07': search_dir = os.path.join(workspace, 'coarse_interferograms') path_list = get_xml(search_dir,step) for xml_path in path_list: print('xml_path:', xml_path) remove_ISCEApp_from_xml(xml_path) if step == '07': search_dir = os.path.join(workspace, 'coreg_secondarys') path_list = get_xml(search_dir,step) for xml_path in path_list: print('xml_path:', xml_path) remove_ISCEApp_from_xml(xml_path) def auto_run_main(iargs=None): global root_path ,py_folder_name,log_path,runAsExeFlag,cygwinFlag print('auto_run_main iargs:{}'.format(sys.argv)) runAsExeFlag = True inputArg = cmdLineParse(iargs) if len(inputArg.outpath) > 0: root_path = inputArg.outpath else: root_path = os.getcwd() cygwinFlag = inputArg.cygwinFlag if len(inputArg.execpath) > 0: py_folder_name = inputArg.execpath else: py_folder_name = os.getcwd() print("root_path:{0}".format(root_path)) log_path = os.path.join(root_path, "log.txt") run_files_path = os.path.join(root_path, "run_files") if not os.path.exists(run_files_path): os.makedirs(run_files_path) cmd_text_filename = os.listdir(run_files_path) sorted(cmd_text_filename, key=lambda filename:float(filename.split("_")[1])) for filename in cmd_text_filename: logger.info(log_path + "=Excetding data=================") step = filename.split("_")[1] if inputArg.step!=-1 and int(step)!=inputArg.step: #print("step:{0} inputArg.step:{0}".format(int(step),inputArg.step)) continue ''' else: print("step:{0} inputArg.step:{0}".format(int(step),inputArg.step)) continue ''' logger.info(log_path + "=command text:{}=========".format(filename)) cmd_lines = [] with open(os.path.join(run_files_path, filename), 'r', encoding='utf-8') as fp: cmd_lines = fp.readlines() # if step in ['01']:#,'01',"02","03","04",'05','06','07','08','09','10',11','12','13','14','15','16']: # 已经执行过的指令 # continue msg = "=" * 10 + 'running step:' + step + "=" * 10 logger.info(msg) print(msg) # modify_error_xml(root_path, step) if step in ['04', '05', '08', '09', '11','12']: sigProcess(cmd_lines, log_path) elif step in ["01", '02', "03", '06', '07', '13', '14', '15']: muilProcess(cmd_lines, log_path, process_num=4) elif step in ['10']: muilProcess(cmd_lines, log_path, process_num=4) elif step in ['09']: muilProcess(cmd_lines, log_path, process_num=2) elif step in ['16']: muilProcess(cmd_lines, log_path, process_num=4) modify_error_xml(root_path, step) logger.info( "\n====================================exceted=========================\n") if __name__ == '__main__': # sys.argv.extend([r'-oI:/MicroWorkspace/C-SAR/Deformation/Temporary/processing/isce_workspace','-eI:/onestar/deformation-sentiral/ISCEApp']) # sys.argv.extend([r'-oI:/MicroWorkspace/C-SAR/Deformation/Temporary/processing/isce_workspace','-eI:/onestar/deformation-sentiral/ISCEProgram']) auto_run_main(sys.argv[1:]) # -o I:/MicroWorkspace/C-SAR/Deformation/Temporary/processing/isce_workspace -e I:/onestar/deformation-sentiral/ISCEApp