# -*- coding: UTF-8 -*- """ @Project ：microproduct @File ：SurfaceSalinityMain.py @Author ：SHJ @Contact：土壤粗糙度算法主函数 @Date ：2022/11/7 9:04 @Version ：2.0.0 """ import logging from tool.algorithm.algtools.PreProcess import PreProcess as pp from tool.algorithm.image.ImageHandle import ImageHandler from tool.algorithm.xml.AlgXmlHandle import ManageAlgXML, CheckSource, InitPara from tool.algorithm.algtools.logHandler import LogHandler from tool.algorithm.block.blockprocess import BlockProcess from tool.algorithm.algtools.MetaDataHandler import MetaDataHandler from tool.algorithm.xml.CreatMetafile import CreateMetafile from tool.config.ConfigeHandle import Config as cf from tool.algorithm.algtools.ROIAlg import ROIAlg from tool.file.fileHandle import fileHandle import pyproj._compat from tool.csv.csvHandle import csvHandle from tool.algorithm.algtools.filter.lee_Filter import Filter from SurfaceRoughnessALg import RoughnessAlg as alg from SurfaceRoughnessXmlInfo import CreateDict, CreateStadardXmlFile import os import datetime import numpy as np import shutil import scipy #解决打包错误 import scipy.spatial.transform # 用于解决打包错误 import scipy.spatial.transform._rotation_groups # 用于解决打包错误 import scipy.special.cython_special # 用于解决打包错误 import sys from scipy.interpolate import griddata # 引入scipy中的二维插值库 from scipy.special import lambertw import math import multiprocessing surface_roughness_value_min = float(cf.get('product_value_min')) surface_roughness_value_max = float(cf.get('product_value_max')) if cf.get('debug') == 'True': DEBUG = True else: DEBUG = False file = fileHandle(DEBUG) EXE_NAME = cf.get('exe_name') FILTER_SIZE = int(cf.get('filter_size')) env_str = os.path.split(os.path.realpath(__file__))[0] os.environ['PROJ_LIB'] = env_str LogHandler.init_log_handler('run_log\\' + EXE_NAME) logger = logging.getLogger("mylog") csvh = csvHandle() class RoughnessMain: """ 算法主函数 """ def __init__(self, alg_xml_path): self.alg_xml_path = alg_xml_path self.imageHandler = ImageHandler() self.__alg_xml_handler = ManageAlgXML(alg_xml_path) self.__check_handler = CheckSource(self.__alg_xml_handler) self.__workspace_path, self.__sar_tif_name, self.__out_para = None, None, None self.__input_paras, self.__output_paras, self.__processing_paras, self.__preprocessed_paras = {}, {}, {}, {} # 参考影像路径,坐标系 self.__ref_img_path, self.__proj = '', '' # 宽/列数,高/行数 self.__cols, self.__rows = 0, 0 # 影像投影变换矩阵 self.__geo = [0, 0, 0, 0, 0, 0] def check_source(self): """ 检查算法相关的配置文件，图像，辅助文件是否齐全 """ env_str = os.getcwd() logger.info("sysdir: %s", env_str) self.__check_handler.check_alg_xml() self.__check_handler.check_run_env() self.__workspace_path = self.__alg_xml_handler.get_workspace_path() self.__input_paras = self.__alg_xml_handler.get_input_paras() Polarization = self.__input_paras['Polarization']['ParaValue'] if Polarization == "VV": self.__sar_tif_name = "VV" elif Polarization == "HH": self.__sar_tif_name = "HH" else: raise ValueError("input Para 'Polarization' is not 'HH' or 'VV'!") input_para_names = ["SingleSAR", "MeasuredData", "Covering", "NDVI"] if self.__check_handler.check_input_paras(input_para_names) is False: return False self.__create_work_space() self.__processing_paras = InitPara.init_processing_paras(self.__input_paras) self.__processing_paras.update(self.get_tar_gz_inf(self.__processing_paras["sar_path0"])) self.__out_para = os.path.join(self.__workspace_path, EXE_NAME, 'Output', r"SurfaceRoughnessProduct.tar.gz") self.__alg_xml_handler.write_out_para("SurfaceRoughnessProduct", self.__out_para) #写入输出参数 logger.info('check_source success!') logger.info('progress bar: 10%') return True def get_tar_gz_inf(self, tar_gz_path): para_dic = {} name = os.path.split(tar_gz_path)[1].rstrip('.tar.gz') file_dir = os.path.join(self.__workspace_preprocessing_path, name + '\\') file.de_targz(tar_gz_path, file_dir) # 元文件字典 para_dic.update(InitPara.get_meta_dic(InitPara.get_meta_paths(file_dir, name), name)) # tif路径字典 pol_dic = InitPara.get_polarization_mode(InitPara.get_tif_paths(file_dir, name)) flag_list = [0, 0, 0,0] VVVH_list = [0, 0] # 获取极化类型 if "HH" in pol_dic.keys() and self.__sar_tif_name == 'HH': para_dic.update({self.__sar_tif_name: pol_dic['HH']}) flag_list[0] = 1 if "VV" in pol_dic.keys() and self.__sar_tif_name == 'VV': para_dic.update({self.__sar_tif_name: pol_dic['VV']}) flag_list[1] = 1 if "HV" in pol_dic.keys(): para_dic.update({'HV': pol_dic['HV']}) flag_list[3] = 1 if "LocalIncidenceAngle" in pol_dic.keys(): para_dic.update({'LocalIncidenceAngle': pol_dic['LocalIncidenceAngle']}) flag_list[2] = 1 if 'VH' in pol_dic.keys(): # 判断VV、VH是否存在，用来计算参数L para_dic.update({'VH': pol_dic['VH']}) VVVH_list[0] = 1 if 'VV' in pol_dic.keys(): VVVH_list[1] = 1 if self.__sar_tif_name == 'HH': # 只有手动选择的是'HH'，需要把VV路径添加到字典中， para_dic.update({'VV': pol_dic['VV']}) else: # 反之代表则选择的是'VV'，不需要把VV路径重复添加到字典中 pass if flag_list != [1, 0, 1, 1] and flag_list != [0, 1, 1, 1]: raise Exception('There are not ' + self.__sar_tif_name + '、LocalIncidenceAngle or meta.xml in path:', tar_gz_path) if VVVH_list != [1, 1]: raise Exception('There are not VH or VV:', tar_gz_path) return para_dic def __create_work_space(self): """ 删除原有工作区文件夹,创建新工作区文件夹 """ self.__workspace_preprocessing_path = self.__workspace_path + EXE_NAME + r"\Temporary\preprocessing""\\" self.__workspace_preprocessed_path = self.__workspace_path + EXE_NAME + r"\Temporary\preprocessed""\\" self.__workspace_processing_path = self.__workspace_path + EXE_NAME + r"\Temporary\processing""\\" self.__workspace_block_tif_path = self.__workspace_path + EXE_NAME + r"\Temporary\blockTif""\\" self.__workspace_block_tif_processed_path = self.__workspace_path + EXE_NAME + r"\Temporary\blockTifProcessed""\\" self.__workspace_Zs_block_tif_processed_path = self.__workspace_path + EXE_NAME + r"\Temporary\ZsblockTifProcessed""\\" self.__feature_tif_dic = self.__workspace_processing_path + 'feature_tif' self.__product_dic = self.__workspace_processing_path + 'product\\' path_list = [self.__workspace_preprocessing_path, self.__workspace_preprocessed_path, self.__workspace_processing_path, self.__workspace_block_tif_path, self.__workspace_block_tif_processed_path, self.__workspace_Zs_block_tif_processed_path, self.__feature_tif_dic, self.__product_dic] file.creat_dirs(path_list) logger.info('create new workspace success!') def del_temp_workspace(self): """ 临时工作区 """ if DEBUG is True: return path = self.__workspace_path + EXE_NAME + r"\Temporary" if os.path.exists(path): file.del_folder(path) def preprocess_handle(self): """ 预处理 """ para_names=[] # 读取每一张图像,检查图像坐标系 if self.__sar_tif_name == 'HH': # VH与VV参与预处理 para_names = [self.__sar_tif_name, 'VH','HV', 'VV', "Covering", "NDVI", "LocalIncidenceAngle"] elif self.__sar_tif_name == 'VV': para_names = [self.__sar_tif_name,'VH','HV','HH' "Covering", "NDVI", "LocalIncidenceAngle"] # if self.__sar_tif_name == 'HH': # VH与VV参与预处理 # para_names = [self.__sar_tif_name, 'VH', 'VV', "Covering", "NDVI", "LocalIncidenceAngle"] # elif self.__sar_tif_name == 'VV': # para_names = [self.__sar_tif_name, 'VH', "Covering", "NDVI", "LocalIncidenceAngle"] ref_img_name = self.__sar_tif_name p = pp() self.__preprocessed_paras = p.preprocessing(para_names, ref_img_name, self.__processing_paras, self.__workspace_preprocessing_path, self.__workspace_preprocessed_path) self.__ref_img_path, self.__cols, self.__rows, self.__proj, self.__geo = p.get_ref_inf( self.__preprocessed_paras[ref_img_name]) logger.info('preprocess_handle success!') logger.info('progress bar: 30%') def create_roi(self): """ 计算ROI掩膜 :return: 掩膜路径 """ names = ['LocalIncidenceAngle', self.__sar_tif_name, 'Covering', 'NDVI'] r = ROIAlg() bare_land_mask_path = r.roi_process(names, self.__workspace_processing_path + "/roi/", self.__processing_paras, self.__preprocessed_paras) logger.info('create masks success!') # 计算roi区域 ROIAlg().cal_roi(self.__preprocessed_paras["LocalIncidenceAngle"], self.__preprocessed_paras["LocalIncidenceAngle"], bare_land_mask_path, background_value=1) ROIAlg().cal_roi(self.__preprocessed_paras[self.__sar_tif_name], self.__preprocessed_paras[self.__sar_tif_name], bare_land_mask_path, background_value=1) shutil.copyfile(bare_land_mask_path, self.__workspace_preprocessed_path + 'mask.tif') logger.info('create ROI image success!') return bare_land_mask_path def create_soilDLCT_tif(self): """ 创建土壤介电常数影像 """ # 读取实测值，从经纬度坐标系转为图像坐标系 measured_data_img = csvh.trans_measuredata(csvh.readcsv(self.__processing_paras['MeasuredData']), self.__ref_img_path) # 插值法生成土壤介电常数二维影像 grid_x, grid_y = np.mgrid[0:self.__rows:1, 0:self.__cols:1] points = np.array(measured_data_img)[:,0:2] values = np.array(measured_data_img)[:,2].T grid_z0 = griddata(points, values, (grid_x, grid_y), method='nearest') self.__processing_paras['DLCT'] = os.path.join(self.__workspace_preprocessed_path,'DLCT.tif') self.imageHandler.write_img(self.__processing_paras['DLCT'],self.__proj,self.__geo,grid_z0) logger.info('create soilDLCT image success!') def Calu_Q(self,meta_file_path): """计算参数交叉极化比值Q""" VV_proj, VV_geotrans, VV_arr=self.imageHandler.read_img(self.__preprocessed_paras["VV"]) VH_proj, VH_geotrans, VH_arr = self.imageHandler.read_img(self.__preprocessed_paras["VH"]) Array_Q=VH_arr/VV_arr Out_Q_path=os.path.join(self.__workspace_preprocessed_path,"CrossPolarRatior_q.tif") self.imageHandler.write_img(Out_Q_path, VV_proj, VV_geotrans, Array_Q) self.__preprocessed_paras.update({"Ratior_q": Out_Q_path}) return Out_Q_path def Calu_H(self,Q_path,angle_path,lamda): """ 计算几何特征复合因子H Q_path：交叉极化比值Q数据 angle_path：角度文件 lamda：波长 """ wave_k=2*math.pi/lamda array_angle=self.imageHandler.get_data(angle_path) # 弧度 Q_proj, Q_geo, Q_arr = self.imageHandler.read_img(Q_path) temp_val=np.power((0.13 + np.sin(1.5 * array_angle)),1.4) temp_A=Q_arr/(0.095*temp_val) Array_H=np.power((-1*np.log(1-temp_A)/1.3),10/9)/wave_k Out_H_path=os.path.join(self.__workspace_preprocessed_path, "Factor_H.tif") self.imageHandler.write_img(Out_H_path, Q_proj, Q_geo, Array_H) def Calu_L_Zs(self, Zs_path,L_path, H_path, Fr_path, lamda, seita_Path): """ 计算几何特征复合因子L、Zs Zs_path：输出几何特征复合因子 L文件 H_path：输入几何特征复合因子 Fr文件 Fr_path：输入的Fr_path lamda：输入的波长 seita_Path：输入的入射角 """ H_proj, H_geo, H_arr = self.imageHandler.read_img(H_path) Fr_proj, Fr_geo, Fr_arr = self.imageHandler.read_img(Fr_path) array_angle=self.imageHandler.get_data(seita_Path) # 弧度 k=2*math.pi/lamda a=0.5*H_arr*H_arr b=-k*np.sin(array_angle) y=Fr_arr ans1 = 2 * lambertw(-b * np.sqrt(y / a) / 2) / b # 计算L的两个值 ans2 = 2 * lambertw(b * np.sqrt(y / a) / 2) / b # 计算L的两个值 if "complex" in ans1.dtype.name: ans1_real=ans1.real ans2_real = ans2.real else: ans1_real=ans1 ans2_real = ans2 chazhi=ans1_real-ans2_real L=np.where(chazhi>0,ans1_real,ans2_real) L[np.isnan(L)] = 0 # 必须剔除异常值，否则无法写入到tif中 self.imageHandler.write_img(L_path, Fr_proj, Fr_geo, L) Zs=np.power(H_arr, 2)/L Zs[np.isnan(Zs)] = 0 # 必须剔除异常值，否则无法写入到tif中 self.imageHandler.write_img(Zs_path, Fr_proj, Fr_geo, Zs) def process_handle(self, start): """ 算法主处理函数 :return: True or False """ # 单视复数转DB meta_file_path = self.__processing_paras['META'] # 获取元文件波长数据,单位转为cm lamda = MetaDataHandler.get_lamda(meta_file_path) * 100 # 创建土壤介电常数影像 # self.create_soilDLCT_tif() logger.info('progress bar: 50%') # 计算roi bare_land_mask_path = self.create_roi() # 计算参数交叉极化比值Q、几何特征复合因子H,参考论文2.3 Oh模型 # Out_Q_path=self.Calu_Q(meta_file_path) # self.Calu_H(Out_Q_path, self.__preprocessed_paras['LocalIncidenceAngle'], self.__processing_paras['lamda']) # 分块 bp = BlockProcess() block_size = bp.get_block_size(self.__rows, self.__cols) bp.cut(self.__workspace_preprocessed_path, self.__workspace_block_tif_path, ['tif', 'tiff'], 'tif', block_size) # 分块裁剪 logger.info('blocking tifs success!') img_dir, img_name = bp.get_file_names(self.__workspace_block_tif_path, ['tif']) dir_dict = bp.get_same_img(img_dir, img_name) # 同一类别放在一个字典下 #改2-取分块的影像VV,VH hh_list, vv_list, vh_list, hv_list, mask_list = None, None, None, None, None angle_list = None sar_tif_list = None dlct_list = None Factor_H_list = None for key in dir_dict.keys(): tmp = key.split("_", 2)[0] if tmp == "LocalIncidenceAngle": angle_list = dir_dict[key] elif tmp == 'HH': hh_list = dir_dict[key] elif tmp == 'VV': vv_list = dir_dict[key] elif tmp == 'VH': vh_list = dir_dict[key] elif tmp == 'HV': hv_list = dir_dict[key] elif tmp == 'mask': mask_list = dir_dict[key] # elif tmp == self.__sar_tif_name: # sar_tif_list = dir_dict[key] elif tmp == "DLCT": dlct_list = dir_dict[key] elif tmp == "Factor": Factor_H_list = dir_dict[key] # if angle_list is None or sar_tif_list is None or dlct_list is None or Factor_H_list is None: # raise ValueError('angle_list is None or sar_tif_list is None or dlct_list or Factor_H_list is None') # # if not len(angle_list) == len(sar_tif_list) == len(dlct_list) == len(dlct_list): # raise ValueError('[en(angle_list) == len(sar_tif_list) == len(dlct_list)] is False!') #原方法 if hh_list is None or vv_list is None or vh_list is None or hv_list is None or mask_list is None: logger.info('angle_list is None or hh_list is None or vv_list is None or vh_list is None or hv_list is None or ndwi_list is None') return False if not len(hh_list) == len(vv_list) == len(vh_list) == len(hv_list) == len(mask_list): logger.info('[len(angle_list) == len(hh_list) == len(vv_list) == len(vh_list) == len(hv_list) == len(ndwi_list)] is False!') return False #lee滤波 f = Filter() processes_num = min([len(hh_list), 4]) # f.lee_filter_multiprocess(sar_tif_list, sar_tif_list, FILTER_SIZE, processes_num) f.lee_filter_multiprocess(hh_list, hh_list, FILTER_SIZE, processes_num) f.lee_filter_multiprocess(vv_list, vv_list, FILTER_SIZE, processes_num) f.lee_filter_multiprocess(vh_list, vh_list, FILTER_SIZE, processes_num) f.lee_filter_multiprocess(hv_list, hv_list, FILTER_SIZE, processes_num) pool = multiprocessing.Pool(processes=processes_num) pl = [] # 计算地表粗糙度 for i in range(len(angle_list)): name = os.path.basename(angle_list[i]) suffix = '_' + name.split('_')[-4] + "_" + name.split('_')[-3] + "_" + name.split('_')[-2] + "_" + \ name.split('_')[-1] # dielectric_const_factor_path = self.__workspace_processing_path + "dielectric_const_factor.tif" # alg.cal_dielectric_const_factor(dielectric_const_factor_path, dlct_list[i], # angle_list[i], self.__sar_tif_name) surface_roughness_path = self.__workspace_block_tif_processed_path + "surface_roughness" + suffix pl.append(pool.apply_async(alg.cal_soil_roug, (surface_roughness_path, hh_list[i], vv_list[i], hv_list[i], vh_list[i], angle_list[i], mask_list[i], lamda))) # alg.cal_soil_roug(surface_roughness_path, hh_list[i], vv_list[i], hv_list[i], vh_list[i], angle_list[i], # mask_list[i], lamda) # alg.cal_soil_roughness(surface_roughness_path, sar_tif_list[i], # dielectric_const_factor_path, angle_list[i], # self.__processing_paras['lamda']) # #改5- 用l和h计算出ZS，参考论文2.2 公式7 # H_path=Factor_H_list[i] # Fr_path=surface_roughness_path # k=self.__processing_paras['lamda'] # seita_Path=angle_list[i] # # Zs_path=self.__workspace_Zs_block_tif_processed_path + "Zs" + suffix # L_path = self.__workspace_Zs_block_tif_processed_path + "Param_L" + suffix # self.Calu_L_Zs(Zs_path,L_path, H_path, Fr_path,k,seita_Path) logger.info('total:%s, block:%s cal surfaceRoughness success!', len(angle_list), i) pool.close() pool.join() # 合并处理后的影像 data_dir = self.__workspace_block_tif_processed_path w = ImageHandler.get_img_width(self.__preprocessed_paras['HH']) h = ImageHandler.get_img_height(self.__preprocessed_paras["HH"]) out_path = self.__workspace_processing_path[0:-1] tif_type = bp.get_tif_dtype(self.__preprocessed_paras['HH']) bp.combine(data_dir, w, h, out_path, file_type=['tif'], datetype=tif_type) # 添加地理信息 surface_roughness_path = self.__workspace_processing_path + "surface_roughness.tif" bp.assign_spatial_reference_byfile(self.__preprocessed_paras['HH'], surface_roughness_path) logger.info('progress bar: 80%') # # Zs拼接成一幅tif数据 # Zs_data_dir = self.__workspace_Zs_block_tif_processed_path # ref_proj, ref_geo, im_arr = ImageHandler.read_img(surface_roughness_path) # bp.combine_Tif(Zs_data_dir, w, h, out_path, ref_proj, ref_geo, out_type='tif', file_type=['tif', 'tiff'], datetype='float16') # Zs_path = self.__workspace_processing_path + "Zs.tif" # L_path = self.__workspace_processing_path + "Param_L.tif" # Out_H_path = os.path.join(self.__workspace_preprocessed_path, "Factor_H.tif") # # # product_Zs_path = os.path.join(self.__product_dic, "Zs.tif") # product_L_path = os.path.join(self.__product_dic, "L.tif") # product_H_path = os.path.join(self.__product_dic, "H.tif") # # alg.cal_roi(product_Zs_path, Zs_path, bare_land_mask_path, background_value=np.nan) # ROIAlg().cal_roi(product_L_path, L_path, bare_land_mask_path, background_value=np.nan) # ROIAlg().cal_roi(product_H_path, Out_H_path, bare_land_mask_path, background_value=np.nan) #生成roi区域 product_mask_path = self.__workspace_processing_path + 'SurfaceRoughnessProductMask.tif' ROIAlg().cal_roi(product_mask_path, surface_roughness_path, bare_land_mask_path, background_value=np.nan) product_path = os.path.join(self.__product_dic, 'SurfaceRoughnessProduct.tif') self.imageHandler.limit_field(product_path, product_mask_path, surface_roughness_value_min, surface_roughness_value_max) logger.info('cal soil_roughness success!') # 生成快视图 self.imageHandler.write_quick_view(product_path) # self.imageHandler.write_quick_view(product_Zs_path) # 新增加三个 # self.imageHandler.write_quick_view(product_L_path) # self.imageHandler.write_quick_view(product_H_path) logger.info('packing!') xml_path = "./model_meta.xml" tem_folder = self.__workspace_path + EXE_NAME + r"\Temporary""\\" image_path = product_path out_path1 = os.path.join(tem_folder, "trans_geo_projcs.tif") out_path2 = os.path.join(tem_folder, "trans_projcs_geo.tif") par_dict = CreateDict(image_path, out_path1, out_path2).calu_nature(start) model_xml_path = os.path.join(tem_folder, "creat_standard.meta.xml") # 输出xml路径 CreateStadardXmlFile(xml_path, self.alg_xml_path, par_dict, model_xml_path).create_standard_xml() # 文件夹打包 SrcImagePath = self.__input_paras["SingleSAR"]['ParaValue'] paths = SrcImagePath.split(';') SrcImageName = os.path.split(paths[0])[1].split('.tar.gz')[0] if len(paths) >= 2: for i in range(1, len(paths)): SrcImageName=SrcImageName+";"+os.path.split(paths[i])[1].split('.tar.gz')[0] meta_xml_path = self.__product_dic+EXE_NAME+"Product.meta.xml" CreateMetafile(self.__processing_paras['META'], self.alg_xml_path, model_xml_path, meta_xml_path).process(SrcImageName) file.make_targz(self.__out_para, self.__product_dic) logger.info('process_handle success!') logger.info('progress bar: 100%') if __name__ == '__main__': multiprocessing.freeze_support() #解决多进程打包成exe后，运行错误 start = datetime.datetime.now() try: if len(sys.argv) < 2: xml_path = 'SurfaceRoughness.xml' else: xml_path = sys.argv[1] main_handler = RoughnessMain(xml_path) if main_handler.check_source() is False: raise Exception('check_source() failed!') if main_handler.preprocess_handle() is False: raise Exception('preprocess_handle() failed!') if main_handler.process_handle(start) is False: raise Exception('process_handle() failed!') logger.info('successful production of ' + EXE_NAME + ' products!') except Exception: logger.exception("run-time error!") finally: main_handler.del_temp_workspace() end = datetime.datetime.now() msg = 'running use time: %s ' % (end - start) logger.info(msg)