# -*- coding: UTF-8 -*- """ @Project ：microproduct @File ：SoilMoistureMain.py @Author ：SHJ @Contact： @Date ：2021/7/26 17:11 @Version ：1.0.0 """ import logging import os import datetime import sys import numpy as np from PIL import Image import multiprocessing from scipy.optimize import minimize, fmin from tool.algorithm.algtools.PreProcess import PreProcess as pp from tool.algorithm.image.ImageHandle import ImageHandler from tool.algorithm.xml.AlgXmlHandle import ManageAlgXML, CheckSource,InitPara from tool.algorithm.algtools.logHandler import LogHandler from tool.algorithm.block.blockprocess import BlockProcess from tool.config.ConfigeHandle import Config as cf from tool.algorithm.xml.CreatMetafile import CreateMetafile from tool.algorithm.algtools.ROIAlg import ROIAlg as roi from SoilMoistureXmlInfo import CreateDict, CreateStadardXmlFile from tool.file.fileHandle import fileHandle from tool.algorithm.algtools.MetaDataHandler import MetaDataHandler from tool.algorithm.algtools.filter.lee_Filter import Filter import math multiprocessing_num = int(cf.get('multiprocessing_num')) if cf.get('debug') == 'True': DEBUG = True else: DEBUG = False EXE_NAME = cf.get('exe_name') FILTER_SIZE = int(cf.get('filter_size')) soil_moisture_value_min = float(cf.get('product_value_min')) soil_moisture_value_max = float(cf.get('product_value_max')) LogHandler.init_log_handler('run_log\\' + EXE_NAME) logger = logging.getLogger("mylog") env_str = os.path.split(os.path.realpath(__file__))[0] os.environ['PROJ_LIB'] = env_str file =fileHandle(DEBUG) class MoistureMain: """ 土壤水分处理主函数 """ def __init__(self, alg_xml_path): self.alg_xml_path = alg_xml_path self.imageHandler = ImageHandler() self.__alg_xml_handler = ManageAlgXML(alg_xml_path) self.__check_handler = CheckSource(self.__alg_xml_handler) self.__workspace_path = None self.__input_paras = {} self.__out_para = None self.__processing_paras = {} self.__preprocessed_paras = {} # 参考影像路径 self.__ref_img_path = '' # 宽/列数,高/行数 self.__cols, self.__rows = 0, 0 # 坐标系 self.__proj = '' # 影像投影变换矩阵 self.__geo = [0, 0, 0, 0, 0, 0] # 极化影像的名称 self.__polar_name = None # 多时相影像名称 self.__sar_names_list = [] def check_source(self): """ 检查算法相关的配置文件，图像，辅助文件是否齐全 """ env_str = os.getcwd() logger.info("sysdir: %s", env_str) self.__check_handler.check_alg_xml() self.__check_handler.check_run_env() self.__input_paras = self.__alg_xml_handler.get_input_paras() if self.__check_handler.check_input_paras(self.__input_paras) is False: return False self.__workspace_path = self.__alg_xml_handler.get_workspace_path() self.__create_work_space() self.__out_para = os.path.join(self.__workspace_path, EXE_NAME, 'Output', r"SoilMoistureProduct.tar.gz") self.__alg_xml_handler.write_out_para("SoilMoistureProduct", self.__out_para) #写入输出参数 Polarization = self.__input_paras['Polarization']['ParaValue'] if Polarization == "HH": self.__polar_name = "HH" elif Polarization == "VV": self.__polar_name = "VV" elif Polarization == 'empty' or Polarization == 'Empty' or Polarization == 'EMPTY': self.__polar_name = 'empty' else: raise ValueError("input Para 'Polarization' is not 'HH'、'VV'or 'empty'!") self.__processing_paras = InitPara.init_processing_paras(self.__input_paras) self.__processing_paras.update(InitPara(DEBUG).get_mult_tar_gz_infs(self.__processing_paras, self.__workspace_preprocessing_path)) self.__sar_names_list = self.__processing_paras['name_list'] if len(self.__sar_names_list) != 2: raise ValueError('number of SingleTempSAR(input para) != 2 !') logger.info('check_source success!') logger.info('progress bar: 10%') return True def __create_work_space(self): """ 删除原有工作区文件夹,创建新工作区文件夹 """ self.__workspace_preprocessing_path = self.__workspace_path + EXE_NAME + r'\Temporary\preprocessing''\\' self.__workspace_preprocessed_path = self.__workspace_path + EXE_NAME + r'\Temporary\preprocessed''\\' self.__workspace_processing_path = self.__workspace_path + EXE_NAME + r'\Temporary\processing''\\' self.__workspace_block_tif_path = self.__workspace_path + EXE_NAME + r'\Temporary\blockTif''\\' self.__workspace_block_tif_processed_path = self.__workspace_path + EXE_NAME + r'\Temporary\blockTifProcessed''\\' self.__product_dic = self.__workspace_processing_path + 'product\\' path_list = [self.__workspace_preprocessing_path, self.__workspace_preprocessed_path, self.__workspace_processing_path, self.__workspace_block_tif_path, self.__workspace_block_tif_processed_path, self.__product_dic] file.creat_dirs(path_list) logger.info('create new workspace success!') def del_temp_workspace(self): """ 临时工作区 """ if DEBUG is True: return path = self.__workspace_path + EXE_NAME + r'\Temporary' if os.path.exists(path): file.del_folder(path) def preprocess_handle(self): """ 预处理 """ # 读取每一张图像,检查图像坐标系 para_names = ['Covering', 'NDVI'] ref_img_name ='' for name in self.__sar_names_list: para_names.append(name+'_'+self.__polar_name) if ref_img_name == '': ref_img_name = name+'_'+self.__polar_name para_names.append(name+'_LocalIncidenceAngle') p = pp() self.__preprocessed_paras = p.preprocessing(para_names, ref_img_name,self.__processing_paras, self.__workspace_preprocessing_path, self.__workspace_preprocessed_path) self.__ref_img_path, self.__cols, self.__rows, self.__proj, self.__geo = p.get_ref_inf(self.__preprocessed_paras[ref_img_name]) self.__ref_img_name = ref_img_name logger.info('progress bar: 40%') logger.info('preprocess_handle success!') def create_roi(self): """ 计算ROI掩膜 :return: 掩膜路径 """ names = [self.__sar_names_list[0] + '_' +'LocalIncidenceAngle', self.__ref_img_name, 'Covering', 'NDVI'] r = roi() bare_land_mask_path = r.roi_process(names, self.__workspace_processing_path + "/roi/", self.__processing_paras, self.__preprocessed_paras) logger.info('create masks success!') # 计算roi区域 for name in self.__sar_names_list: # roi区域掩盖无效区域 roi.cal_roi(self.__preprocessed_paras[name+'_'+'LocalIncidenceAngle'],self.__preprocessed_paras[name+'_'+'LocalIncidenceAngle'], bare_land_mask_path, background_value=1) logger.info('create ROI image success!') return bare_land_mask_path def cal_bare_soil_moisure(self,moisure1_path,moisure2_path, sar1_path, angle1_path,sar2_path, angle2_path ,mask_path,x_min,x_max,f, T, bd, vsand, vclay,block_num=0,x0=0.3): """ Dobsen模型，土壤水分反演 :param f:微波频率，单位GHz :param T:温度，摄氏度 :param bd:土壤容重 :param vsand:沙土含量，范围0-1 :param vclay:黏土含量，范围0-1 :param soil_dielectric_array:土壤介电常数 :param x0 :土壤水分寻优，初始值 :return: True or False """ alpha = 0.65 sd = 2.65 dcs = (1.01 + 0.44 * sd) ** 2 - 0.062 dc0 = 0.008854 dcw0 = 88.045 - 0.4147 * T + 6.295e-4 * (T ** 2) + 1.075e-5 * (T ** 3) tpt = 0.11109 - 3.824e-3 * T + 6.938e-5 * (T ** 2) - 5.096e-7 * (T ** 3) dcwinf = 4.9 if f >= 1.4: sigma = -1.645 + 1.939 * bd - 2.013e-2 * vsand + 1.594e-2 * vclay else: sigma = 0.0467 + 0.2204 * bd - 4.111e-3 * vsand + 6.614e-3 * vclay dcwr = dcwinf + ((dcw0 - dcwinf) / (1 + (tpt * f) ** 2)) betar = 1.2748 - 0.00519 * vsand - 0.00152 * vclay betai = 1.33797 - 0.00603 * vsand - 0.00166 * vclay fun_mois = lambda vwc: np.abs(np.sqrt( ((1.0 + (bd / sd) * ((dcs ** alpha) - 1.0) + (vwc ** betar) * (dcwr ** alpha) - vwc) ** (1 / alpha)) ** 2 + ((vwc ** (betai / alpha)) * ( (tpt * f * (dcw0 - dcwinf)) / (1 + (tpt * f) ** 2) + sigma * (1.0 - (bd / sd)) / ( 8 * math.atan(1.0) * dc0 * f * vwc))) ** 2 ) - soil_dielectric) # 计算土壤水分 sar1 = self.imageHandler.get_data(sar1_path) angle1 = self.imageHandler.get_data(angle1_path) sar2 = self.imageHandler.get_data(sar2_path) angle2 = self.imageHandler.get_data(angle2_path) mask = self.imageHandler.get_data(mask_path) sin_angle1 = np.sin(angle1) ** 2 cos_angle1 = np.cos(angle1) sin_angle2 = np.sin(angle2) ** 2 cos_angle2 = np.cos(angle2) tmp = (sar2 / sar1) ** 0.5 w = np.where(mask > 0) sin_angle1 = sin_angle1[w] cos_angle1 = cos_angle1[w] sin_angle2 = sin_angle2[w] cos_angle2 = cos_angle2[w] tmp = tmp[w] num = len(w[0]) mois1, mois2, fun_value = np.zeros(num), np.zeros(num), np.zeros(num) if self.__polar_name == 'HH': fun = lambda x: np.abs( np.abs((c2 - (x[1] - s2) ** 0.5) / (c2 + (x[1] - s2) ** 0.5)) - t * np.abs((c1 - (x[0] - s1) ** 0.5) / (c1 + (x[0] - s1) ** 0.5)) ) elif self.__polar_name == 'VV': fun = lambda x: np.abs( np.abs((x[1] - 1) * (s2 - x[1] * (1 + s2)) / (x[1] * c2 + (x[1] - s2) ** 0.5) ** 2) - t * np.abs((x[0] - 1) * (s1 - x[0] * (1 + s1)) / (x[0] * c1 + (x[0] - s1) ** 0.5) ** 2) ) x_mid = 0.5 * (x_max - x_min) bnds = ((x_min, x_max), (x_min, x_max)) start = datetime.datetime.now() bar_list = [0, int(0.1 * num), int(0.2 * num), int(0.3 * num), int(0.4 * num), int(0.5 * num), int(0.6 * num), int(0.7 * num), int(0.8 * num), int(0.9 * num), num - 1] for s1, c1, s2, c2, t, n in zip(sin_angle1, cos_angle1, sin_angle2, cos_angle2, tmp, range(num)): # 计算介电常数 res = minimize(fun, (x_mid, x_mid), method='SLSQP', bounds=bnds) # print(s1, c1, s2, c2, t, n) # print("res::", res) if res.fun < 0.000001: # 计算时相1的土壤水分 soil_dielectric = res.x[0] mois1[n] = fmin(fun_mois, x0, disp=0) # Dobson模型 # 计算时相2的土壤水分 soil_dielectric = res.x[1] mois2[n] = fmin(fun_mois, x0, disp=0) # Dobson模型 if n in bar_list: logger.info('block:{},cal soil_moisture proggress bar:{}%,use time :{}'.format(block_num, round(n / num * 100), (datetime.datetime.now() - start))) sar1[:] = 0 sar2[:] = 0 sar1[w] = mois1 # 土壤水分 sar2[w] = mois2 # 土壤水分 # 保存土壤水分 out_image = Image.fromarray(sar1) out_image.save(moisure1_path) out_image = Image.fromarray(sar2) out_image.save(moisure2_path) def process_handle(self, start): """ 算法主处理函数 :return: True or False """ # 计算ROI区域 bare_land_mask_path = self.create_roi() radar_center_frequency = MetaDataHandler.get_RadarCenterFrequency(self.__processing_paras['META']) file.copyfile2dir(bare_land_mask_path, self.__workspace_preprocessed_path) logger.info('progress bar: 50%') # 分块 bp = BlockProcess() block_size = bp.get_block_size(self.__rows, self.__cols) bp.cut(self.__workspace_preprocessed_path, self.__workspace_block_tif_path, ['tif', 'tiff'], 'tif', block_size) logger.info('blocking tifs success!') img_dir, img_name = bp.get_file_names(self.__workspace_block_tif_path, ['tif']) dir_dict = bp.get_same_img(img_dir, img_name) for key in dir_dict.keys(): tmp = key if self.__sar_names_list[0]+'_LocalIncidenceAngle' in tmp: angle1_list = dir_dict[key] elif self.__sar_names_list[0]+'_'+self.__polar_name in tmp: sar1_list = dir_dict[key] elif self.__sar_names_list[1] + '_LocalIncidenceAngle' in tmp: angle2_list = dir_dict[key] elif self.__sar_names_list[1] + '_' + self.__polar_name in tmp: sar2_list = dir_dict[key] elif 'bare_land_mask'in tmp : bare_land_mask_list = dir_dict[key] if not len(angle1_list) == len(sar1_list) == len(angle2_list) == len(sar2_list) == len(bare_land_mask_list): raise Exception('[len(angle1_list) == len(sar1_list) == len(angle2_list) == len(sar2_list) == len(bare_land_mask_list] is False!') # 开启多进程处理 processes_num = min([len(angle1_list), multiprocessing_num]) if DEBUG == False: f = Filter() f.lee_filter_multiprocess(sar1_list, sar1_list, FILTER_SIZE, processes_num) f.lee_filter_multiprocess(sar2_list, sar2_list, FILTER_SIZE, processes_num) pool = multiprocessing.Pool(processes=processes_num) for i in range(len(angle1_list)): name = os.path.basename(angle1_list[i]) suffix = '_' + name.split('_')[-4] + '_' + name.split('_')[-3] + '_' + name.split('_')[-2] + '_' + \ name.split('_')[-1] # 计算土壤介电常数 moisture1_path = self.__workspace_block_tif_processed_path+self.__sar_names_list[0]+'_moisture' + suffix moisture2_path = self.__workspace_block_tif_processed_path+self.__sar_names_list[1]+'_moisture' + suffix sar1_path = sar1_list[i] angle1_path = angle1_list[i] sar2_path = sar2_list[i] angle2_path = angle2_list[i] mask_path = bare_land_mask_list[i] x_min = self.__processing_paras['SoilCDCMin'] x_max = self.__processing_paras['SoilCDCMax'] logger.info('total:%s, block:%s cal soil_moisture success!', len(angle1_list), i) pool.apply_async(self.cal_bare_soil_moisure, (moisture1_path, moisture2_path, sar1_path, angle1_path, sar2_path, angle2_path, mask_path, x_min, x_max, radar_center_frequency, self.__processing_paras['T'], self.__processing_paras['Pb'], self.__processing_paras['vsand'], self.__processing_paras['vclay'], i)) pool.close() pool.join() logger.info('progress bar: 80%') # 合并处理后的影像 data_dir = self.__workspace_block_tif_processed_path w = self.__cols h = self.__rows out_path = self.__workspace_processing_path[0:-1] tif_type = np.float32 bp.combine(data_dir, w, h, out_path, file_type=['tif'], datetype=tif_type) # 添加地理信息 soil_moisture1_path = self.__workspace_processing_path + self.__sar_names_list[0]+'_moisture'+'.tif' soil_moisture2_path = self.__workspace_processing_path + self.__sar_names_list[1]+'_moisture'+'.tif' bp.assign_spatial_reference_byfile(self.__preprocessed_paras[self.__sar_names_list[0]+'_'+self.__polar_name], soil_moisture1_path) bp.assign_spatial_reference_byfile(self.__preprocessed_paras[self.__sar_names_list[0]+'_'+self.__polar_name], soil_moisture2_path) # 生成roi区域 product1_path = self.__product_dic + self.__sar_names_list[0] + '_SoilMoistureProduct.tif' roi.cal_roi(product1_path, soil_moisture1_path, bare_land_mask_path, background_value=np.nan) product2_path = self.__product_dic + self.__sar_names_list[1] + '_SoilMoistureProduct.tif' roi.cal_roi(product2_path, soil_moisture2_path, bare_land_mask_path, background_value=np.nan) logger.info('cal soil_moisture success!') for path in [product1_path, product2_path]: proj, geos, data = self.imageHandler.read_img(path) data[data < soil_moisture_value_min] = soil_moisture_value_min data[data > soil_moisture_value_max] = soil_moisture_value_max self.imageHandler.write_img(path, proj, geos, data) # 生成快视图 self.imageHandler.write_quick_view(product1_path) self.imageHandler.write_quick_view(product2_path) # 生成元文件案例 xml_path = "./model_meta.xml" tem_folder = self.__workspace_path + EXE_NAME + r"\Temporary""\\" image_path = product1_path out_path1 = os.path.join(tem_folder, "trans_geo_projcs.tif") out_path2 = os.path.join(tem_folder, "trans_projcs_geo.tif") par_dict = CreateDict(image_path, out_path1, out_path2).calu_nature(start) model_xml_path = os.path.join(tem_folder, "creat_standard.meta.xml") # 输出xml路径 id_min = 0 id_max = 1000 threshold_of_ndvi_min = 0 threshold_of_ndvi_max = 1 set_threshold = [id_max, id_min, threshold_of_ndvi_min, threshold_of_ndvi_max] CreateStadardXmlFile(xml_path, self.alg_xml_path, par_dict, set_threshold, model_xml_path).create_standard_xml() SrcImagePath = self.__input_paras["SingleTempSAR"]['ParaValue'] paths = SrcImagePath.split(';') SrcImageName=os.path.split(paths[0])[1].split('.tar.gz')[0] if len(paths) >= 2: for i in range(1, len(paths)): SrcImageName=SrcImageName+";"+os.path.split(paths[i])[1].split('.tar.gz')[0] meta_xml_path = self.__product_dic+EXE_NAME+"Product.meta.xml" CreateMetafile(self.__processing_paras['META'], self.alg_xml_path, model_xml_path, meta_xml_path).process(SrcImageName) # 文件夹打包 file.make_targz(self.__out_para, self.__product_dic) logger.info('process_handle success!') logger.info('progress bar: 100%') if __name__ == '__main__': multiprocessing.freeze_support() start = datetime.datetime.now() try: if len(sys.argv) < 2: xml_path = 'SoilMoisture_geo.xml' else: xml_path = sys.argv[1] main_handler = MoistureMain(xml_path) if main_handler.check_source() is False: raise Exception('check_source() failed!') if main_handler.preprocess_handle() is False: raise Exception('preprocess_handle() failed!') if main_handler.process_handle(start) is False: raise Exception('process_handle() failed!') logger.info('successful production of ' + EXE_NAME + ' products!') except Exception: logger.exception('run-time error!') finally: main_handler.del_temp_workspace() end = datetime.datetime.now() msg = 'running use time: %s ' % (end - start) logger.info(msg)