# -*- coding: UTF-8 -*- """ @Project ：microproduct @File ：SoilMoistureMain.py @Author ：SHJ @Contact： @Date ：2021/7/26 17:11 @Version ：1.0.0 """ import logging import os import datetime import sys import numpy as np from PIL import Image import multiprocessing from scipy.optimize import minimize, fmin from tool.algorithm.algtools.PreProcess import PreProcess as pp from tool.algorithm.image.ImageHandle import ImageHandler from tool.algorithm.xml.AlgXmlHandle import ManageAlgXML, CheckSource,InitPara from tool.algorithm.algtools.logHandler import LogHandler from tool.algorithm.block.blockprocess import BlockProcess from tool.config.ConfigeHandle import Config as cf from tool.algorithm.xml.CreatMetafile import CreateMetafile from tool.algorithm.algtools.ROIAlg import ROIAlg as roi from SoilMoistureXmlInfo import CreateDict, CreateStadardXmlFile from tool.file.fileHandle import fileHandle from tool.algorithm.algtools.MetaDataHandler import MetaDataHandler from tool.algorithm.algtools.filter.lee_Filter import Filter import math multiprocessing_num = int(cf.get('multiprocessing_num')) if cf.get('debug') == 'True': DEBUG = True else: DEBUG = False EXE_NAME = cf.get('exe_name') FILTER_SIZE = int(cf.get('filter_size')) soil_moisture_value_min = float(cf.get('product_value_min')) soil_moisture_value_max = float(cf.get('product_value_max')) LogHandler.init_log_handler('run_log\\' + EXE_NAME) logger = logging.getLogger("mylog") env_str = os.path.split(os.path.realpath(__file__))[0] os.environ['PROJ_LIB'] = env_str file =fileHandle(DEBUG) class MoistureMain: """ 土壤水分处理主函数 """ def __init__(self, alg_xml_path): self.alg_xml_path = alg_xml_path self.imageHandler = ImageHandler() self.__alg_xml_handler = ManageAlgXML(alg_xml_path) self.__check_handler = CheckSource(self.__alg_xml_handler) self.__workspace_path = None self.__input_paras = {} self.__out_para = None self.__processing_paras = {} self.__preprocessed_paras = {} # 参考影像路径 self.__ref_img_path = '' # 宽/列数,高/行数 self.__cols, self.__rows = 0, 0 # 坐标系 self.__proj = '' # 影像投影变换矩阵 self.__geo = [0, 0, 0, 0, 0, 0] # 极化影像的名称 self.__polar_name = None # 多时相影像名称 self.__sar_names_list = [] def check_source(self): """ 检查算法相关的配置文件，图像，辅助文件是否齐全 """ env_str = os.getcwd() logger.info("sysdir: %s", env_str) self.__check_handler.check_alg_xml() self.__check_handler.check_run_env() self.__input_paras = self.__alg_xml_handler.get_input_paras() if self.__check_handler.check_input_paras(self.__input_paras) is False: return False self.__workspace_path = self.__alg_xml_handler.get_workspace_path() self.__create_work_space() self.__out_para = os.path.join(self.__workspace_path, EXE_NAME, 'Output', r"SoilMoistureProduct.tar.gz") self.__alg_xml_handler.write_out_para("SoilMoistureProduct", self.__out_para) #写入输出参数 Polarization = self.__input_paras['Polarization']['ParaValue'] if Polarization == "HH": self.__polar_name = "HH" elif Polarization == "VV": self.__polar_name = "VV" elif Polarization == 'empty' or Polarization == 'Empty' or Polarization == 'EMPTY': self.__polar_name = 'empty' else: raise ValueError("input Para 'Polarization' is not 'HH'、'VV'or 'empty'!") self.__processing_paras = InitPara.init_processing_paras(self.__input_paras, self.__workspace_preprocessed_path) self.__processing_paras.update(InitPara(DEBUG).get_mult_tar_gz_infs(self.__processing_paras, self.__workspace_preprocessing_path)) self.__sar_names_list = self.__processing_paras['name_list'] if len(self.__sar_names_list) != 2: raise ValueError('number of SingleTempSAR(input para) != 2 !') logger.info('check_source success!') logger.info('progress bar: 10%') return True def __create_work_space(self): """ 删除原有工作区文件夹,创建新工作区文件夹 """ self.__workspace_preprocessing_path = self.__workspace_path + EXE_NAME + r'\Temporary\preprocessing''\\' self.__workspace_preprocessed_path = self.__workspace_path + EXE_NAME + r'\Temporary\preprocessed''\\' self.__workspace_processing_path = self.__workspace_path + EXE_NAME + r'\Temporary\processing''\\' self.__workspace_block_tif_path = self.__workspace_path + EXE_NAME + r'\Temporary\blockTif''\\' self.__workspace_block_tif_processed_path = self.__workspace_path + EXE_NAME + r'\Temporary\blockTifProcessed''\\' self.__product_dic = self.__workspace_processing_path + 'product\\' path_list = [self.__workspace_preprocessing_path, self.__workspace_preprocessed_path, self.__workspace_processing_path, self.__workspace_block_tif_path, self.__workspace_block_tif_processed_path, self.__product_dic] file.creat_dirs(path_list) logger.info('create new workspace success!') def del_temp_workspace(self): """ 临时工作区 """ if DEBUG is True: return path = self.__workspace_path + EXE_NAME + r'\Temporary' if os.path.exists(path): file.del_folder(path) def preprocess_handle(self): """ 预处理 """ # 读取每一张图像,检查图像坐标系 para_names = ['Covering', 'NDVI'] ref_img_name ='' for name in self.__sar_names_list: para_names.append(name+'_'+self.__polar_name) # todo 名称根据LT04实际命名进行缩短 if ref_img_name == '': ref_img_name = name+'_'+self.__polar_name para_names.append(name+'_LocalIncidenceAngle') p = pp() self.__preprocessed_paras = p.preprocessing(para_names, ref_img_name,self.__processing_paras, self.__workspace_preprocessing_path, self.__workspace_preprocessed_path) self.__ref_img_path, self.__cols, self.__rows, self.__proj, self.__geo = p.get_ref_inf(self.__preprocessed_paras[ref_img_name]) self.__ref_img_name = ref_img_name logger.info('progress bar: 40%') logger.info('preprocess_handle success!') def create_roi(self): """ 计算ROI掩膜 :return: 掩膜路径 """ names = [self.__sar_names_list[0] + '_' +'LocalIncidenceAngle', self.__ref_img_name, 'Covering', 'NDVI'] r = roi() bare_land_mask_path = r.roi_process(names, self.__workspace_processing_path + "/roi/", self.__processing_paras, self.__preprocessed_paras) logger.info('create masks success!') # 计算roi区域 for name in self.__sar_names_list: # roi区域掩盖无效区域 roi.cal_roi(self.__preprocessed_paras[name+'_'+'LocalIncidenceAngle'],self.__preprocessed_paras[name+'_'+'LocalIncidenceAngle'], bare_land_mask_path, background_value=1) logger.info('create ROI image success!') return bare_land_mask_path def cal_bare_soil_moisure(self,moisure1_path,moisure2_path, sar1_path, angle1_path,sar2_path, angle2_path ,mask_path,x_min,x_max,f, T, bd, vsand, vclay,block_num=0,x0=0.3): """ Dobsen模型，土壤水分反演 :param f:微波频率，单位GHz :param T:温度，摄氏度 :param bd:土壤容重 :param vsand:沙土含量，范围0-1 :param vclay:黏土含量，范围0-1 :param soil_dielectric_array:土壤介电常数 :param x0 :土壤水分寻优，初始值 :return: True or False """ alpha = 0.65 sd = 2.65 dcs = (1.01 + 0.44 * sd) ** 2 - 0.062 dc0 = 0.008854 dcw0 = 88.045 - 0.4147 * T + 6.295e-4 * (T ** 2) + 1.075e-5 * (T ** 3) tpt = 0.11109 - 3.824e-3 * T + 6.938e-5 * (T ** 2) - 5.096e-7 * (T ** 3) dcwinf = 4.9 if f >= 1.4: sigma = -1.645 + 1.939 * bd - 2.013e-2 * vsand + 1.594e-2 * vclay else: sigma = 0.0467 + 0.2204 * bd - 4.111e-3 * vsand + 6.614e-3 * vclay dcwr = dcwinf + ((dcw0 - dcwinf) / (1 + (tpt * f) ** 2)) betar = 1.2748 - 0.00519 * vsand - 0.00152 * vclay betai = 1.33797 - 0.00603 * vsand - 0.00166 * vclay fun_mois = lambda vwc: np.abs(np.sqrt( ((1.0 + (bd / sd) * ((dcs ** alpha) - 1.0) + (vwc ** betar) * (dcwr ** alpha) - vwc) ** (1 / alpha)) ** 2 + ((vwc ** (betai / alpha)) * ( (tpt * f * (dcw0 - dcwinf)) / (1 + (tpt * f) ** 2) + sigma * (1.0 - (bd / sd)) / ( 8 * math.atan(1.0) * dc0 * f * vwc))) ** 2 ) - soil_dielectric) # 计算土壤水分 sar1 = self.imageHandler.get_data(sar1_path) angle1 = self.imageHandler.get_data(angle1_path) sar2 = self.imageHandler.get_data(sar2_path) angle2 = self.imageHandler.get_data(angle2_path) mask = self.imageHandler.get_data(mask_path) sin_angle1 = np.sin(angle1) ** 2 cos_angle1 = np.cos(angle1) sin_angle2 = np.sin(angle2) ** 2 cos_angle2 = np.cos(angle2) tmp = (sar2 / sar1) ** 0.5 w = np.where(mask > 0) sin_angle1 = sin_angle1[w] cos_angle1 = cos_angle1[w] sin_angle2 = sin_angle2[w] cos_angle2 = cos_angle2[w] tmp = tmp[w] num = len(w[0]) mois1, mois2, fun_value = np.zeros(num), np.zeros(num), np.zeros(num) if self.__polar_name == 'HH': fun = lambda x: np.abs( np.abs((c2 - (x[1] - s2) ** 0.5) / (c2 + (x[1] - s2) ** 0.5)) - t * np.abs((c1 - (x[0] - s1) ** 0.5) / (c1 + (x[0] - s1) ** 0.5)) ) elif self.__polar_name == 'VV': fun = lambda x: np.abs( np.abs((x[1] - 1) * (s2 - x[1] * (1 + s2)) / (x[1] * c2 + (x[1] - s2) ** 0.5) ** 2) - t * np.abs((x[0] - 1) * (s1 - x[0] * (1 + s1)) / (x[0] * c1 + (x[0] - s1) ** 0.5) ** 2) ) x_mid = 0.5 * (x_max - x_min) bnds = ((x_min, x_max), (x_min, x_max)) start = datetime.datetime.now() bar_list = [0, int(0.1 * num), int(0.2 * num), int(0.3 * num), int(0.4 * num), int(0.5 * num), int(0.6 * num), int(0.7 * num), int(0.8 * num), int(0.9 * num), num - 1] for s1, c1, s2, c2, t, n in zip(sin_angle1, cos_angle1, sin_angle2, cos_angle2, tmp, range(num)): # 计算介电常数 res = minimize(fun, (x_mid, x_mid), method='SLSQP', bounds=bnds) # print(s1, c1, s2, c2, t, n) # print("res::", res) if res.fun < 0.000001: # 计算时相1的土壤水分 soil_dielectric = res.x[0] mois1[n] = fmin(fun_mois, x0, disp=0) # Dobson模型 # 计算时相2的土壤水分 soil_dielectric = res.x[1] mois2[n] = fmin(fun_mois, x0, disp=0) # Dobson模型 if n in bar_list: logger.info('block:{},cal soil_moisture proggress bar:{}%,use time :{}'.format(block_num, round(n / num * 100), (datetime.datetime.now() - start))) sar1[:] = 0 sar2[:] = 0 sar1[w] = mois1 # 土壤水分 sar2[w] = mois2 # 土壤水分 # 保存土壤水分 out_image = Image.fromarray(sar1) out_image.save(moisure1_path) out_image = Image.fromarray(sar2) out_image.save(moisure2_path) ''' f: 频率，通常以赫兹（Hz）为单位，它可能与土壤中电磁波的传播有关。 T: 温度，通常以摄氏度（°C）为单位，它影响土壤水分的介电常数。 er: 测量的土壤介电常数，这是一个无量纲的量，它表示土壤对电磁波的响应。 bd: 土壤的干容重（Bulk Density），通常以克/立方厘米（g/cm³）为单位，它表示单位体积土壤中固体的质量。 vsand: 砂土体积比例，这是一个介于0和1之间的无量纲量，表示土壤中砂土的体积占比。 vclay: 黏土体积比例，这也是一个介于0和1之间的无量纲量，表示土壤中黏土的体积占比。 x0: 初始猜测值，用于数值优化算法的起始点，通常是一个介于0和1之间的量，表示体积含水量（Volume Water Content）的初始估计。 ''' def dobsen_inverse(self, f, T, er, bd, vsand, vclay, x0): # 设置模型参数 alpha = 0.65 sd = 2.65 dcs = (1.01 + 0.44 * sd) ** 2 - 0.062 dc0 = 0.008854 dcw0 = 88.045 - 0.4147 * T + 6.295e-4 * T ** 2 + 1.075e-5 * T ** 3 tpt = 0.11109 - 3.824e-3 * T + 6.938e-5 * T ** 2 - 5.096e-7 * T ** 3 dcwinf = 4.9 # 计算土壤的电导率 sigma if f >= 1.4: sigma = -1.645 + 1.939 * bd - 0.0225622 * vsand + 0.01594 * vclay else: sigma = 0.0467 + 0.2204 * bd - 0.004111 * vsand + 0.006614 * vclay # 计算水分相关土壤颗粒的直径 dcwr dcwr = dcwinf + ((dcw0 - dcwinf) / (1 + (tpt * f) ** 2)) # 计算与土壤水分保持能力相关的参数 betar 和 betai betar = 1.2748 - 0.00519 * vsand - 0.00152 * vclay betai = 1.33797 - 0.00603 * vsand - 0.00166 * vclay # 定义目标函数 def objective(vwc): return np.abs(np.sqrt( ((1.0 + (bd / sd) * ((dcs ** alpha) - 1.0) + (vwc ** betar) * (dcwr ** alpha) - vwc) ** ( 1 / alpha)) ** 2 + ((vwc ** (betai / alpha)) * ( (tpt * f * (dcw0 - dcwinf)) / (1 + (tpt * f) ** 2) + sigma * (1.0 - (bd / sd)) / (8 * math.atan(1.0) * dc0 * f * vwc))) ** 2 ) - er) # 使用数值优化方法找到最佳的 vwc 值 result = minimize(objective, x0, method='Nelder-Mead') # 返回最优解 return result.x[0] def process_handle(self, start): """ 算法主处理函数 :return: True or False """ # 计算ROI区域 bare_land_mask_path = self.create_roi() radar_center_frequency = MetaDataHandler.get_RadarCenterFrequency(self.__processing_paras[self.__sar_names_list[0] + '_Origin_META']) file.copyfile2dir(bare_land_mask_path, self.__workspace_preprocessed_path) logger.info('progress bar: 50%') for name in self.__sar_names_list: file_key = name + '_' + self.__polar_name if os.path.exists(self.__preprocessed_paras[file_key]): lee_path = os.path.join(self.__workspace_preprocessed_path, os.path.basename(self.__preprocessed_paras[file_key]).split("-")[0] + '_lee.tif') Filter().lee_process_sar(self.__preprocessed_paras[file_key], lee_path, 3, 0.25) logger.info('lee process finish: ' + self.__preprocessed_paras[file_key]) os.remove(self.__preprocessed_paras[file_key]) self.__preprocessed_paras.update({file_key: lee_path}) # 分块 bp = BlockProcess() block_size = bp.get_block_size(self.__rows, self.__cols) bp.cut(self.__workspace_preprocessed_path, self.__workspace_block_tif_path, ['tif', 'tiff'], 'tif', block_size) logger.info('blocking tifs success!') img_dir, img_name = bp.get_file_names(self.__workspace_block_tif_path, ['tif']) dir_dict = bp.get_same_img(img_dir, img_name) for key in dir_dict.keys(): tmp = key if self.__sar_names_list[0]+'_LocalIncidenceAngle' in tmp: angle1_list = dir_dict[key] elif self.__sar_names_list[0].split('-')[0]+'_lee' in tmp: sar1_list = dir_dict[key] elif self.__sar_names_list[1] + '_LocalIncidenceAngle' in tmp: angle2_list = dir_dict[key] elif self.__sar_names_list[1].split('-')[0]+'_lee' in tmp: sar2_list = dir_dict[key] elif 'bare_land_mask'in tmp : bare_land_mask_list = dir_dict[key] if not len(angle1_list) == len(sar1_list) == len(angle2_list) == len(sar2_list) == len(bare_land_mask_list): raise Exception('[len(angle1_list) == len(sar1_list) == len(angle2_list) == len(sar2_list) == len(bare_land_mask_list] is False!') # 开启多进程处理 processes_num = min([len(angle1_list), multiprocessing_num]) pool = multiprocessing.Pool(processes=processes_num) for i in range(len(angle1_list)): name = os.path.basename(angle1_list[i]) suffix = '_' + name.split('_')[-4] + '_' + name.split('_')[-3] + '_' + name.split('_')[-2] + '_' + \ name.split('_')[-1] # 计算土壤介电常数 moisture1_path = self.__workspace_block_tif_processed_path+self.__sar_names_list[0]+'_moisture' + suffix moisture2_path = self.__workspace_block_tif_processed_path+self.__sar_names_list[1]+'_moisture' + suffix sar1_path = sar1_list[i] angle1_path = angle1_list[i] sar2_path = sar2_list[i] angle2_path = angle2_list[i] mask_path = bare_land_mask_list[i] x_min = self.__processing_paras['SoilCDCMin'] x_max = self.__processing_paras['SoilCDCMax'] logger.info('total:%s, block:%s cal soil_moisture success!', len(angle1_list), i) pool.apply_async(self.cal_bare_soil_moisure, (moisture1_path, moisture2_path, sar1_path, angle1_path, sar2_path, angle2_path, mask_path, x_min, x_max, radar_center_frequency, self.__processing_paras['T'], self.__processing_paras['Pb'], self.__processing_paras['vsand'], self.__processing_paras['vclay'], i)) pool.close() pool.join() logger.info('progress bar: 80%') # 合并处理后的影像 data_dir = self.__workspace_block_tif_processed_path w = self.__cols h = self.__rows out_path = self.__workspace_processing_path[0:-1] tif_type = np.float32 bp.combine(data_dir, w, h, out_path, file_type=['tif'], datetype=tif_type) # 添加地理信息 soil_moisture1_path = self.__workspace_processing_path + self.__sar_names_list[0]+'_moisture'+'.tif' soil_moisture2_path = self.__workspace_processing_path + self.__sar_names_list[1]+'_moisture'+'.tif' bp.assign_spatial_reference_byfile(self.__preprocessed_paras[self.__sar_names_list[0]+'_'+self.__polar_name], soil_moisture1_path) bp.assign_spatial_reference_byfile(self.__preprocessed_paras[self.__sar_names_list[0]+'_'+self.__polar_name], soil_moisture2_path) # 生成roi区域 product1_path = self.__product_dic + self.__sar_names_list[0] + '_SoilMoistureProduct.tif' roi.cal_roi(product1_path, soil_moisture1_path, bare_land_mask_path, background_value=np.nan) product2_path = self.__product_dic + self.__sar_names_list[1] + '_SoilMoistureProduct.tif' roi.cal_roi(product2_path, soil_moisture2_path, bare_land_mask_path, background_value=np.nan) logger.info('cal soil_moisture success!') for path in [product1_path, product2_path]: proj, geos, data = self.imageHandler.read_img(path) data[data < soil_moisture_value_min] = soil_moisture_value_min data[data > soil_moisture_value_max] = soil_moisture_value_max self.imageHandler.write_img(path, proj, geos, data) # 生成快视图 self.imageHandler.write_quick_view(product1_path) self.imageHandler.write_quick_view(product2_path) # 生成元文件案例 xml_path = "./model_meta.xml" tem_folder = self.__workspace_path + EXE_NAME + r"\Temporary""\\" image_path = product1_path out_path1 = os.path.join(tem_folder, "trans_geo_projcs.tif") out_path2 = os.path.join(tem_folder, "trans_projcs_geo.tif") par_dict = CreateDict(image_path, out_path1, out_path2).calu_nature(start) model_xml_path = os.path.join(tem_folder, "creat_standard.meta.xml") # 输出xml路径 id_min = 0 id_max = 1000 threshold_of_ndvi_min = 0 threshold_of_ndvi_max = 1 set_threshold = [id_max, id_min, threshold_of_ndvi_min, threshold_of_ndvi_max] CreateStadardXmlFile(xml_path, self.alg_xml_path, par_dict, set_threshold, model_xml_path).create_standard_xml() SrcImagePath = self.__input_paras["SingleTempSAR"]['ParaValue'] paths = SrcImagePath.split(';') SrcImageName=os.path.split(paths[0])[1].split('.tar.gz')[0] if len(paths) >= 2: for i in range(1, len(paths)): SrcImageName=SrcImageName+";"+os.path.split(paths[i])[1].split('.tar.gz')[0] meta_xml_path = self.__product_dic+EXE_NAME+"Product.meta.xml" CreateMetafile(self.__processing_paras['META'], self.alg_xml_path, model_xml_path, meta_xml_path).process(SrcImageName) # 文件夹打包 file.make_targz(self.__out_para, self.__product_dic) logger.info('process_handle success!') logger.info('progress bar: 100%') if __name__ == '__main__': multiprocessing.freeze_support() start = datetime.datetime.now() try: if len(sys.argv) < 2: xml_path = 'SoilMoisture_geo.xml' else: xml_path = sys.argv[1] main_handler = MoistureMain(xml_path) if main_handler.check_source() is False: raise Exception('check_source() failed!') if main_handler.preprocess_handle() is False: raise Exception('preprocess_handle() failed!') if main_handler.process_handle(start) is False: raise Exception('process_handle() failed!') logger.info('successful production of ' + EXE_NAME + ' products!') except Exception: logger.exception('run-time error!') finally: main_handler.del_temp_workspace() end = datetime.datetime.now() msg = 'running use time: %s ' % (end - start) logger.info(msg)