microproduct/soilMoisture_geo_sar/SoilMoistureMain.py

# -*- coding: UTF-8 -*-
"""
@Project ：microproduct 
@File ：SoilMoistureMain.py
@Author ：SHJ
@Contact：
@Date ：2021/7/26 17:11 
@Version ：1.0.0
"""
import logging
import os
import datetime
import sys
import numpy as np
from PIL import Image
import multiprocessing
from scipy.optimize import minimize, fmin
from tool.algorithm.algtools.PreProcess import PreProcess as pp
from tool.algorithm.image.ImageHandle import ImageHandler
from tool.algorithm.xml.AlgXmlHandle import ManageAlgXML, CheckSource,InitPara
from tool.algorithm.algtools.logHandler import LogHandler
from tool.algorithm.block.blockprocess import BlockProcess
from tool.config.ConfigeHandle import Config as cf
from tool.algorithm.xml.CreatMetafile import CreateMetafile
from tool.algorithm.algtools.ROIAlg import ROIAlg as roi
from SoilMoistureXmlInfo import CreateDict, CreateStadardXmlFile
from tool.file.fileHandle import fileHandle
from tool.algorithm.algtools.MetaDataHandler import MetaDataHandler
from tool.algorithm.algtools.filter.lee_Filter import Filter
import math

multiprocessing_num = int(cf.get('multiprocessing_num'))
if cf.get('debug') == 'True':
    DEBUG = True
else:
    DEBUG = False
EXE_NAME = cf.get('exe_name')
FILTER_SIZE = int(cf.get('filter_size'))
soil_moisture_value_min = float(cf.get('product_value_min'))
soil_moisture_value_max = float(cf.get('product_value_max'))

LogHandler.init_log_handler('run_log\\' + EXE_NAME)
logger = logging.getLogger("mylog")
env_str = os.path.split(os.path.realpath(__file__))[0]
os.environ['PROJ_LIB'] = env_str
file =fileHandle(DEBUG)

class MoistureMain:
    """
    土壤水分处理主函数
    """
    def __init__(self, alg_xml_path):
        self.alg_xml_path = alg_xml_path
        self.imageHandler = ImageHandler()
        self.__alg_xml_handler = ManageAlgXML(alg_xml_path)
        self.__check_handler = CheckSource(self.__alg_xml_handler)
        self.__workspace_path = None
        self.__input_paras = {}
        self.__out_para = None
        self.__processing_paras = {}
        self.__preprocessed_paras = {}
        # 参考影像路径
        self.__ref_img_path = ''
        # 宽/列数,高/行数
        self.__cols, self.__rows = 0, 0
        # 坐标系
        self.__proj = ''
        # 影像投影变换矩阵
        self.__geo = [0, 0, 0, 0, 0, 0]

        # 极化影像的名称
        self.__polar_name = None
        # 多时相影像名称
        self.__sar_names_list = []

    def check_source(self):
        """
        检查算法相关的配置文件，图像，辅助文件是否齐全
        """
        env_str = os.getcwd()
        logger.info("sysdir: %s", env_str)

        self.__check_handler.check_alg_xml()

        self.__check_handler.check_run_env()

        self.__input_paras = self.__alg_xml_handler.get_input_paras()
        if self.__check_handler.check_input_paras(self.__input_paras) is False:
            return False

        self.__workspace_path = self.__alg_xml_handler.get_workspace_path()
        self.__create_work_space()

        self.__out_para = os.path.join(self.__workspace_path, EXE_NAME, 'Output', r"SoilMoistureProduct.tar.gz")
        self.__alg_xml_handler.write_out_para("SoilMoistureProduct", self.__out_para)  #写入输出参数

        Polarization = self.__input_paras['Polarization']['ParaValue']
        if Polarization == "HH":
            self.__polar_name = "HH"
        elif Polarization == "VV":
            self.__polar_name = "VV"
        elif Polarization == 'empty' or Polarization == 'Empty' or Polarization == 'EMPTY':
            self.__polar_name = 'empty'
        else:
            raise ValueError("input Para 'Polarization' is not 'HH'、'VV'or 'empty'!")

        self.__processing_paras = InitPara.init_processing_paras(self.__input_paras, self.__workspace_preprocessed_path)
        self.__processing_paras.update(InitPara(DEBUG).get_mult_tar_gz_infs(self.__processing_paras, self.__workspace_preprocessing_path))

        self.__sar_names_list = self.__processing_paras['name_list']
        if len(self.__sar_names_list) != 2:
            raise ValueError('number of SingleTempSAR(input para) != 2 !')
        logger.info('check_source success!')
        logger.info('progress bar: 10%')
        return True

    def __create_work_space(self):
        """
        删除原有工作区文件夹,创建新工作区文件夹
        """
        self.__workspace_preprocessing_path = self.__workspace_path + EXE_NAME + r'\Temporary\preprocessing''\\'
        self.__workspace_preprocessed_path = self.__workspace_path + EXE_NAME + r'\Temporary\preprocessed''\\'
        self.__workspace_processing_path = self.__workspace_path + EXE_NAME + r'\Temporary\processing''\\'
        self.__workspace_block_tif_path = self.__workspace_path + EXE_NAME + r'\Temporary\blockTif''\\'
        self.__workspace_block_tif_processed_path = self.__workspace_path + EXE_NAME + r'\Temporary\blockTifProcessed''\\'
        self.__product_dic = self.__workspace_processing_path + 'product\\'
        path_list = [self.__workspace_preprocessing_path, self.__workspace_preprocessed_path,
                     self.__workspace_processing_path, self.__workspace_block_tif_path,
                     self.__workspace_block_tif_processed_path, self.__product_dic]

        file.creat_dirs(path_list)
        logger.info('create new workspace success!')

    def del_temp_workspace(self):
        """
        临时工作区
        """
        if DEBUG is True:
            return
        path = self.__workspace_path + EXE_NAME + r'\Temporary'
        if os.path.exists(path):
            file.del_folder(path)

    def preprocess_handle(self):
        """
        预处理
        """
        # 读取每一张图像,检查图像坐标系

        para_names = ['Covering', 'NDVI']
        ref_img_name =''
        for name in self.__sar_names_list:
            para_names.append(name+'_'+self.__polar_name)      # todo 名称根据LT04实际命名进行缩短
            if ref_img_name == '':
                ref_img_name = name+'_'+self.__polar_name
            para_names.append(name+'_LocalIncidenceAngle')

        p = pp()
        self.__preprocessed_paras  = p.preprocessing(para_names, ref_img_name,self.__processing_paras,
                                                     self.__workspace_preprocessing_path, self.__workspace_preprocessed_path)
        self.__ref_img_path, self.__cols, self.__rows, self.__proj, self.__geo = p.get_ref_inf(self.__preprocessed_paras[ref_img_name])
        self.__ref_img_name = ref_img_name
        logger.info('progress bar: 40%')
        logger.info('preprocess_handle success!')

    def create_roi(self):
        """
        计算ROI掩膜
        :return: 掩膜路径
        """

        names = [self.__sar_names_list[0] + '_' +'LocalIncidenceAngle', self.__ref_img_name, 'Covering', 'NDVI']
        r = roi()
        bare_land_mask_path = r.roi_process(names, self.__workspace_processing_path + "/roi/", self.__processing_paras, self.__preprocessed_paras)

        logger.info('create masks success!')

        # 计算roi区域
        for name in self.__sar_names_list:
            # roi区域掩盖无效区域
            roi.cal_roi(self.__preprocessed_paras[name+'_'+'LocalIncidenceAngle'],self.__preprocessed_paras[name+'_'+'LocalIncidenceAngle'],
                        bare_land_mask_path, background_value=1)

        logger.info('create ROI image success!')
        return bare_land_mask_path


    def cal_bare_soil_moisure(self,moisure1_path,moisure2_path, sar1_path, angle1_path,sar2_path, angle2_path ,mask_path,x_min,x_max,f, T, bd, vsand, vclay,block_num=0,x0=0.3):
        """
         Dobsen模型，土壤水分反演
        :param f:微波频率，单位GHz
        :param T:温度，摄氏度
        :param bd:土壤容重
        :param vsand:沙土含量，范围0-1
        :param vclay:黏土含量，范围0-1
        :param soil_dielectric_array:土壤介电常数
        :param x0 :土壤水分寻优，初始值
        :return: True or False
        """
        alpha = 0.65
        sd = 2.65
        dcs = (1.01 + 0.44 * sd) ** 2 - 0.062
        dc0 = 0.008854
        dcw0 = 88.045 - 0.4147 * T + 6.295e-4 * (T ** 2) + 1.075e-5 * (T ** 3)
        tpt = 0.11109 - 3.824e-3 * T + 6.938e-5 * (T ** 2) - 5.096e-7 * (T ** 3)
        dcwinf = 4.9
        if f >= 1.4:
            sigma = -1.645 + 1.939 * bd - 2.013e-2 * vsand + 1.594e-2 * vclay
        else:
            sigma = 0.0467 + 0.2204 * bd - 4.111e-3 * vsand + 6.614e-3 * vclay

        dcwr = dcwinf + ((dcw0 - dcwinf) / (1 + (tpt * f) ** 2))

        betar = 1.2748 - 0.00519 * vsand - 0.00152 * vclay
        betai = 1.33797 - 0.00603 * vsand - 0.00166 * vclay

        fun_mois = lambda vwc: np.abs(np.sqrt(
            ((1.0 + (bd / sd) * ((dcs ** alpha) - 1.0) + (vwc ** betar) * (dcwr ** alpha) - vwc) ** (1 / alpha)) ** 2 +
            ((vwc ** (betai / alpha)) * (
                        (tpt * f * (dcw0 - dcwinf)) / (1 + (tpt * f) ** 2) + sigma * (1.0 - (bd / sd)) / (
                            8 * math.atan(1.0) * dc0 * f * vwc))) ** 2
        ) - soil_dielectric)

        # 计算土壤水分
        sar1 = self.imageHandler.get_data(sar1_path)
        angle1 = self.imageHandler.get_data(angle1_path)
        sar2 = self.imageHandler.get_data(sar2_path)
        angle2 = self.imageHandler.get_data(angle2_path)
        mask = self.imageHandler.get_data(mask_path)

        sin_angle1 = np.sin(angle1) ** 2
        cos_angle1 = np.cos(angle1)
        sin_angle2 = np.sin(angle2) ** 2
        cos_angle2 = np.cos(angle2)
        tmp = (sar2 / sar1) ** 0.5

        w = np.where(mask > 0)
        sin_angle1 = sin_angle1[w]
        cos_angle1 = cos_angle1[w]
        sin_angle2 = sin_angle2[w]
        cos_angle2 = cos_angle2[w]
        tmp = tmp[w]
        num = len(w[0])

        mois1, mois2, fun_value = np.zeros(num), np.zeros(num), np.zeros(num)

        if self.__polar_name == 'HH':
            fun = lambda x: np.abs(
                np.abs((c2 - (x[1] - s2) ** 0.5) / (c2 + (x[1] - s2) ** 0.5))
                - t * np.abs((c1 - (x[0] - s1) ** 0.5) / (c1 + (x[0] - s1) ** 0.5))
            )

        elif self.__polar_name == 'VV':
            fun = lambda x: np.abs(
                np.abs((x[1] - 1) * (s2 - x[1] * (1 + s2)) / (x[1] * c2 + (x[1] - s2) ** 0.5) ** 2)
                - t * np.abs((x[0] - 1) * (s1 - x[0] * (1 + s1)) / (x[0] * c1 + (x[0] - s1) ** 0.5) ** 2)
            )

        x_mid = 0.5 * (x_max - x_min)
        bnds = ((x_min, x_max), (x_min, x_max))
        start = datetime.datetime.now()

        bar_list = [0, int(0.1 * num), int(0.2 * num), int(0.3 * num), int(0.4 * num), int(0.5 * num), int(0.6 * num),
                    int(0.7 * num), int(0.8 * num), int(0.9 * num), num - 1]

        for s1, c1, s2, c2, t, n in zip(sin_angle1, cos_angle1, sin_angle2, cos_angle2, tmp, range(num)):
            # 计算介电常数
            res = minimize(fun, (x_mid, x_mid), method='SLSQP', bounds=bnds)
            # print(s1, c1, s2, c2, t, n)
            # print("res::", res)
            if res.fun < 0.000001:
                # 计算时相1的土壤水分
                soil_dielectric = res.x[0]
                mois1[n] = fmin(fun_mois, x0, disp=0) # Dobson模型
                # 计算时相2的土壤水分
                soil_dielectric = res.x[1]
                mois2[n] = fmin(fun_mois, x0, disp=0) # Dobson模型
            if n in bar_list:
                logger.info('block:{},cal soil_moisture proggress bar:{}%,use time :{}'.format(block_num, round(n / num * 100), (datetime.datetime.now() - start)))
        sar1[:] = 0
        sar2[:] = 0
        sar1[w] = mois1 # 土壤水分
        sar2[w] = mois2 # 土壤水分

        # 保存土壤水分
        out_image = Image.fromarray(sar1)
        out_image.save(moisure1_path)

        out_image = Image.fromarray(sar2)
        out_image.save(moisure2_path)


    '''
    f: 频率，通常以赫兹（Hz）为单位，它可能与土壤中电磁波的传播有关。
    T: 温度，通常以摄氏度（°C）为单位，它影响土壤水分的介电常数。
    er: 测量的土壤介电常数，这是一个无量纲的量，它表示土壤对电磁波的响应。
    bd: 土壤的干容重（Bulk Density），通常以克/立方厘米（g/cm³）为单位，它表示单位体积土壤中固体的质量。
    vsand: 砂土体积比例，这是一个介于0和1之间的无量纲量，表示土壤中砂土的体积占比。
    vclay: 黏土体积比例，这也是一个介于0和1之间的无量纲量，表示土壤中黏土的体积占比。
    x0: 初始猜测值，用于数值优化算法的起始点，通常是一个介于0和1之间的量，表示体积含水量（Volume Water Content）的初始估计。
    '''
    def dobsen_inverse(self, f, T, er, bd, vsand, vclay, x0):
        # 设置模型参数
        alpha = 0.65
        sd = 2.65
        dcs = (1.01 + 0.44 * sd) ** 2 - 0.062
        dc0 = 0.008854
        dcw0 = 88.045 - 0.4147 * T + 6.295e-4 * T ** 2 + 1.075e-5 * T ** 3
        tpt = 0.11109 - 3.824e-3 * T + 6.938e-5 * T ** 2 - 5.096e-7 * T ** 3
        dcwinf = 4.9

        # 计算土壤的电导率 sigma
        if f >= 1.4:
            sigma = -1.645 + 1.939 * bd - 0.0225622 * vsand + 0.01594 * vclay
        else:
            sigma = 0.0467 + 0.2204 * bd - 0.004111 * vsand + 0.006614 * vclay

        # 计算水分相关土壤颗粒的直径 dcwr
        dcwr = dcwinf + ((dcw0 - dcwinf) / (1 + (tpt * f) ** 2))

        # 计算与土壤水分保持能力相关的参数 betar 和 betai
        betar = 1.2748 - 0.00519 * vsand - 0.00152 * vclay
        betai = 1.33797 - 0.00603 * vsand - 0.00166 * vclay

        # 定义目标函数
        def objective(vwc):
            return np.abs(np.sqrt(
                ((1.0 + (bd / sd) * ((dcs ** alpha) - 1.0) + (vwc ** betar) * (dcwr ** alpha) - vwc) ** (
                            1 / alpha)) ** 2 +
                ((vwc ** (betai / alpha)) * (
                        (tpt * f * (dcw0 - dcwinf)) / (1 + (tpt * f) ** 2) + sigma * (1.0 - (bd / sd)) /
                        (8 * math.atan(1.0) * dc0 * f * vwc))) ** 2
            ) - er)

        # 使用数值优化方法找到最佳的 vwc 值
        result = minimize(objective, x0, method='Nelder-Mead')

        # 返回最优解
        return result.x[0]

    def process_handle(self, start):
        """
        算法主处理函数
        :return: True or False
        """
        # 计算ROI区域
        bare_land_mask_path = self.create_roi()
        radar_center_frequency = MetaDataHandler.get_RadarCenterFrequency(self.__processing_paras[self.__sar_names_list[0] + '_Origin_META'])
        file.copyfile2dir(bare_land_mask_path, self.__workspace_preprocessed_path)
        logger.info('progress bar: 50%')

        for name in self.__sar_names_list:
            file_key = name + '_' + self.__polar_name
            if os.path.exists(self.__preprocessed_paras[file_key]):
                lee_path = os.path.join(self.__workspace_preprocessed_path,
                                        os.path.basename(self.__preprocessed_paras[file_key]).split("-")[0] + '_lee.tif')
                Filter().lee_process_sar(self.__preprocessed_paras[file_key], lee_path, 3, 0.25)
                logger.info('lee process finish: ' + self.__preprocessed_paras[file_key])
                os.remove(self.__preprocessed_paras[file_key])
                self.__preprocessed_paras.update({file_key: lee_path})

        # 分块
        bp = BlockProcess()
        block_size = bp.get_block_size(self.__rows, self.__cols)
        bp.cut(self.__workspace_preprocessed_path, self.__workspace_block_tif_path, ['tif', 'tiff'], 'tif', block_size)
        logger.info('blocking tifs success!')

        img_dir, img_name = bp.get_file_names(self.__workspace_block_tif_path, ['tif'])
        dir_dict = bp.get_same_img(img_dir, img_name)

        for key in dir_dict.keys():
            tmp = key
            if self.__sar_names_list[0]+'_LocalIncidenceAngle' in tmp:
                angle1_list = dir_dict[key]
            elif self.__sar_names_list[0].split('-')[0]+'_lee' in tmp:
                sar1_list = dir_dict[key]
            elif self.__sar_names_list[1] + '_LocalIncidenceAngle' in tmp:
                angle2_list = dir_dict[key]
            elif self.__sar_names_list[1].split('-')[0]+'_lee' in tmp:
                sar2_list = dir_dict[key]
            elif 'bare_land_mask'in tmp :
                bare_land_mask_list = dir_dict[key]

        if not len(angle1_list) == len(sar1_list) == len(angle2_list) == len(sar2_list) == len(bare_land_mask_list):
            raise Exception('[len(angle1_list) == len(sar1_list) == len(angle2_list) == len(sar2_list) == len(bare_land_mask_list] is False!')


        # 开启多进程处理
        processes_num = min([len(angle1_list), multiprocessing_num])

        pool = multiprocessing.Pool(processes=processes_num)
        for i in range(len(angle1_list)):
            name = os.path.basename(angle1_list[i])
            suffix = '_' + name.split('_')[-4] + '_' + name.split('_')[-3] + '_' + name.split('_')[-2] + '_' + \
                     name.split('_')[-1]

            # 计算土壤介电常数
            moisture1_path = self.__workspace_block_tif_processed_path+self.__sar_names_list[0]+'_moisture' + suffix
            moisture2_path = self.__workspace_block_tif_processed_path+self.__sar_names_list[1]+'_moisture' + suffix
            sar1_path = sar1_list[i]
            angle1_path = angle1_list[i]
            sar2_path = sar2_list[i]
            angle2_path = angle2_list[i]
            mask_path = bare_land_mask_list[i]
            x_min = self.__processing_paras['SoilCDCMin']
            x_max = self.__processing_paras['SoilCDCMax']
            logger.info('total:%s, block:%s cal soil_moisture success!', len(angle1_list), i)
            pool.apply_async(self.cal_bare_soil_moisure, (moisture1_path, moisture2_path, sar1_path, angle1_path, sar2_path, angle2_path, mask_path, x_min, x_max, radar_center_frequency, self.__processing_paras['T'],
                                         self.__processing_paras['Pb'], self.__processing_paras['vsand'], self.__processing_paras['vclay'], i))

        pool.close()
        pool.join()
        logger.info('progress bar: 80%')
        # 合并处理后的影像
        data_dir = self.__workspace_block_tif_processed_path
        w = self.__cols
        h = self.__rows
        out_path = self.__workspace_processing_path[0:-1]
        tif_type = np.float32
        bp.combine(data_dir, w, h, out_path, file_type=['tif'], datetype=tif_type)

        # 添加地理信息
        soil_moisture1_path = self.__workspace_processing_path + self.__sar_names_list[0]+'_moisture'+'.tif'
        soil_moisture2_path = self.__workspace_processing_path + self.__sar_names_list[1]+'_moisture'+'.tif'
        bp.assign_spatial_reference_byfile(self.__preprocessed_paras[self.__sar_names_list[0]+'_'+self.__polar_name], soil_moisture1_path)
        bp.assign_spatial_reference_byfile(self.__preprocessed_paras[self.__sar_names_list[0]+'_'+self.__polar_name], soil_moisture2_path)

        # 生成roi区域
        product1_path = self.__product_dic + self.__sar_names_list[0] + '_SoilMoistureProduct.tif'
        roi.cal_roi(product1_path, soil_moisture1_path, bare_land_mask_path, background_value=np.nan)
        product2_path = self.__product_dic + self.__sar_names_list[1] + '_SoilMoistureProduct.tif'
        roi.cal_roi(product2_path, soil_moisture2_path, bare_land_mask_path, background_value=np.nan)
        logger.info('cal soil_moisture success!')

        for path in [product1_path, product2_path]:
            proj, geos, data = self.imageHandler.read_img(path)
            data[data < soil_moisture_value_min] = soil_moisture_value_min
            data[data > soil_moisture_value_max] = soil_moisture_value_max
            self.imageHandler.write_img(path, proj, geos, data)

        # 生成快视图
        self.imageHandler.write_quick_view(product1_path)
        self.imageHandler.write_quick_view(product2_path)

        # 生成元文件案例
        xml_path = "./model_meta.xml"
        tem_folder = self.__workspace_path + EXE_NAME + r"\Temporary""\\"
        image_path = product1_path
        out_path1 = os.path.join(tem_folder, "trans_geo_projcs.tif")
        out_path2 = os.path.join(tem_folder, "trans_projcs_geo.tif")
        par_dict = CreateDict(image_path, out_path1, out_path2).calu_nature(start)
        model_xml_path = os.path.join(tem_folder, "creat_standard.meta.xml")  # 输出xml路径
        id_min = 0
        id_max = 1000
        threshold_of_ndvi_min = 0
        threshold_of_ndvi_max = 1
        set_threshold = [id_max, id_min, threshold_of_ndvi_min, threshold_of_ndvi_max]
        CreateStadardXmlFile(xml_path, self.alg_xml_path, par_dict, set_threshold, model_xml_path).create_standard_xml()


        SrcImagePath = self.__input_paras["SingleTempSAR"]['ParaValue']
        paths = SrcImagePath.split(';')
        SrcImageName=os.path.split(paths[0])[1].split('.tar.gz')[0]
        if len(paths) >= 2:
            for i in range(1, len(paths)):
                SrcImageName=SrcImageName+";"+os.path.split(paths[i])[1].split('.tar.gz')[0]
        meta_xml_path = self.__product_dic+EXE_NAME+"Product.meta.xml"
        CreateMetafile(self.__processing_paras['META'], self.alg_xml_path, model_xml_path, meta_xml_path).process(SrcImageName)

        # 文件夹打包

        file.make_targz(self.__out_para, self.__product_dic)
        logger.info('process_handle success!')
        logger.info('progress bar: 100%')


if __name__ == '__main__':
    multiprocessing.freeze_support()
    start = datetime.datetime.now()
    try:
        if len(sys.argv) < 2:
            xml_path = 'SoilMoisture_geo.xml'
        else:
            xml_path = sys.argv[1]
        main_handler = MoistureMain(xml_path)
        if main_handler.check_source() is False:
            raise Exception('check_source() failed!')
        if main_handler.preprocess_handle() is False:
            raise Exception('preprocess_handle() failed!')
        if main_handler.process_handle(start) is False:
            raise Exception('process_handle() failed!')
        logger.info('successful production of ' + EXE_NAME + ' products!')
    except Exception:
        logger.exception('run-time error!')
    finally:
        main_handler.del_temp_workspace()

    end = datetime.datetime.now()
    msg = 'running use time: %s ' % (end - start)
    logger.info(msg)