447 lines
19 KiB
Python
447 lines
19 KiB
Python
# -*- coding: UTF-8 -*-
|
||
"""
|
||
@Project ：microproduct
|
||
@File ：SoilMoistureMain.py
|
||
@Author ：SHJ
|
||
@Contact：
|
||
@Date ：2021/7/26 17:11
|
||
@Version ：1.0.0
|
||
"""
|
||
import logging
|
||
import os
|
||
import datetime
|
||
import sys
|
||
import numpy as np
|
||
from PIL import Image
|
||
import multiprocessing
|
||
from scipy.optimize import minimize, fmin
|
||
from tool.algorithm.algtools.PreProcess import PreProcess as pp
|
||
from tool.algorithm.image.ImageHandle import ImageHandler
|
||
from tool.algorithm.xml.AlgXmlHandle import ManageAlgXML, CheckSource,InitPara
|
||
from tool.algorithm.algtools.logHandler import LogHandler
|
||
from tool.algorithm.block.blockprocess import BlockProcess
|
||
from tool.config.ConfigeHandle import Config as cf
|
||
from tool.algorithm.xml.CreatMetafile import CreateMetafile
|
||
from tool.algorithm.algtools.ROIAlg import ROIAlg as roi
|
||
from SoilMoistureXmlInfo import CreateDict, CreateStadardXmlFile
|
||
from tool.file.fileHandle import fileHandle
|
||
from tool.algorithm.algtools.MetaDataHandler import MetaDataHandler
|
||
from tool.algorithm.algtools.filter.lee_Filter import Filter
|
||
import math
|
||
|
||
multiprocessing_num = int(cf.get('multiprocessing_num'))
|
||
if cf.get('debug') == 'True':
|
||
DEBUG = True
|
||
else:
|
||
DEBUG = False
|
||
EXE_NAME = cf.get('exe_name')
|
||
FILTER_SIZE = int(cf.get('filter_size'))
|
||
soil_moisture_value_min = float(cf.get('product_value_min'))
|
||
soil_moisture_value_max = float(cf.get('product_value_max'))
|
||
|
||
LogHandler.init_log_handler('run_log\\' + EXE_NAME)
|
||
logger = logging.getLogger("mylog")
|
||
env_str = os.path.split(os.path.realpath(__file__))[0]
|
||
os.environ['PROJ_LIB'] = env_str
|
||
file =fileHandle(DEBUG)
|
||
|
||
class MoistureMain:
|
||
"""
|
||
土壤水分处理主函数
|
||
"""
|
||
def __init__(self, alg_xml_path):
|
||
self.alg_xml_path = alg_xml_path
|
||
self.imageHandler = ImageHandler()
|
||
self.__alg_xml_handler = ManageAlgXML(alg_xml_path)
|
||
self.__check_handler = CheckSource(self.__alg_xml_handler)
|
||
self.__workspace_path = None
|
||
self.__input_paras = {}
|
||
self.__out_para = None
|
||
self.__processing_paras = {}
|
||
self.__preprocessed_paras = {}
|
||
# 参考影像路径
|
||
self.__ref_img_path = ''
|
||
# 宽/列数,高/行数
|
||
self.__cols, self.__rows = 0, 0
|
||
# 坐标系
|
||
self.__proj = ''
|
||
# 影像投影变换矩阵
|
||
self.__geo = [0, 0, 0, 0, 0, 0]
|
||
|
||
# 极化影像的名称
|
||
self.__polar_name = None
|
||
# 多时相影像名称
|
||
self.__sar_names_list = []
|
||
|
||
def check_source(self):
|
||
"""
|
||
检查算法相关的配置文件，图像，辅助文件是否齐全
|
||
"""
|
||
env_str = os.getcwd()
|
||
logger.info("sysdir: %s", env_str)
|
||
|
||
self.__check_handler.check_alg_xml()
|
||
|
||
self.__check_handler.check_run_env()
|
||
|
||
self.__input_paras = self.__alg_xml_handler.get_input_paras()
|
||
if self.__check_handler.check_input_paras(self.__input_paras) is False:
|
||
return False
|
||
|
||
self.__workspace_path = self.__alg_xml_handler.get_workspace_path()
|
||
self.__create_work_space()
|
||
|
||
self.__out_para = os.path.join(self.__workspace_path, EXE_NAME, 'Output', r"SoilMoistureProduct.tar.gz")
|
||
self.__alg_xml_handler.write_out_para("SoilMoistureProduct", self.__out_para) #写入输出参数
|
||
|
||
Polarization = self.__input_paras['Polarization']['ParaValue']
|
||
if Polarization == "HH":
|
||
self.__polar_name = "HH"
|
||
elif Polarization == "VV":
|
||
self.__polar_name = "VV"
|
||
elif Polarization == 'empty' or Polarization == 'Empty' or Polarization == 'EMPTY':
|
||
self.__polar_name = 'empty'
|
||
else:
|
||
raise ValueError("input Para 'Polarization' is not 'HH'、'VV'or 'empty'!")
|
||
|
||
self.__processing_paras = InitPara.init_processing_paras(self.__input_paras)
|
||
self.__processing_paras.update(InitPara(DEBUG).get_mult_tar_gz_infs(self.__processing_paras, self.__workspace_preprocessing_path))
|
||
|
||
self.__sar_names_list = self.__processing_paras['name_list']
|
||
if len(self.__sar_names_list) != 2:
|
||
raise ValueError('number of SingleTempSAR(input para) != 2 !')
|
||
logger.info('check_source success!')
|
||
logger.info('progress bar: 10%')
|
||
return True
|
||
|
||
def __create_work_space(self):
|
||
"""
|
||
删除原有工作区文件夹,创建新工作区文件夹
|
||
"""
|
||
self.__workspace_preprocessing_path = self.__workspace_path + EXE_NAME + r'\Temporary\preprocessing''\\'
|
||
self.__workspace_preprocessed_path = self.__workspace_path + EXE_NAME + r'\Temporary\preprocessed''\\'
|
||
self.__workspace_processing_path = self.__workspace_path + EXE_NAME + r'\Temporary\processing''\\'
|
||
self.__workspace_block_tif_path = self.__workspace_path + EXE_NAME + r'\Temporary\blockTif''\\'
|
||
self.__workspace_block_tif_processed_path = self.__workspace_path + EXE_NAME + r'\Temporary\blockTifProcessed''\\'
|
||
self.__product_dic = self.__workspace_processing_path + 'product\\'
|
||
path_list = [self.__workspace_preprocessing_path, self.__workspace_preprocessed_path,
|
||
self.__workspace_processing_path, self.__workspace_block_tif_path,
|
||
self.__workspace_block_tif_processed_path, self.__product_dic]
|
||
|
||
file.creat_dirs(path_list)
|
||
logger.info('create new workspace success!')
|
||
|
||
def del_temp_workspace(self):
|
||
"""
|
||
临时工作区
|
||
"""
|
||
if DEBUG is True:
|
||
return
|
||
path = self.__workspace_path + EXE_NAME + r'\Temporary'
|
||
if os.path.exists(path):
|
||
file.del_folder(path)
|
||
|
||
def preprocess_handle(self):
|
||
"""
|
||
预处理
|
||
"""
|
||
# 读取每一张图像,检查图像坐标系
|
||
|
||
para_names = ['Covering', 'NDVI']
|
||
ref_img_name =''
|
||
for name in self.__sar_names_list:
|
||
para_names.append(name+'_'+self.__polar_name)
|
||
if ref_img_name == '':
|
||
ref_img_name = name+'_'+self.__polar_name
|
||
para_names.append(name+'_LocalIncidenceAngle')
|
||
|
||
p = pp()
|
||
self.__preprocessed_paras = p.preprocessing(para_names, ref_img_name,self.__processing_paras,
|
||
self.__workspace_preprocessing_path, self.__workspace_preprocessed_path)
|
||
self.__ref_img_path, self.__cols, self.__rows, self.__proj, self.__geo = p.get_ref_inf(self.__preprocessed_paras[ref_img_name])
|
||
self.__ref_img_name = ref_img_name
|
||
logger.info('progress bar: 40%')
|
||
logger.info('preprocess_handle success!')
|
||
|
||
def create_roi(self):
|
||
"""
|
||
计算ROI掩膜
|
||
:return: 掩膜路径
|
||
"""
|
||
|
||
names = [self.__sar_names_list[0] + '_' +'LocalIncidenceAngle', self.__ref_img_name, 'Covering', 'NDVI']
|
||
r = roi()
|
||
bare_land_mask_path = r.roi_process(names, self.__workspace_processing_path + "/roi/", self.__processing_paras, self.__preprocessed_paras)
|
||
|
||
logger.info('create masks success!')
|
||
|
||
# 计算roi区域
|
||
for name in self.__sar_names_list:
|
||
# roi区域掩盖无效区域
|
||
roi.cal_roi(self.__preprocessed_paras[name+'_'+'LocalIncidenceAngle'],self.__preprocessed_paras[name+'_'+'LocalIncidenceAngle'],
|
||
bare_land_mask_path, background_value=1)
|
||
|
||
logger.info('create ROI image success!')
|
||
return bare_land_mask_path
|
||
|
||
|
||
def cal_bare_soil_moisure(self,moisure1_path,moisure2_path, sar1_path, angle1_path,sar2_path, angle2_path ,mask_path,x_min,x_max,f, T, bd, vsand, vclay,block_num=0,x0=0.3):
|
||
"""
|
||
Dobsen模型，土壤水分反演
|
||
:param f:微波频率，单位GHz
|
||
:param T:温度，摄氏度
|
||
:param bd:土壤容重
|
||
:param vsand:沙土含量，范围0-1
|
||
:param vclay:黏土含量，范围0-1
|
||
:param soil_dielectric_array:土壤介电常数
|
||
:param x0 :土壤水分寻优，初始值
|
||
:return: True or False
|
||
"""
|
||
alpha = 0.65
|
||
sd = 2.65
|
||
dcs = (1.01 + 0.44 * sd) ** 2 - 0.062
|
||
dc0 = 0.008854
|
||
dcw0 = 88.045 - 0.4147 * T + 6.295e-4 * (T ** 2) + 1.075e-5 * (T ** 3)
|
||
tpt = 0.11109 - 3.824e-3 * T + 6.938e-5 * (T ** 2) - 5.096e-7 * (T ** 3)
|
||
dcwinf = 4.9
|
||
if f >= 1.4:
|
||
sigma = -1.645 + 1.939 * bd - 2.013e-2 * vsand + 1.594e-2 * vclay
|
||
else:
|
||
sigma = 0.0467 + 0.2204 * bd - 4.111e-3 * vsand + 6.614e-3 * vclay
|
||
|
||
dcwr = dcwinf + ((dcw0 - dcwinf) / (1 + (tpt * f) ** 2))
|
||
|
||
betar = 1.2748 - 0.00519 * vsand - 0.00152 * vclay
|
||
betai = 1.33797 - 0.00603 * vsand - 0.00166 * vclay
|
||
|
||
fun_mois = lambda vwc: np.abs(np.sqrt(
|
||
((1.0 + (bd / sd) * ((dcs ** alpha) - 1.0) + (vwc ** betar) * (dcwr ** alpha) - vwc) ** (1 / alpha)) ** 2 +
|
||
((vwc ** (betai / alpha)) * (
|
||
(tpt * f * (dcw0 - dcwinf)) / (1 + (tpt * f) ** 2) + sigma * (1.0 - (bd / sd)) / (
|
||
8 * math.atan(1.0) * dc0 * f * vwc))) ** 2
|
||
) - soil_dielectric)
|
||
|
||
# 计算土壤水分
|
||
sar1 = self.imageHandler.get_data(sar1_path)
|
||
angle1 = self.imageHandler.get_data(angle1_path)
|
||
sar2 = self.imageHandler.get_data(sar2_path)
|
||
angle2 = self.imageHandler.get_data(angle2_path)
|
||
mask = self.imageHandler.get_data(mask_path)
|
||
|
||
sin_angle1 = np.sin(angle1) ** 2
|
||
cos_angle1 = np.cos(angle1)
|
||
sin_angle2 = np.sin(angle2) ** 2
|
||
cos_angle2 = np.cos(angle2)
|
||
tmp = (sar2 / sar1) ** 0.5
|
||
|
||
w = np.where(mask > 0)
|
||
sin_angle1 = sin_angle1[w]
|
||
cos_angle1 = cos_angle1[w]
|
||
sin_angle2 = sin_angle2[w]
|
||
cos_angle2 = cos_angle2[w]
|
||
tmp = tmp[w]
|
||
num = len(w[0])
|
||
|
||
mois1, mois2, fun_value = np.zeros(num), np.zeros(num), np.zeros(num)
|
||
|
||
if self.__polar_name == 'HH':
|
||
fun = lambda x: np.abs(
|
||
np.abs((c2 - (x[1] - s2) ** 0.5) / (c2 + (x[1] - s2) ** 0.5))
|
||
- t * np.abs((c1 - (x[0] - s1) ** 0.5) / (c1 + (x[0] - s1) ** 0.5))
|
||
)
|
||
|
||
elif self.__polar_name == 'VV':
|
||
fun = lambda x: np.abs(
|
||
np.abs((x[1] - 1) * (s2 - x[1] * (1 + s2)) / (x[1] * c2 + (x[1] - s2) ** 0.5) ** 2)
|
||
- t * np.abs((x[0] - 1) * (s1 - x[0] * (1 + s1)) / (x[0] * c1 + (x[0] - s1) ** 0.5) ** 2)
|
||
)
|
||
|
||
x_mid = 0.5 * (x_max - x_min)
|
||
bnds = ((x_min, x_max), (x_min, x_max))
|
||
start = datetime.datetime.now()
|
||
|
||
bar_list = [0, int(0.1 * num), int(0.2 * num), int(0.3 * num), int(0.4 * num), int(0.5 * num), int(0.6 * num),
|
||
int(0.7 * num), int(0.8 * num), int(0.9 * num), num - 1]
|
||
|
||
for s1, c1, s2, c2, t, n in zip(sin_angle1, cos_angle1, sin_angle2, cos_angle2, tmp, range(num)):
|
||
# 计算介电常数
|
||
res = minimize(fun, (x_mid, x_mid), method='SLSQP', bounds=bnds)
|
||
# print(s1, c1, s2, c2, t, n)
|
||
# print("res::", res)
|
||
if res.fun < 0.000001:
|
||
# 计算时相1的土壤水分
|
||
soil_dielectric = res.x[0]
|
||
mois1[n] = fmin(fun_mois, x0, disp=0) # Dobson模型
|
||
# 计算时相2的土壤水分
|
||
soil_dielectric = res.x[1]
|
||
mois2[n] = fmin(fun_mois, x0, disp=0) # Dobson模型
|
||
if n in bar_list:
|
||
logger.info('block:{},cal soil_moisture proggress bar:{}%,use time :{}'.format(block_num, round(n / num * 100), (datetime.datetime.now() - start)))
|
||
|
||
sar1[:] = 0
|
||
sar2[:] = 0
|
||
sar1[w] = mois1 # 土壤水分
|
||
sar2[w] = mois2 # 土壤水分
|
||
|
||
# 保存土壤水分
|
||
out_image = Image.fromarray(sar1)
|
||
out_image.save(moisure1_path)
|
||
|
||
out_image = Image.fromarray(sar2)
|
||
out_image.save(moisure2_path)
|
||
|
||
def process_handle(self, start):
|
||
"""
|
||
算法主处理函数
|
||
:return: True or False
|
||
"""
|
||
# 计算ROI区域
|
||
bare_land_mask_path = self.create_roi()
|
||
radar_center_frequency = MetaDataHandler.get_RadarCenterFrequency(self.__processing_paras['META'])
|
||
file.copyfile2dir(bare_land_mask_path, self.__workspace_preprocessed_path)
|
||
logger.info('progress bar: 50%')
|
||
|
||
# 分块
|
||
bp = BlockProcess()
|
||
block_size = bp.get_block_size(self.__rows, self.__cols)
|
||
bp.cut(self.__workspace_preprocessed_path, self.__workspace_block_tif_path, ['tif', 'tiff'], 'tif', block_size)
|
||
logger.info('blocking tifs success!')
|
||
|
||
img_dir, img_name = bp.get_file_names(self.__workspace_block_tif_path, ['tif'])
|
||
dir_dict = bp.get_same_img(img_dir, img_name)
|
||
|
||
for key in dir_dict.keys():
|
||
tmp = key
|
||
if self.__sar_names_list[0]+'_LocalIncidenceAngle' in tmp:
|
||
angle1_list = dir_dict[key]
|
||
elif self.__sar_names_list[0]+'_'+self.__polar_name in tmp:
|
||
sar1_list = dir_dict[key]
|
||
elif self.__sar_names_list[1] + '_LocalIncidenceAngle' in tmp:
|
||
angle2_list = dir_dict[key]
|
||
elif self.__sar_names_list[1] + '_' + self.__polar_name in tmp:
|
||
sar2_list = dir_dict[key]
|
||
elif 'bare_land_mask'in tmp :
|
||
bare_land_mask_list = dir_dict[key]
|
||
|
||
if not len(angle1_list) == len(sar1_list) == len(angle2_list) == len(sar2_list) == len(bare_land_mask_list):
|
||
raise Exception('[len(angle1_list) == len(sar1_list) == len(angle2_list) == len(sar2_list) == len(bare_land_mask_list] is False!')
|
||
|
||
|
||
# 开启多进程处理
|
||
processes_num = min([len(angle1_list), multiprocessing_num])
|
||
if DEBUG == False:
|
||
f = Filter()
|
||
f.lee_filter_multiprocess(sar1_list, sar1_list, FILTER_SIZE, processes_num)
|
||
f.lee_filter_multiprocess(sar2_list, sar2_list, FILTER_SIZE, processes_num)
|
||
|
||
pool = multiprocessing.Pool(processes=processes_num)
|
||
for i in range(len(angle1_list)):
|
||
name = os.path.basename(angle1_list[i])
|
||
suffix = '_' + name.split('_')[-4] + '_' + name.split('_')[-3] + '_' + name.split('_')[-2] + '_' + \
|
||
name.split('_')[-1]
|
||
|
||
# 计算土壤介电常数
|
||
moisture1_path = self.__workspace_block_tif_processed_path+self.__sar_names_list[0]+'_moisture' + suffix
|
||
moisture2_path = self.__workspace_block_tif_processed_path+self.__sar_names_list[1]+'_moisture' + suffix
|
||
sar1_path = sar1_list[i]
|
||
angle1_path = angle1_list[i]
|
||
sar2_path = sar2_list[i]
|
||
angle2_path = angle2_list[i]
|
||
mask_path = bare_land_mask_list[i]
|
||
x_min = self.__processing_paras['SoilCDCMin']
|
||
x_max = self.__processing_paras['SoilCDCMax']
|
||
logger.info('total:%s, block:%s cal soil_moisture success!', len(angle1_list), i)
|
||
pool.apply_async(self.cal_bare_soil_moisure, (moisture1_path, moisture2_path, sar1_path, angle1_path, sar2_path, angle2_path, mask_path, x_min, x_max, radar_center_frequency, self.__processing_paras['T'],
|
||
self.__processing_paras['Pb'], self.__processing_paras['vsand'], self.__processing_paras['vclay'], i))
|
||
|
||
pool.close()
|
||
pool.join()
|
||
logger.info('progress bar: 80%')
|
||
# 合并处理后的影像
|
||
data_dir = self.__workspace_block_tif_processed_path
|
||
w = self.__cols
|
||
h = self.__rows
|
||
out_path = self.__workspace_processing_path[0:-1]
|
||
tif_type = np.float32
|
||
bp.combine(data_dir, w, h, out_path, file_type=['tif'], datetype=tif_type)
|
||
|
||
# 添加地理信息
|
||
soil_moisture1_path = self.__workspace_processing_path + self.__sar_names_list[0]+'_moisture'+'.tif'
|
||
soil_moisture2_path = self.__workspace_processing_path + self.__sar_names_list[1]+'_moisture'+'.tif'
|
||
bp.assign_spatial_reference_byfile(self.__preprocessed_paras[self.__sar_names_list[0]+'_'+self.__polar_name], soil_moisture1_path)
|
||
bp.assign_spatial_reference_byfile(self.__preprocessed_paras[self.__sar_names_list[0]+'_'+self.__polar_name], soil_moisture2_path)
|
||
|
||
# 生成roi区域
|
||
product1_path = self.__product_dic + self.__sar_names_list[0] + '_SoilMoistureProduct.tif'
|
||
roi.cal_roi(product1_path, soil_moisture1_path, bare_land_mask_path, background_value=np.nan)
|
||
product2_path = self.__product_dic + self.__sar_names_list[1] + '_SoilMoistureProduct.tif'
|
||
roi.cal_roi(product2_path, soil_moisture2_path, bare_land_mask_path, background_value=np.nan)
|
||
logger.info('cal soil_moisture success!')
|
||
|
||
for path in [product1_path, product2_path]:
|
||
proj, geos, data = self.imageHandler.read_img(path)
|
||
data[data < soil_moisture_value_min] = soil_moisture_value_min
|
||
data[data > soil_moisture_value_max] = soil_moisture_value_max
|
||
self.imageHandler.write_img(path, proj, geos, data)
|
||
|
||
# 生成快视图
|
||
self.imageHandler.write_quick_view(product1_path)
|
||
self.imageHandler.write_quick_view(product2_path)
|
||
|
||
# 生成元文件案例
|
||
xml_path = "./model_meta.xml"
|
||
tem_folder = self.__workspace_path + EXE_NAME + r"\Temporary""\\"
|
||
image_path = product1_path
|
||
out_path1 = os.path.join(tem_folder, "trans_geo_projcs.tif")
|
||
out_path2 = os.path.join(tem_folder, "trans_projcs_geo.tif")
|
||
par_dict = CreateDict(image_path, out_path1, out_path2).calu_nature(start)
|
||
model_xml_path = os.path.join(tem_folder, "creat_standard.meta.xml") # 输出xml路径
|
||
id_min = 0
|
||
id_max = 1000
|
||
threshold_of_ndvi_min = 0
|
||
threshold_of_ndvi_max = 1
|
||
set_threshold = [id_max, id_min, threshold_of_ndvi_min, threshold_of_ndvi_max]
|
||
CreateStadardXmlFile(xml_path, self.alg_xml_path, par_dict, set_threshold, model_xml_path).create_standard_xml()
|
||
|
||
|
||
SrcImagePath = self.__input_paras["SingleTempSAR"]['ParaValue']
|
||
paths = SrcImagePath.split(';')
|
||
SrcImageName=os.path.split(paths[0])[1].split('.tar.gz')[0]
|
||
if len(paths) >= 2:
|
||
for i in range(1, len(paths)):
|
||
SrcImageName=SrcImageName+";"+os.path.split(paths[i])[1].split('.tar.gz')[0]
|
||
meta_xml_path = self.__product_dic+EXE_NAME+"Product.meta.xml"
|
||
CreateMetafile(self.__processing_paras['META'], self.alg_xml_path, model_xml_path, meta_xml_path).process(SrcImageName)
|
||
|
||
# 文件夹打包
|
||
|
||
file.make_targz(self.__out_para, self.__product_dic)
|
||
logger.info('process_handle success!')
|
||
logger.info('progress bar: 100%')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
multiprocessing.freeze_support()
|
||
start = datetime.datetime.now()
|
||
try:
|
||
if len(sys.argv) < 2:
|
||
xml_path = 'SoilMoisture_geo.xml'
|
||
else:
|
||
xml_path = sys.argv[1]
|
||
main_handler = MoistureMain(xml_path)
|
||
if main_handler.check_source() is False:
|
||
raise Exception('check_source() failed!')
|
||
if main_handler.preprocess_handle() is False:
|
||
raise Exception('preprocess_handle() failed!')
|
||
if main_handler.process_handle(start) is False:
|
||
raise Exception('process_handle() failed!')
|
||
logger.info('successful production of ' + EXE_NAME + ' products!')
|
||
except Exception:
|
||
logger.exception('run-time error!')
|
||
finally:
|
||
main_handler.del_temp_workspace()
|
||
|
||
end = datetime.datetime.now()
|
||
msg = 'running use time: %s ' % (end - start)
|
||
logger.info(msg)
|