KJ135
/
microproduct
mirror of http://172.16.0.12:5000/WBJY/microproduct.git
4
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
microproduct/surfaceRoughness/SurfaceRoughnessMain.py

519 lines
24 KiB
Python
Raw Normal View History

入所测试定稿上传
2023-08-28 10:17:29 +00:00
# -*- coding: UTF-8 -*-
"""
@Project ：microproduct
@File ：SurfaceSalinityMain.py
@Author ：SHJ
@Contact：土壤粗糙度算法主函数
@Date ：2022/11/7 9:04
@Version ：2.0.0
"""
import logging
from tool.algorithm.algtools.PreProcess import PreProcess as pp
from tool.algorithm.image.ImageHandle import ImageHandler
from tool.algorithm.xml.AlgXmlHandle import ManageAlgXML, CheckSource, InitPara
from tool.algorithm.algtools.logHandler import LogHandler
from tool.algorithm.block.blockprocess import BlockProcess
from tool.algorithm.algtools.MetaDataHandler import MetaDataHandler
from tool.algorithm.xml.CreatMetafile import CreateMetafile
from tool.config.ConfigeHandle import Config as cf
from tool.algorithm.algtools.ROIAlg import ROIAlg
from tool.file.fileHandle import fileHandle
import pyproj._compat
from tool.csv.csvHandle import csvHandle
from tool.algorithm.algtools.filter.lee_Filter import Filter
from SurfaceRoughnessALg import RoughnessAlg as alg
from SurfaceRoughnessXmlInfo import CreateDict, CreateStadardXmlFile
import os
import datetime
import numpy as np
import shutil
import scipy #解决打包错误
import scipy.spatial.transform # 用于解决打包错误
import scipy.spatial.transform._rotation_groups # 用于解决打包错误
import scipy.special.cython_special # 用于解决打包错误
import sys
from scipy.interpolate import griddata # 引入scipy中的二维插值库
from scipy.special import lambertw
import math
import multiprocessing
surface_roughness_value_min = float(cf.get('product_value_min'))
surface_roughness_value_max = float(cf.get('product_value_max'))
if cf.get('debug') == 'True':
DEBUG = True
else:
DEBUG = False
file = fileHandle(DEBUG)
EXE_NAME = cf.get('exe_name')
FILTER_SIZE = int(cf.get('filter_size'))
env_str = os.path.split(os.path.realpath(__file__))[0]
os.environ['PROJ_LIB'] = env_str
LogHandler.init_log_handler('run_log\\' + EXE_NAME)
logger = logging.getLogger("mylog")
csvh = csvHandle()
class RoughnessMain:
"""
算法主函数
"""
def __init__(self, alg_xml_path):
self.alg_xml_path = alg_xml_path
self.imageHandler = ImageHandler()
self.__alg_xml_handler = ManageAlgXML(alg_xml_path)
self.__check_handler = CheckSource(self.__alg_xml_handler)
self.__workspace_path, self.__sar_tif_name, self.__out_para = None, None, None
self.__input_paras, self.__output_paras, self.__processing_paras, self.__preprocessed_paras = {}, {}, {}, {}
# 参考影像路径,坐标系
self.__ref_img_path, self.__proj = '', ''
# 宽/列数,高/行数
self.__cols, self.__rows = 0, 0
# 影像投影变换矩阵
self.__geo = [0, 0, 0, 0, 0, 0]
def check_source(self):
"""
检查算法相关的配置文件，图像，辅助文件是否齐全
"""
env_str = os.getcwd()
logger.info("sysdir: %s", env_str)
self.__check_handler.check_alg_xml()
self.__check_handler.check_run_env()
self.__workspace_path = self.__alg_xml_handler.get_workspace_path()
self.__input_paras = self.__alg_xml_handler.get_input_paras()
Polarization = self.__input_paras['Polarization']['ParaValue']
if Polarization == "VV":
self.__sar_tif_name = "VV"
elif Polarization == "HH":
self.__sar_tif_name = "HH"
else:
raise ValueError("input Para 'Polarization' is not 'HH' or 'VV'!")
input_para_names = ["SingleSAR", "MeasuredData", "Covering", "NDVI"]
if self.__check_handler.check_input_paras(input_para_names) is False:
return False
self.__create_work_space()
self.__processing_paras = InitPara.init_processing_paras(self.__input_paras)
self.__processing_paras.update(self.get_tar_gz_inf(self.__processing_paras["sar_path0"]))
self.__out_para = os.path.join(self.__workspace_path, EXE_NAME, 'Output', r"SurfaceRoughnessProduct.tar.gz")
self.__alg_xml_handler.write_out_para("SurfaceRoughnessProduct", self.__out_para) #写入输出参数
logger.info('check_source success!')
logger.info('progress bar: 10%')
return True
def get_tar_gz_inf(self, tar_gz_path):
para_dic = {}
name = os.path.split(tar_gz_path)[1].rstrip('.tar.gz')
file_dir = os.path.join(self.__workspace_preprocessing_path, name + '\\')
file.de_targz(tar_gz_path, file_dir)
# 元文件字典
para_dic.update(InitPara.get_meta_dic(InitPara.get_meta_paths(file_dir, name), name))
# tif路径字典
pol_dic = InitPara.get_polarization_mode(InitPara.get_tif_paths(file_dir, name))
flag_list = [0, 0, 0,0]
VVVH_list = [0, 0]
# 获取极化类型
if "HH" in pol_dic.keys() and self.__sar_tif_name == 'HH':
para_dic.update({self.__sar_tif_name: pol_dic['HH']})
flag_list[0] = 1
if "VV" in pol_dic.keys() and self.__sar_tif_name == 'VV':
para_dic.update({self.__sar_tif_name: pol_dic['VV']})
flag_list[1] = 1
if "HV" in pol_dic.keys():
para_dic.update({'HV': pol_dic['HV']})
flag_list[3] = 1
if "LocalIncidenceAngle" in pol_dic.keys():
para_dic.update({'LocalIncidenceAngle': pol_dic['LocalIncidenceAngle']})
flag_list[2] = 1
if 'VH' in pol_dic.keys(): # 判断VV、VH是否存在，用来计算参数L
para_dic.update({'VH': pol_dic['VH']})
VVVH_list[0] = 1
if 'VV' in pol_dic.keys():
VVVH_list[1] = 1
if self.__sar_tif_name == 'HH': # 只有手动选择的是'HH'，需要把VV路径添加到字典中，
para_dic.update({'VV': pol_dic['VV']})
else: # 反之代表则选择的是'VV'，不需要把VV路径重复添加到字典中
pass
if flag_list != [1, 0, 1, 1] and flag_list != [0, 1, 1, 1]:
raise Exception('There are not ' + self.__sar_tif_name + '、LocalIncidenceAngle or meta.xml in path:', tar_gz_path)
if VVVH_list != [1, 1]:
raise Exception('There are not VH or VV:', tar_gz_path)
return para_dic
def __create_work_space(self):
"""
删除原有工作区文件夹,创建新工作区文件夹
"""
self.__workspace_preprocessing_path = self.__workspace_path + EXE_NAME + r"\Temporary\preprocessing""\\"
self.__workspace_preprocessed_path = self.__workspace_path + EXE_NAME + r"\Temporary\preprocessed""\\"
self.__workspace_processing_path = self.__workspace_path + EXE_NAME + r"\Temporary\processing""\\"
self.__workspace_block_tif_path = self.__workspace_path + EXE_NAME + r"\Temporary\blockTif""\\"
self.__workspace_block_tif_processed_path = self.__workspace_path + EXE_NAME + r"\Temporary\blockTifProcessed""\\"
self.__workspace_Zs_block_tif_processed_path = self.__workspace_path + EXE_NAME + r"\Temporary\ZsblockTifProcessed""\\"
self.__feature_tif_dic = self.__workspace_processing_path + 'feature_tif'
self.__product_dic = self.__workspace_processing_path + 'product\\'
path_list = [self.__workspace_preprocessing_path, self.__workspace_preprocessed_path,
self.__workspace_processing_path, self.__workspace_block_tif_path,
self.__workspace_block_tif_processed_path, self.__workspace_Zs_block_tif_processed_path,
self.__feature_tif_dic, self.__product_dic]
file.creat_dirs(path_list)
logger.info('create new workspace success!')
def del_temp_workspace(self):
"""
临时工作区
"""
if DEBUG is True:
return
path = self.__workspace_path + EXE_NAME + r"\Temporary"
if os.path.exists(path):
file.del_folder(path)
def preprocess_handle(self):
"""
预处理
"""
para_names=[]
# 读取每一张图像,检查图像坐标系
if self.__sar_tif_name == 'HH': # VH与VV参与预处理
para_names = [self.__sar_tif_name, 'VH','HV', 'VV', "Covering", "NDVI", "LocalIncidenceAngle"]
elif self.__sar_tif_name == 'VV':
para_names = [self.__sar_tif_name,'VH','HV','HH' "Covering", "NDVI", "LocalIncidenceAngle"]
# if self.__sar_tif_name == 'HH': # VH与VV参与预处理
# para_names = [self.__sar_tif_name, 'VH', 'VV', "Covering", "NDVI", "LocalIncidenceAngle"]
# elif self.__sar_tif_name == 'VV':
# para_names = [self.__sar_tif_name, 'VH', "Covering", "NDVI", "LocalIncidenceAngle"]
ref_img_name = self.__sar_tif_name
p = pp()
self.__preprocessed_paras = p.preprocessing(para_names, ref_img_name,
self.__processing_paras,
self.__workspace_preprocessing_path,
self.__workspace_preprocessed_path)
self.__ref_img_path, self.__cols, self.__rows, self.__proj, self.__geo = p.get_ref_inf(
self.__preprocessed_paras[ref_img_name])
logger.info('preprocess_handle success!')
logger.info('progress bar: 30%')
def create_roi(self):
"""
计算ROI掩膜
:return: 掩膜路径
"""
names = ['LocalIncidenceAngle', self.__sar_tif_name, 'Covering', 'NDVI']
r = ROIAlg()
bare_land_mask_path = r.roi_process(names, self.__workspace_processing_path + "/roi/", self.__processing_paras, self.__preprocessed_paras)
logger.info('create masks success!')
# 计算roi区域
ROIAlg().cal_roi(self.__preprocessed_paras["LocalIncidenceAngle"], self.__preprocessed_paras["LocalIncidenceAngle"],
bare_land_mask_path, background_value=1)
ROIAlg().cal_roi(self.__preprocessed_paras[self.__sar_tif_name], self.__preprocessed_paras[self.__sar_tif_name],
bare_land_mask_path, background_value=1)
shutil.copyfile(bare_land_mask_path, self.__workspace_preprocessed_path + 'mask.tif')
logger.info('create ROI image success!')
return bare_land_mask_path
def create_soilDLCT_tif(self):
"""
创建土壤介电常数影像
"""
# 读取实测值，从经纬度坐标系转为图像坐标系
measured_data_img = csvh.trans_measuredata(csvh.readcsv(self.__processing_paras['MeasuredData']), self.__ref_img_path)
# 插值法生成土壤介电常数二维影像
grid_x, grid_y = np.mgrid[0:self.__rows:1, 0:self.__cols:1]
points = np.array(measured_data_img)[:,0:2]
values = np.array(measured_data_img)[:,2].T
grid_z0 = griddata(points, values, (grid_x, grid_y), method='nearest')
self.__processing_paras['DLCT'] = os.path.join(self.__workspace_preprocessed_path,'DLCT.tif')
self.imageHandler.write_img(self.__processing_paras['DLCT'],self.__proj,self.__geo,grid_z0)
logger.info('create soilDLCT image success!')
def Calu_Q(self,meta_file_path):
"""计算参数交叉极化比值Q"""
VV_proj, VV_geotrans, VV_arr=self.imageHandler.read_img(self.__preprocessed_paras["VV"])
VH_proj, VH_geotrans, VH_arr = self.imageHandler.read_img(self.__preprocessed_paras["VH"])
Array_Q=VH_arr/VV_arr
Out_Q_path=os.path.join(self.__workspace_preprocessed_path,"CrossPolarRatior_q.tif")
self.imageHandler.write_img(Out_Q_path, VV_proj, VV_geotrans, Array_Q)
self.__preprocessed_paras.update({"Ratior_q": Out_Q_path})
return Out_Q_path
def Calu_H(self,Q_path,angle_path,lamda):
"""
计算几何特征复合因子H
Q_path：交叉极化比值Q数据
angle_path：角度文件
lamda：波长
"""
wave_k=2*math.pi/lamda
array_angle=self.imageHandler.get_data(angle_path) # 弧度
Q_proj, Q_geo, Q_arr = self.imageHandler.read_img(Q_path)
temp_val=np.power((0.13 + np.sin(1.5 * array_angle)),1.4)
temp_A=Q_arr/(0.095*temp_val)
Array_H=np.power((-1*np.log(1-temp_A)/1.3),10/9)/wave_k
Out_H_path=os.path.join(self.__workspace_preprocessed_path, "Factor_H.tif")
self.imageHandler.write_img(Out_H_path, Q_proj, Q_geo, Array_H)
def Calu_L_Zs(self, Zs_path,L_path, H_path, Fr_path, lamda, seita_Path):
"""
计算几何特征复合因子L、Zs
Zs_path：输出几何特征复合因子 L文件
H_path：输入几何特征复合因子 Fr文件
Fr_path：输入的Fr_path
lamda：输入的波长
seita_Path：输入的入射角
"""
H_proj, H_geo, H_arr = self.imageHandler.read_img(H_path)
Fr_proj, Fr_geo, Fr_arr = self.imageHandler.read_img(Fr_path)
array_angle=self.imageHandler.get_data(seita_Path) # 弧度
k=2*math.pi/lamda
a=0.5*H_arr*H_arr
b=-k*np.sin(array_angle)
y=Fr_arr
ans1 = 2 * lambertw(-b * np.sqrt(y / a) / 2) / b # 计算L的两个值
ans2 = 2 * lambertw(b * np.sqrt(y / a) / 2) / b # 计算L的两个值
if "complex" in ans1.dtype.name:
ans1_real=ans1.real
ans2_real = ans2.real
else:
ans1_real=ans1
ans2_real = ans2
chazhi=ans1_real-ans2_real
L=np.where(chazhi>0,ans1_real,ans2_real)
L[np.isnan(L)] = 0 # 必须剔除异常值，否则无法写入到tif中
self.imageHandler.write_img(L_path, Fr_proj, Fr_geo, L)
Zs=np.power(H_arr, 2)/L
Zs[np.isnan(Zs)] = 0 # 必须剔除异常值，否则无法写入到tif中
self.imageHandler.write_img(Zs_path, Fr_proj, Fr_geo, Zs)
def process_handle(self, start):
"""
算法主处理函数
:return: True or False
"""
# 单视复数转DB
meta_file_path = self.__processing_paras['META']
# 获取元文件波长数据,单位转为cm
lamda = MetaDataHandler.get_lamda(meta_file_path) * 100
# 创建土壤介电常数影像
# self.create_soilDLCT_tif()
logger.info('progress bar: 50%')
# 计算roi
bare_land_mask_path = self.create_roi()
# 计算参数交叉极化比值Q、几何特征复合因子H,参考论文2.3 Oh模型
# Out_Q_path=self.Calu_Q(meta_file_path)
# self.Calu_H(Out_Q_path, self.__preprocessed_paras['LocalIncidenceAngle'], self.__processing_paras['lamda'])
# 分块
bp = BlockProcess()
block_size = bp.get_block_size(self.__rows, self.__cols)
bp.cut(self.__workspace_preprocessed_path, self.__workspace_block_tif_path, ['tif', 'tiff'], 'tif', block_size) # 分块裁剪
logger.info('blocking tifs success!')
img_dir, img_name = bp.get_file_names(self.__workspace_block_tif_path, ['tif'])
dir_dict = bp.get_same_img(img_dir, img_name) # 同一类别放在一个字典下
#改2-取分块的影像VV,VH
hh_list, vv_list, vh_list, hv_list, mask_list = None, None, None, None, None
angle_list = None
sar_tif_list = None
dlct_list = None
Factor_H_list = None
for key in dir_dict.keys():
tmp = key.split("_", 2)[0]
if tmp == "LocalIncidenceAngle":
angle_list = dir_dict[key]
elif tmp == 'HH':
hh_list = dir_dict[key]
elif tmp == 'VV':
vv_list = dir_dict[key]
elif tmp == 'VH':
vh_list = dir_dict[key]
elif tmp == 'HV':
hv_list = dir_dict[key]
elif tmp == 'mask':
mask_list = dir_dict[key]
# elif tmp == self.__sar_tif_name:
# sar_tif_list = dir_dict[key]
elif tmp == "DLCT":
dlct_list = dir_dict[key]
elif tmp == "Factor":
Factor_H_list = dir_dict[key]
# if angle_list is None or sar_tif_list is None or dlct_list is None or Factor_H_list is None:
# raise ValueError('angle_list is None or sar_tif_list is None or dlct_list or Factor_H_list is None')
#
# if not len(angle_list) == len(sar_tif_list) == len(dlct_list) == len(dlct_list):
# raise ValueError('[en(angle_list) == len(sar_tif_list) == len(dlct_list)] is False!') #原方法
if hh_list is None or vv_list is None or vh_list is None or hv_list is None or mask_list is None:
logger.info('angle_list is None or hh_list is None or vv_list is None or vh_list is None or hv_list is None or ndwi_list is None')
return False
if not len(hh_list) == len(vv_list) == len(vh_list) == len(hv_list) == len(mask_list):
logger.info('[len(angle_list) == len(hh_list) == len(vv_list) == len(vh_list) == len(hv_list) == len(ndwi_list)] is False!')
return False
#lee滤波
f = Filter()
processes_num = min([len(hh_list), 4])
# f.lee_filter_multiprocess(sar_tif_list, sar_tif_list, FILTER_SIZE, processes_num)
f.lee_filter_multiprocess(hh_list, hh_list, FILTER_SIZE, processes_num)
f.lee_filter_multiprocess(vv_list, vv_list, FILTER_SIZE, processes_num)
f.lee_filter_multiprocess(vh_list, vh_list, FILTER_SIZE, processes_num)
f.lee_filter_multiprocess(hv_list, hv_list, FILTER_SIZE, processes_num)
pool = multiprocessing.Pool(processes=processes_num)
pl = []
# 计算地表粗糙度
for i in range(len(angle_list)):
name = os.path.basename(angle_list[i])
suffix = '_' + name.split('_')[-4] + "_" + name.split('_')[-3] + "_" + name.split('_')[-2] + "_" + \
name.split('_')[-1]
# dielectric_const_factor_path = self.__workspace_processing_path + "dielectric_const_factor.tif"
# alg.cal_dielectric_const_factor(dielectric_const_factor_path, dlct_list[i],
# angle_list[i], self.__sar_tif_name)
surface_roughness_path = self.__workspace_block_tif_processed_path + "surface_roughness" + suffix
pl.append(pool.apply_async(alg.cal_soil_roug, (surface_roughness_path, hh_list[i], vv_list[i], hv_list[i], vh_list[i], angle_list[i], mask_list[i], lamda)))
# alg.cal_soil_roug(surface_roughness_path, hh_list[i], vv_list[i], hv_list[i], vh_list[i], angle_list[i],
# mask_list[i], lamda)
# alg.cal_soil_roughness(surface_roughness_path, sar_tif_list[i],
# dielectric_const_factor_path, angle_list[i],
# self.__processing_paras['lamda'])
# #改5- 用l和h计算出ZS，参考论文2.2 公式7
# H_path=Factor_H_list[i]
# Fr_path=surface_roughness_path
# k=self.__processing_paras['lamda']
# seita_Path=angle_list[i]
#
# Zs_path=self.__workspace_Zs_block_tif_processed_path + "Zs" + suffix
# L_path = self.__workspace_Zs_block_tif_processed_path + "Param_L" + suffix
# self.Calu_L_Zs(Zs_path,L_path, H_path, Fr_path,k,seita_Path)
logger.info('total:%s, block:%s cal surfaceRoughness success!', len(angle_list), i)
pool.close()
pool.join()
# 合并处理后的影像
data_dir = self.__workspace_block_tif_processed_path
w = ImageHandler.get_img_width(self.__preprocessed_paras['HH'])
h = ImageHandler.get_img_height(self.__preprocessed_paras["HH"])
out_path = self.__workspace_processing_path[0:-1]
tif_type = bp.get_tif_dtype(self.__preprocessed_paras['HH'])
bp.combine(data_dir, w, h, out_path, file_type=['tif'], datetype=tif_type)
# 添加地理信息
surface_roughness_path = self.__workspace_processing_path + "surface_roughness.tif"
bp.assign_spatial_reference_byfile(self.__preprocessed_paras['HH'], surface_roughness_path)
logger.info('progress bar: 80%')
# # Zs拼接成一幅tif数据
# Zs_data_dir = self.__workspace_Zs_block_tif_processed_path
# ref_proj, ref_geo, im_arr = ImageHandler.read_img(surface_roughness_path)
# bp.combine_Tif(Zs_data_dir, w, h, out_path, ref_proj, ref_geo, out_type='tif', file_type=['tif', 'tiff'], datetype='float16')
# Zs_path = self.__workspace_processing_path + "Zs.tif"
# L_path = self.__workspace_processing_path + "Param_L.tif"
# Out_H_path = os.path.join(self.__workspace_preprocessed_path, "Factor_H.tif")
#
# # product_Zs_path = os.path.join(self.__product_dic, "Zs.tif")
# product_L_path = os.path.join(self.__product_dic, "L.tif")
# product_H_path = os.path.join(self.__product_dic, "H.tif")
# # alg.cal_roi(product_Zs_path, Zs_path, bare_land_mask_path, background_value=np.nan)
# ROIAlg().cal_roi(product_L_path, L_path, bare_land_mask_path, background_value=np.nan)
# ROIAlg().cal_roi(product_H_path, Out_H_path, bare_land_mask_path, background_value=np.nan)
#生成roi区域
product_mask_path = self.__workspace_processing_path + 'SurfaceRoughnessProductMask.tif'
ROIAlg().cal_roi(product_mask_path, surface_roughness_path, bare_land_mask_path, background_value=np.nan)
product_path = os.path.join(self.__product_dic, 'SurfaceRoughnessProduct.tif')
self.imageHandler.limit_field(product_path, product_mask_path, surface_roughness_value_min, surface_roughness_value_max)
logger.info('cal soil_roughness success!')
# 生成快视图
self.imageHandler.write_quick_view(product_path)
# self.imageHandler.write_quick_view(product_Zs_path) # 新增加三个
# self.imageHandler.write_quick_view(product_L_path)
# self.imageHandler.write_quick_view(product_H_path)
logger.info('packing!')
xml_path = "./model_meta.xml"
tem_folder = self.__workspace_path + EXE_NAME + r"\Temporary""\\"
image_path = product_path
out_path1 = os.path.join(tem_folder, "trans_geo_projcs.tif")
out_path2 = os.path.join(tem_folder, "trans_projcs_geo.tif")
par_dict = CreateDict(image_path, out_path1, out_path2).calu_nature(start)
model_xml_path = os.path.join(tem_folder, "creat_standard.meta.xml") # 输出xml路径
CreateStadardXmlFile(xml_path, self.alg_xml_path, par_dict, model_xml_path).create_standard_xml()
# 文件夹打包
SrcImagePath = self.__input_paras["SingleSAR"]['ParaValue']
paths = SrcImagePath.split(';')
SrcImageName = os.path.split(paths[0])[1].split('.tar.gz')[0]
if len(paths) >= 2:
for i in range(1, len(paths)):
SrcImageName=SrcImageName+";"+os.path.split(paths[i])[1].split('.tar.gz')[0]
meta_xml_path = self.__product_dic+EXE_NAME+"Product.meta.xml"
CreateMetafile(self.__processing_paras['META'], self.alg_xml_path, model_xml_path, meta_xml_path).process(SrcImageName)
file.make_targz(self.__out_para, self.__product_dic)
logger.info('process_handle success!')
logger.info('progress bar: 100%')
if __name__ == '__main__':
multiprocessing.freeze_support() #解决多进程打包成exe后，运行错误
start = datetime.datetime.now()
try:
if len(sys.argv) < 2:
xml_path = 'SurfaceRoughness.xml'
else:
xml_path = sys.argv[1]
main_handler = RoughnessMain(xml_path)
if main_handler.check_source() is False:
raise Exception('check_source() failed!')
if main_handler.preprocess_handle() is False:
raise Exception('preprocess_handle() failed!')
if main_handler.process_handle(start) is False:
raise Exception('process_handle() failed!')
logger.info('successful production of ' + EXE_NAME + ' products!')
except Exception:
logger.exception("run-time error!")
finally:
main_handler.del_temp_workspace()
end = datetime.datetime.now()
msg = 'running use time: %s ' % (end - start)
logger.info(msg)