# -*- coding: UTF-8 -*- """ @Project ：microproduct @File ：LandCoverMain.py @Function ：实现对数据地表覆盖的训练和分类识别 @Author ：SHJ @Date ：2021/11/15 @Version ：1.0.0 """ import datetime import logging import os import shutil import sys import numpy as np import cv2 import pyproj._compat # 导入PreProcess模块要在其他包含gdal库的模块前面，不然剪切影像会报错，详见https://blog.csdn.net/u014656611/article/details/106450006 from tool.algorithm.algtools.PreProcess import PreProcess as pp from tool.algorithm.image.ImageHandle import ImageHandler from LandCoverAuxData import LandCoverMeasCsv from tool.algorithm.xml.AlgXmlHandle import ManageAlgXML, CheckSource, InitPara from tool.algorithm.algtools.logHandler import LogHandler from tool.algorithm.xml.CreatMetafile import CreateMetafile from LandCoverXmlInfo import CreateDict, CreateStadardXmlFile from tool.config.ConfigeHandle import Config as cf from tool.csv.csvHandle import csvHandle from tool.file.fileHandle import fileHandle from tool.algorithm.ml.machineLearning import MachineLeaning as ml from tool.algorithm.algtools.ROIAlg import ROIAlg as roi import multiprocessing csvh = csvHandle() if cf.get('debug') == 'True': FILE_DEBUG = True else: FILE_DEBUG = False EXE_NAME = cf.get('exe_name') LogHandler.init_log_handler('run_log\\' + EXE_NAME) logger = logging.getLogger("mylog") FILTER_SIZE = int(cf.get('filter_size')) file =fileHandle(FILE_DEBUG) MAX_TRAN_NUM = int(cf.get('max_tran__num_per_class')) env_str = os.path.split(os.path.realpath(__file__))[0] os.environ['PROJ_LIB'] = env_str DEBUG = False class LandCoverMain: """ 算法主函数 """ def __init__(self, alg_xml_path): self.alg_xml_path = alg_xml_path self.imageHandler = ImageHandler() self.__alg_xml_handler = ManageAlgXML(alg_xml_path) self.__check_handler = CheckSource(self.__alg_xml_handler) self.__workspace_path = None self.__out_para = None self.__input_paras, self.__processing_paras, self.__preprocessed_paras = {}, {}, {} # 多时相影像名称 self.__HHHV_names_list, self.__VVVH_names_list, self.__HH_names_list, self.__VV_names_list = [], [],[],[] # 保存特征数据的名称 self.__feature_name_list = [] # xml参数字典 self.__parameters_dic = {} # 参考影像路径 self.__ref_img_path = '' # 宽/列数,高/行数 self.__cols, self.__rows = 0, 0 # 坐标系 self.__proj = '' # 影像投影变换矩阵 self.__geo = [0, 0, 0, 0, 0, 0] self.__polar = 1 def check_source(self): """ 检查算法相关的配置文件，图像，辅助文件是否齐全 """ env_str = os.getcwd() logger.info("sysdir: %s", env_str) self.__check_handler.check_alg_xml() self.__check_handler.check_run_env() # 检查景影像是否为全极化 self.__input_paras = self.__alg_xml_handler.get_input_paras() checkFlag, self.__parameters_dic = self.__check_handler.check_input_paras(self.__input_paras) self.__workspace_path = self.__alg_xml_handler.get_workspace_path() self.__create_work_space() self.__processing_paras = InitPara.init_processing_paras(self.__input_paras) self.__processing_paras.update(InitPara(DEBUG).get_mult_tar_gz_infs(self.__processing_paras, self.__workspace_preprocessing_path)) self.__polar = self.__processing_paras['Polarization'] self.image_meta_xml = self.__processing_paras['META'] for name in self.__processing_paras['name_list']: flag_list = self.__processing_paras[name + '_pola'] if self.__polar == 1: if flag_list[0] == 1: self.__HH_names_list.append(name) elif flag_list[3] == 1: self.__VV_names_list.append(name) elif self.__polar == 2: if flag_list[0] == 1 and flag_list[1] == 1: self.__HHHV_names_list.append(name) elif flag_list[2] == 1 and flag_list[3] == 1: self.__VVVH_names_list.append(name) self.__feature_name_list = [key for key in self.__processing_paras.keys() if ('FeatureMap' in key)] self.__out_para = os.path.join(self.__workspace_path, EXE_NAME, 'Output', r"LandCoverProduct.tar.gz") self.__alg_xml_handler.write_out_para("LandCoverProductDir", self.__out_para) # 写入输出参数 logger.info('check_source success!') return True def __create_work_space(self): """ 删除原有工作区文件夹,创建新工作区文件夹 """ self.__workspace_preprocessing_path = self.__workspace_path + EXE_NAME + r"\Temporary\preprocessing""\\" self.__workspace_preprocessed_path = self.__workspace_path + EXE_NAME + r"\Temporary\preprocessed""\\" self.__workspace_processing_path = self.__workspace_path + EXE_NAME + r"\Temporary\processing""\\" self.__workspace_block_tif_path = self.__workspace_path + EXE_NAME + r"\Temporary\blockTif""\\" self.__workspace_block_tif_processed_path = self.__workspace_path + EXE_NAME + r"\Temporary\blockTifProcessed""\\" self.__feature_tif_dir = self.__workspace_processing_path + 'feature_tif' self.__product_dir = self.__workspace_processing_path + 'product\\' path_list = [self.__workspace_preprocessing_path, self.__workspace_preprocessed_path, self.__workspace_processing_path, self.__workspace_block_tif_path, self.__workspace_block_tif_processed_path, self.__feature_tif_dir,self.__product_dir] file.creat_dirs(path_list) logger.info('create new workspace success!') def del_temp_workspace(self): """ 临时工作区 """ path = self.__workspace_path + EXE_NAME + r"\Temporary" if os.path.exists(path): file.del_folder(path) def get_name_tif_list(self): para_names = [] for tif_name in self.__HHHV_names_list: para_names.append(tif_name + '_HH') para_names.append(tif_name + '_HV') for tif_name in self.__VVVH_names_list: para_names.append(tif_name + '_VH') para_names.append(tif_name + '_VV') for tif_name in self.__HH_names_list: para_names.append(tif_name + '_HH') for tif_name in self.__VV_names_list: para_names.append(tif_name + '_VV') return para_names def preprocess_handle(self): """ 预处理 """ # 读取每一张图像,检查图像坐标系 para_names = self.get_name_tif_list() ref_img_name = para_names[0] para_names = para_names + self.__feature_name_list p = pp() self.__preprocessed_paras = p.preprocessing(para_names, ref_img_name, self.__processing_paras, self.__workspace_preprocessing_path, self.__workspace_preprocessed_path) self.__ref_img_path, self.__cols, self.__rows, self.__proj, self.__geo = p.get_ref_inf(self.__preprocessed_paras[ref_img_name]) logger.info('progress bar: 40%') logger.info('preprocess_handle success!') def create_roi(self): """ 计算ROI掩膜 :return:掩膜路径 """ para_names = self.get_name_tif_list() tif_mask_path = roi().roi_process(para_names, self.__workspace_processing_path + "/roi/", self.__processing_paras, self.__preprocessed_paras) logger.info('create ROI image success!') return tif_mask_path def create_features(self): """ 使用每一景全极化影像生成极化特征影像 :para name_list: 每一景影像的路径 """ for fea_name in self.__feature_name_list: path = self.__preprocessed_paras[fea_name] feature_path = os.path.join(self.__feature_tif_dir, os.path.basename(path)) shutil.copyfile(path, feature_path) feature_array = ImageHandler.get_band_array(feature_path) feature_array = ml.standardization(feature_array) self.imageHandler.write_img(feature_path, self.__proj, self.__geo, feature_array) if self.__polar == 2: self.create_double_polar_features() elif self.__polar == 1: self.create_single_polar_features() def create_single_polar_features(self): name_list = [] + self.__HH_names_list + self.__VV_names_list for name in name_list: if name in self.__HH_names_list: tif_name = name + '_HH' else: tif_name = name + '_VV' path = self.__preprocessed_paras[tif_name] array_single_pol = ImageHandler.get_band_array(path) # 后向散射无效值设置为nan array_single_pol[np.where(array_single_pol < -100.0)] = np.nan array_single_pol = ml.standardization(array_single_pol) feature_path = os.path.join(self.__feature_tif_dir, os.path.basename(path)) self.imageHandler.write_img(feature_path, self.__proj, self.__geo, array_single_pol) def create_double_polar_features(self): name_list = [] + self.__HHHV_names_list + self.__VVVH_names_list # # 生成极化组合特征 for name in name_list: if name in self.__HHHV_names_list: array_single_pol = ImageHandler.get_band_array(self.__preprocessed_paras[name+'_HH']) array_cross_pol = ImageHandler.get_band_array(self.__preprocessed_paras[name+'_HV']) else: array_single_pol = ImageHandler.get_band_array(self.__preprocessed_paras[name + '_VV']) array_cross_pol = ImageHandler.get_band_array(self.__preprocessed_paras[name + '_VH']) # 后向散射无效值设置为nan array_single_pol[np.where(array_single_pol < -100.0)] = np.nan array_cross_pol[np.where(array_cross_pol < -100.0)] = np.nan decomposeList = self.__parameters_dic["FeatureCombination"].split(',') if len(decomposeList) == 1: decomposeList = self.__parameters_dic["FeatureCombination"].split(';') for n in range(8): if str(n) in decomposeList: feature_path = os.path.join(self.__feature_tif_dir, name + '_feature'+str(n)+'.tif') if n == 0: array = array_single_pol elif n == 1: array = array_cross_pol elif n == 2: array = array_single_pol + array_cross_pol elif n == 3: array = array_single_pol - array_cross_pol elif n == 4: array = (array_single_pol - array_cross_pol)/(array_single_pol + array_cross_pol) elif n == 5: array = array_single_pol/array_cross_pol elif n == 6: array = array_cross_pol + array_single_pol/array_cross_pol elif n == 7: array = (array_single_pol**2 + array_cross_pol**2)/(array_single_pol**2 - array_cross_pol**2) else: continue array = ml.standardization(array) self.imageHandler.write_img(feature_path, self.__proj, self.__geo, array) logging.info("feature0 = array_single_pol, feature1= array_cross_pol") logging.info("feature1 = array_cross_pol") logging.info("feature2 = array_single_pol + array_cross_pol") logging.info("feature3 = array_single_pol - array_cross_pol") logging.info("feature4 = (array_single_pol - array_cross_pol)/(array_single_pol + array_cross_pol)") logging.info("feature5 = array_single_pol/array_cross_pol") logging.info("feature6 = array_cross_pol + array_single_pol/array_cross_pol") logging.info("feature7 = (array_single_pol**2 + array_cross_pol**2)/(array_single_pol**2 - array_cross_pol**2)") def process_handle(self, start): """ 算法主处理函数 """ # 读取实测值，获取多边形区域内所有的点，分为训练集数据和测试集数据 pm = LandCoverMeasCsv(self.__processing_paras['LabelData'], self.__ref_img_path,MAX_TRAN_NUM) train_data_list = pm.api_read_measure() train_data_dic = csvh.trans_landCover_list2dic(train_data_list) csvh_roi = csvHandle(self.__rows, self.__cols) label_img = csvh_roi.get_roi_img() if (len(label_img) != 0): self.imageHandler.write_img(self.__workspace_processing_path + "label_img.tif", "", [0, 0, 0, 0, 0, 0], label_img) # csvh_roi = csvHandle(self.__rows, self.__cols) # train_data_dic = csvh_roi.trans_landCover_measuredata_dic( # csvh_roi.readcsv(self.__processing_paras['LabelData']), self.__preprocessed_paras['ori_sim'], MAX_TRAN_NUM) # label_img = csvh_roi.get_roi_img() # if (len(label_img) != 0): # self.imageHandler.write_img(self.__workspace_processing_path + "label_img.tif", "", [0, 0, 0, 0, 0, 0], # label_img) type_id_name, type_id_parent = pm.class_list() self.create_meta_file(self.__ref_img_path, type_id_name, type_id_parent) logger.info("read csv data success!") logger.info('progress bar: 45%') roi_img = self.imageHandler.get_band_array(self.create_roi()) # out_tif_path = self.ref_lee_filter(name_list) self.create_features() logger.info("create_features success!") logger.info('progress bar: 50%') # 生成最优特征子集训练集 X_train, Y_train, optimal_feature = ml.gene_optimal_train_set(train_data_dic, self.__feature_tif_dir) # RF clf = ml.trainRF(X_train, Y_train) logger.info('progress bar: 60%') # 生成测试集 X_test_path_list = ml.gene_test_set(self.__feature_tif_dir, optimal_feature) # 预测 logger.info('testing') cover_path = ml.predict(clf, X_test_path_list, EXE_NAME, self.__workspace_processing_path, self.__rows, self.__cols) logger.info('test success!') logger.info('progress bar: 95%') # 转换数据 proj, geo, cover_data = self.imageHandler.read_img(cover_path) # 形态学（闭运算）去roi区域噪点 cover_data = np.uint8(cover_data) kernel = np.ones((5, 5), np.uint8) cover_data = cv2.erode(cv2.dilate(cover_data, kernel), kernel) for id, class_id in zip(train_data_dic['ids'], train_data_dic['class_ids']): cover_data[np.where(cover_data == id)] = class_id cover_data = np.int16(cover_data) # 获取影像roi区域 cover_data = cover_data * roi_img product_dir = self.__workspace_processing_path + 'product\\' product_path = os.path.join(product_dir, 'LandCoverProduct.tif') self.imageHandler.write_img(product_path, self.__proj, self.__geo, cover_data) # 生成快视图 self.imageHandler.write_quick_view(product_path, color_img=True) # 文件夹打包 file.make_targz(self.__out_para, product_dir) logger.info('process_handle success!') def create_meta_file(self, product_path, type_id_name, type_id_parent): product_dir = self.__workspace_processing_path + 'product\\' xml_path = "./model_meta.xml" tem_folder = self.__workspace_path + EXE_NAME + r"\Temporary""\\" image_path = product_path out_path1 = os.path.join(tem_folder, "trans_geo_projcs.tif") out_path2 = os.path.join(tem_folder, "trans_projcs_geo.tif") par_dict = CreateDict(image_path, out_path1, out_path2).calu_nature(start) model_xml_path = os.path.join(tem_folder, "creat_standard.meta.xml") # 输出xml路径 CreateStadardXmlFile(xml_path, self.alg_xml_path, par_dict, model_xml_path).create_standard_xml() SrcImagePath = self.__input_paras["MultiTempSAR"]['ParaValue'] paths = SrcImagePath.split(';') SrcImageName=os.path.split(paths[0])[1].split('.tar.gz')[0] if len(paths) >= 2: for i in range(1, len(paths)): SrcImageName=SrcImageName+";"+os.path.split(paths[i])[1].split('.tar.gz')[0] meta_xml_path = product_dir+EXE_NAME+"Product.meta.xml" CreateMetafile(self.__processing_paras['META'], self.alg_xml_path, model_xml_path, meta_xml_path).process2(type_id_name, type_id_parent, SrcImageName) if __name__ == '__main__': multiprocessing.freeze_support() #解决打包与运行错误 start = datetime.datetime.now() try: if len(sys.argv) < 2: xml_path = 'LandCover_geo.xml' else: xml_path = sys.argv[1] main_handler = LandCoverMain(xml_path) if main_handler.check_source() is False: raise Exception('check_source() failed!') if main_handler.preprocess_handle() is False: raise Exception('preprocess_handle() failed!') if main_handler.process_handle(start) is False: raise Exception('process_handle() failed!') logger.info('successful production of ' + EXE_NAME + ' products!') except Exception: logger.exception("run-time error!") finally: main_handler.del_temp_workspace() pass end = datetime.datetime.now() msg = 'running use time: %s ' % (end - start) logger.info(msg)