#! /usr/bin/env python ''' File: st_run_rttov.py Purpose: Runs RTTOV on a directory structure of points with input information. Project: Land Satellites Data Systems Science Research and Development (LSRD) at the USGS EROS License: NASA Open Source Agreement 1.3 ''' import os import sys import logging import glob from argparse import ArgumentParser from multiprocessing import Pool from subprocess import * import st_utilities as util import st_rttov_defines as st_rttov from st_grid_points import read_grid_points from espa import Metadata, Sys # Global variables logger = logging.getLogger(__name__) def retrieve_command_line_arguments(): """Read arguments from the command line Returns: args : The arguments read from the command line """ parser = ArgumentParser(description='Runs RTTOV on a pre-determined' ' set of points') parser.add_argument('--version', action='version', version=util.Version.version_text()) parser.add_argument('--rttov_data_path', action='store', dest='rttov_data_path', required=False, default=None, help='Path to the RTTOV \'DATA\' directory') parser.add_argument('--xml', action='store', dest='xml_filename', required=True, default=None, help='The XML metadata file to use') parser.add_argument('--process_count', action='store', dest='process_count', required=False, default=1, help='Number of processes to utilize') args = parser.parse_args() if args.rttov_data_path is None: raise Exception('--rttov_data_path must be specified on the ' 'command line') if args.rttov_data_path == '': raise Exception('The RTTOV data directory provided was empty') # Verify that the --xml parameter was specified if args.xml_filename is None: raise Exception('--xml must be specified on the command line') return args class RTTOVProcessingError(Exception): """Exception specifically for RTTOV errors""" pass def process_point_dir(point_data): """Run RTTOV for a point and parse/format the results for later use Args: point_data : path to point, path to RTTOV, and satellite """ (point_dir, rttov_dir, satellite) = point_data current_directory = os.getcwd() # Check point directory if not os.path.isdir(point_dir): message = 'Point directory {0} does not exist'.format(point_dir) raise Exception(message) # Location of RTTOV binaries RTTOV_BIN_PATH = rttov_dir + '/' + st_rttov.RTTOV_BIN_DIR # RTTOV wrapper program location and filename RTTOV_WRAPPER_PATH = RTTOV_BIN_PATH + '/' + st_rttov.RTTOV_WRAPPER_FILENAME # Coefficients directory RTTOV_COEF_PATH = st_rttov.RTTOV_COEF_BASE_DIR + '/' \ + st_rttov.RTTOV_COEF_DIR # Determine RTTOV satellite/sensor coefficient file if satellite == "LANDSAT_4": RTTOV_COEF_FILENAME = st_rttov.RTTOV_L4_COEF_FILENAME elif satellite == "LANDSAT_5": RTTOV_COEF_FILENAME = st_rttov.RTTOV_L5_COEF_FILENAME elif satellite == "LANDSAT_7": RTTOV_COEF_FILENAME = st_rttov.RTTOV_L7_COEF_FILENAME elif satellite == "LANDSAT_8": RTTOV_COEF_FILENAME = st_rttov.RTTOV_L8_COEF_FILENAME elif satellite == "LANDSAT_9": RTTOV_COEF_FILENAME = st_rttov.RTTOV_L9_COEF_FILENAME else: raise RTTOVProcessingError('Unsupported satellite' ' {}'.format(satellite)) # Symlink source RTTOV_COEF_SYMLINK_SOURCE = RTTOV_COEF_PATH + '/' + RTTOV_COEF_FILENAME #r_paths = [os.path.realpath(path) # for path in glob.glob(os.path.join(point_dir, '*'))] #print("r_paths:", r_paths) try: logger.debug('Processing Directory [{}]'.format(point_dir)) os.chdir(point_dir) # RTTOV requires a link to the coefficient file Sys.create_link(RTTOV_COEF_SYMLINK_SOURCE, RTTOV_COEF_FILENAME) wrapper_log = open(st_rttov.RTTOV_LOG_FILENAME, "a") wrapper_subprocess = Popen([RTTOV_WRAPPER_PATH], stdout=wrapper_log, stdin=PIPE, stderr=wrapper_log, universal_newlines=True) wrapper_subprocess.communicate(os.linesep.join([ RTTOV_COEF_FILENAME, st_rttov.RTTOV_BINARY_PROFILE_FILENAME, str(st_rttov.NPROF), str(st_rttov.NLEVELS), str(st_rttov.DO_SOLAR), str(st_rttov.FTYPE), st_rttov.CHANNEL1_PARMS])) if wrapper_subprocess.poll() != 0: msg = ('Error processing RTTOV for point [{}] poll {} ' '\n\nRTTOV log file contents:\n\n' .format(point_dir, wrapper_subprocess.poll())) # Dump log file contents so they appear in the main ESPA log # since they won't be accessible in ESPA if there is a failure with open(st_rttov.RTTOV_LOG_FILENAME) as log: log_lines = log.readlines() log.close() for log_line in log_lines: msg += log_line # Also dump the ASCII profile if it exists if os.path.exists(st_rttov.RTTOV_ASCII_PROFILE_FILENAME): msg += "\nASCII RTTOV profile contents:\n\n" with open(st_rttov.RTTOV_ASCII_PROFILE_FILENAME) as profile: profile_lines = profile.readlines() profile.close() for profile_line in profile_lines: msg += profile_line raise RTTOVProcessingError(msg) if not os.path.isfile(st_rttov.RTTOV_OUTPUT_FILENAME): raise RTTOVProcessingError('Missing RTTOV output file' ' {}'.format(st_rttov.RTTOV_OUTPUT_FILENAME)) finally: os.chdir(current_directory) def main(): """Main processing for running RTTOV """ # Command Line Arguments args = retrieve_command_line_arguments() # Configure logging Sys.setup_logging() logger.info('*** Begin RTTOV Processing ***') # XML Metadata espa_metadata = Metadata() espa_metadata.parse(xml_filename=args.xml_filename) satellite = str(espa_metadata.xml_object.global_metadata.satellite) # Load the grid information (grid_points, dummy1, dummy2) = read_grid_points() # Cut down to just the ones we need to run RTTOV on point_parms = [('{0:03}_{1:03}_{2:03}_{3:03}'.format(point.row, point.col, point.reanalysis_row, point.reanalysis_col), args.rttov_data_path, satellite) for point in grid_points if point.run_model] process_count = int(args.process_count) try: if process_count > 1: pools = Pool(process_count) pools.map(process_point_dir, point_parms) else: list(map(process_point_dir, point_parms)) except: logger.exception('Error processing points') raise logger.info('*** RTTOV Processing - Complete ***') if __name__ == '__main__': main()