import os
import logging
import subprocess
from collections import OrderedDict
from types import SimpleNamespace

from argparse import ArgumentParser
from datetime import datetime
from espa import MetadataDict
from espa import System
from espa import ODL


logger = logging.getLogger(__name__)


# Maps the c types used in the QCAL MIN & MAX fields to the correct value
# NOTE: 0 is reserved for fill in the unsigned data types so the min is
#       set to 1
qcal_range_for_data_type = {
    'UINT16': (1, 65535),
    'INT16': (-32768, 32767),
    'UINT8': (1, 255),
}

# Maps the sensor type to the digital object id if using Albers
albers_digital_object_id = '\"https://doi.org/10.5066/P9HDSXAC\"'

# Maps the sensor type to the digital object id if not using Albers
standard_digital_object_id = {
    'OLI_TIRS': '\"https://doi.org/10.5066/P9OGBGM6\"',
    'ETM': '\"https://doi.org/10.5066/P9C7I13B\"',
    'TM': '\"https://doi.org/10.5066/P9IAXOVV\"',

}


def get_band_based_group(key, band_max, band_min=1, default_value=None):
    """ Creates tuples of key value pairs where the key only changes by the
        band number incrementing each time.

        The MTL file has several groups were the key names are all the same
        except for the band number that increments. This function creates a
        list of tuples of key value pairs that have key names that are
        identical besides the band number.

        Args:
            key (str): The key name without the band number.
            band_max (int): The largest band value to create.
            band_min (int, optional): The band value to start at. Defaults
                to 1.
            default_value (any, optional): The value to insert into each
                key value pair. Defaults to None.

        Returns:
            List of tuples that contain key value pairs. This is the
            expected input for an OrderedDict.
    """
    mtl = []

    # Range will stop at max_val so add one to it so that max_val will be
    # included
    for i in range(band_min, band_max + 1):
        band = key + str(i)
        mtl.append((band, default_value))
    return mtl


def get_surface_reflect_algorithm(sat):
    """ Creates the correct key value pairs to add to the MTL file
        depending on if LaSRC or LEDAPS was used.

        Checks the surface reflectance band 1 to see what app version was
        used. The correct key value pairs are returned depending on if the
        version was LaSRC or LEDAPS. If LaSRC or LEDAPS was not found then
        an empty list is returned.

        Args:
            sat (SimpleNamespace): The SimpleNamespace containing all the
                sensor and algorithm specific values.

        Returns:
            List of tuples that contain key value pairs. This is the
            expected input for an OrderedDict.
    """
    data_source = []
    if sat.sr_algorithm.startswith('LEDAPS'):
        data_source = ([
            ('DATA_SOURCE_OZONE', '\"TOMS\"'),
            ('DATA_SOURCE_PRESSURE', '\"NCEP\"'),
            ('DATA_SOURCE_WATER_VAPOR', '\"NCEP\"'),
            ('DATA_SOURCE_AIR_TEMPERATURE', '\"NCEP\"'),
        ])
    elif sat.sr_algorithm.startswith('LASRC'):
        data_source = ([
            ('DATA_SOURCE_OZONE', '\"MODIS\"'),
            ('DATA_SOURCE_PRESSURE', '\"Calculated\"'),
            ('DATA_SOURCE_WATER_VAPOR', '\"MODIS\"'),
            ('DATA_SOURCE_AIR_TEMPERATURE', '\"MODIS\"'),
        ])

    return tuple(data_source)


def get_data_source_reanalysis(st_id):
    """ Gets the reanalysis value from the reanalysis.txt file.

        Returns "N/A" for Split Window.
        For Single Channel, reads in the reanalysis.txt file and looks for
        one of the valid values it can be. It will return the value if it's
        found or else raise an exception.

        Args:
            st_id (str): The surface temperature algorithm that is
                         being used.

        Returns:
            "N/A" for split window; otherwise,
            the value found in reanalysis.txt

        Raises:
            ValueError if a valid value isn't found in reanalysis.txt.
            FileNotFoundError if reanalysis.txt is missing.
    """
    # Split window does not use a reanalysis auxiliary dataset
    if (st_id == "SW"):
        return "N/A"

    file_name = 'reanalysis.txt'
    with open(file_name) as reanalysis:
        for line in reanalysis:
            # Remove any new line characters
            line = line.strip()
            if line.upper() in ('NARR', 'MERRA-2', 'GEOS-5 FP-IT'):
                return line

        # If the expected value wasn't found
        raise ValueError('Error: Invalid value in ' + file_name)


def reflectance_bands(band_range, xml, is_toa=False):
    """ Calculates the values for the reflectance min max bands and creates
        key value pairs for the reflectance bands.

        The reflectance band max and min values have to be calculated
        rather than pulled from a file. The calculation is:

            reflectance max = quantize cal max * scale factor + add offset
            reflectance min = quantize cal min * scale factor + add offset

        Once the reflectance max and min are calculated for a band then the
        key value pairs for that reflectance band are created. After all
        the key value pairs for the bands are created then they are all
        returned.

        Args:
            band_range (list): List of band numbers
            xml (MetadataDict): The XML data.
            is_toa (flag): Flag for TOA bands

        Returns:
            List of all the bands values in the correct order.

    """
    ref_max_min_bands = []
    quantize_bands = []
    ref_mult_bands = []
    ref_add_bands = []

    # Prepare parameter names
    if is_toa:
        name_var = '_TOA_B'
        band_var = '_TOA_BAND_'
    else:
        name_var = '_B'
        band_var = '_BAND_'

    for band_num in band_range:
        # Get values
        sr_band = name_var + str(band_num)
        (quantize_min, quantize_max) = qcal_range_for_data_type[
            xml.get_value('DATA_TYPE' + sr_band)]
        ref_mult = float(xml.get_value('SCALE_FACTOR' + sr_band))
        ref_add = float(xml.get_value('ADD_OFFSET' + sr_band))

        # Calculate
        ref_max_value = quantize_max * ref_mult + ref_add
        ref_min_value = quantize_min * ref_mult + ref_add

        # format the calculated values to only have 6 decimal places
        band = band_var + str(band_num)
        ref_max_min_bands.append(('REFLECTANCE_MAXIMUM' + band,
                                  '{:0.6f}'.format(ref_max_value)))
        ref_max_min_bands.append(('REFLECTANCE_MINIMUM' + band,
                                  '{:0.6f}'.format(ref_min_value)))
        quantize_bands.append(('QUANTIZE_CAL_MAX' + band, quantize_max))
        quantize_bands.append(('QUANTIZE_CAL_MIN' + band, quantize_min))
        ref_mult_bands.append(('REFLECTANCE_MULT' + band, ref_mult))
        ref_add_bands.append(('REFLECTANCE_ADD' + band, ref_add))

    return ref_max_min_bands + quantize_bands + ref_mult_bands + ref_add_bands


def surface_temp_bands(band, xml, st_id):
    """ Calculates the surface temp min max bands and creates the key value
        pairs for the surface temp bands.

        The surface temp min max band has to be calculated rather than
        pulled from another file. The calculation is:

            temp max = quantize cal max * scale factor + add offset
            temp min = quantize cal min * scale factor + add offset

        Once the surface temp max and min are calculated the key value
        pairs for the surface temp band are created.

        Args:
            band (str): The band name to use.
            xml (MetadataDict): The XML data.
            st_id (str): The ID for the version of surface temperature that is
                         being used.

        Returns:
            List of all the band values in the correct order.
    """
    (quantize_min, quantize_max) = qcal_range_for_data_type[
        xml.get_value('DATA_TYPE_' + st_id)]
    temp_mult = float(xml.get_value(
        'SCALE_FACTOR_' + st_id))
    temp_offset = float(xml.get_value(
        'ADD_OFFSET_' + st_id))

    temp_max = quantize_max * temp_mult + temp_offset
    temp_min = quantize_min * temp_mult + temp_offset

    # format the calculated values to only have 6 decimal places
    return [
        ('TEMPERATURE_MAXIMUM_BAND' + band, '{:0.6f}'.format(temp_max)),
        ('TEMPERATURE_MINIMUM_BAND' + band, '{:0.6f}'.format(temp_min)),
        ('QUANTIZE_CAL_MAXIMUM_BAND' + band, quantize_max),
        ('QUANTIZE_CAL_MINIMUM_BAND' + band, quantize_min),
        ('TEMPERATURE_MULT_BAND' + band, temp_mult),
        ('TEMPERATURE_ADD_BAND' + band, temp_offset)
    ]


def get_st_intermediate_bands(sat):
    """ Gets the correct output fields for the surface temperature algorithm.

        Checks to see if the surface temperature algorithm is split window.
        Depending on the result of that check, the correct output file names
        and data types are returned It also returns a the mapping for the file
        names and data types so that their values can be pulled out of the XML.

        Args:
            sat (SimpleNamespace): The SimpleNamespace containing all the
                sensor and algorithm specific values.

        Returns:
            A list that contains:
                The output file name fields in a list of tuple key, value
                    pairs.
                The output data type fields in a list of tuple key, value
                    pairs.
                A dict of the name mappings for the file name and data type
                    fields that need it.
    """
    if sat.st_algorithm.startswith('SPLIT_WINDOW'):
        st_intermediate_file_name = (
            ('FILE_NAME_EMISSIVITY_BAND_10', None),
            ('FILE_NAME_EMISSIVITY_BAND_11', None),
        )
        st_intermediate_data_type = (
            ('DATA_TYPE_EMISSIVITY_BAND_10', None),
            ('DATA_TYPE_EMISSIVITY_BAND_11', None),
        )

        st_intermediate_name_map = {
            'FILE_NAME_EMISSIVITY_BAND_10': 'FILE_NAME_EMIS_B10',
            'FILE_NAME_EMISSIVITY_BAND_11': 'FILE_NAME_EMIS_B11',
            'DATA_TYPE_EMISSIVITY_BAND_10': 'DATA_TYPE_EMIS_B10',
            'DATA_TYPE_EMISSIVITY_BAND_11': 'DATA_TYPE_EMIS_B11',
        }

    else:
        st_intermediate_file_name = (
            ('FILE_NAME_THERMAL_RADIANCE', None),
            ('FILE_NAME_UPWELL_RADIANCE', None),
            ('FILE_NAME_DOWNWELL_RADIANCE', None),
            ('FILE_NAME_ATMOSPHERIC_TRANSMITTANCE', None),
            ('FILE_NAME_EMISSIVITY', None),
            ('FILE_NAME_EMISSIVITY_STDEV', None),
            ('FILE_NAME_CLOUD_DISTANCE', None)
        )
        st_intermediate_data_type = (
            ('DATA_TYPE_THERMAL_RADIANCE', None),
            ('DATA_TYPE_UPWELL_RADIANCE', None),
            ('DATA_TYPE_DOWNWELL_RADIANCE', None),
            ('DATA_TYPE_ATMOSPHERIC_TRANSMITTANCE', None),
            ('DATA_TYPE_EMISSIVITY', None),
            ('DATA_TYPE_EMISSIVITY_STDEV', None),
            ('DATA_TYPE_CLOUD_DISTANCE', None)
        )

        st_intermediate_name_map = {
            'FILE_NAME_THERMAL_RADIANCE': 'FILE_NAME_TRAD',
            'FILE_NAME_UPWELL_RADIANCE': 'FILE_NAME_URAD',
            'FILE_NAME_DOWNWELL_RADIANCE': 'FILE_NAME_DRAD',
            'FILE_NAME_ATMOSPHERIC_TRANSMITTANCE': 'FILE_NAME_ATRAN',
            'FILE_NAME_EMISSIVITY': 'FILE_NAME_EMIS',
            'FILE_NAME_EMISSIVITY_STDEV': 'FILE_NAME_EMSD',
            'DATA_TYPE_THERMAL_RADIANCE': 'DATA_TYPE_TRAD',
            'DATA_TYPE_UPWELL_RADIANCE': 'DATA_TYPE_URAD',
            'DATA_TYPE_DOWNWELL_RADIANCE': 'DATA_TYPE_DRAD',
            'DATA_TYPE_ATMOSPHERIC_TRANSMITTANCE': 'DATA_TYPE_ATRAN',
            'DATA_TYPE_EMISSIVITY': 'DATA_TYPE_EMIS',
            'DATA_TYPE_EMISSIVITY_STDEV': 'DATA_TYPE_EMSD',
            'FILE_NAME_CLOUD_DISTANCE': 'FILE_NAME_CDIST',
            'DATA_TYPE_CLOUD_DISTANCE': 'DATA_TYPE_CDIST'
        }

    return (st_intermediate_file_name, st_intermediate_data_type,
            st_intermediate_name_map)


def bt_bands(band_range, xml):
    """ Calculates the values for the Brightness Temperature(BT) min max bands
        and creates key value pairs for the BT bands.

        The BT band max and min values have to be calculated rather than pulled
        from a file. The calculation is:

            BT max = quantize cal max * scale factor + add offset
            BT min = quantize cal min * scale factor + add offset

        Once the BT max and BT min are calculated for a band then the key value
        pairs for that BT band are created. After all the key value
        pairs for the bands are created then they are all returned.

        Args:
            band_range (list): List of band numbers
            xml (MetadataDict): The XML data.

        Returns:
            List of all the bands values in the correct order.

    """
    bt_max_min_bands = []
    quantize_bands = []
    bt_mult_bands = []
    bt_add_bands = []

    for band_num in band_range:
        # Get values
        bt_band = '_BT_B' + str(band_num)
        (quantize_min, quantize_max) = qcal_range_for_data_type[
            xml.get_value('DATA_TYPE' + bt_band)]
        bt_mult = float(xml.get_value('SCALE_FACTOR' + bt_band))
        bt_add = float(xml.get_value('ADD_OFFSET' + bt_band))

        # Calculate
        bt_max_value = quantize_max * bt_mult + bt_add
        bt_min_value = quantize_min * bt_mult + bt_add

        # Format the calculated values to only have 6 decimal places
        band = '_BT_BAND_' + str(band_num)
        bt_max_min_bands.append(('TEMPERATURE_MAXIMUM' + band,
                                 '{:0.6f}'.format(bt_max_value)))
        bt_max_min_bands.append(('TEMPERATURE_MINIMUM' + band,
                                 '{:0.6f}'.format(bt_min_value)))
        quantize_bands.append(('QUANTIZE_CAL_MAX' + band, quantize_max))
        quantize_bands.append(('QUANTIZE_CAL_MIN' + band, quantize_min))
        bt_mult_bands.append(('TEMPERATURE_MULT' + band, bt_mult))
        bt_add_bands.append(('TEMPERATURE_ADD' + band, bt_add))

    return bt_max_min_bands + quantize_bands + bt_mult_bands \
        + bt_add_bands


def create_name_mapping_group(new_name, old_name, band_range):
    """ Creates a dict that contains a name mapping group that follows a
        pattern. For example, a mapping for FILE_NAME_BAND_<number> and
        FILE_NAME_B<number>. The mapping is setup so that the new name is the
        key, while the old name is the value.

        Args:
            new_name (str): The new name to map the old name too.
            old_name (str): The old name to map to the new name.
            band_range (list): A list of band numbers for the group.

        Returns:
            A dict containing the the new name as the keys and the
            corresponding old name as the value.
    """
    name_map = {}
    for band_num in band_range:
        name_map.update({
            new_name + str(band_num): old_name + str(band_num)
        })
    return name_map


def get_st_id(l2_xml_obj):
    """ Gets the ID for the surface temperature band, such as ST_B10.

        Args:
            l2_xml_obj (MetadataDict): The level 2 xml MetadataDict object.

        Returns:
            A string containing the ID for the surface temperature band.
    """
    for key, value in l2_xml_obj.data.items():
        if value == '\"Surface Temperature\"':
            st_id = key.split('LONG_NAME_')[1]
            return st_id


def get_sensor_specific_values(sensor, map_projection, sr_algorithm, l2_xml,
                               l2_product_type):
    """ Gets the values that are specific for the sensor and algorithms used.

        Creates a SimpleNamespace object that holds specific values depending
        on the sensor, map projection and algorithms combination. The specific
        values include the available bands, surface temperature bands, surface
        reflectance bands, TOA reflectance bands, brightness temperature bands,
        digital object id and name mapping.

        Note:
            Currently the second surface temperature band for ETM is commented
            out. This is due to some questions as to how it will be handled in
            the XML XSD. Once that is resolved, they can be uncommented and
            this note can be removed.

        Args:
            sensor (str): The sensor value pulled from the level 1 MTL file.
            map_projection (str): the map project value from the level 1 MTL
                file.
            sr_algorithm (str): The surface reflectance algorithm from the
                level 2 XML. May also be None.
            l2_xml (MetadataDict): The level 2 xml MetadataDict object.
            l2_product_type (str): The defined product type. This is either
                    'sr_st_toa_bt'(L2 SR/ST/TOA/BT product) or
                    'sr_st'(L2 SR/ST product) or 'sr'(L2 SR-Only product).

        Returns:
            A SimpleNamespace object that contains the sensor specific values.
    """
    sat = SimpleNamespace()
    sat.sensor = sensor
    if ("st" in l2_product_type):
        sat.st_id = get_st_id(l2_xml)
        sat.st_algorithm = l2_xml.get_value('APP_VERSION_' + sat.st_id).upper()

    sat.specific_name_mapping = {}

    # Get the DOI value based off the map projection
    if map_projection == 'AEA':
        sat.digital_object_id = albers_digital_object_id
    else:
        sat.digital_object_id = standard_digital_object_id[sensor]

    sat.sr_band_range = list(range(1, 8))
    sat.toa_band_range = list(range(1, 8))

    # Band-9 is included in OLITIRS L2 TOA reflectance bands. Whereas band-9
    # is not included in OLITIRS L2 SR bands.
    if (sensor == 'OLI_TIRS'):
        sat.toa_band_range.append(9)
    elif sensor in ('ETM', 'TM'):
        # TOA and SR are not made for thermal band 6 for TM and ETM, so remove
        # from list
        del sat.sr_band_range[5]
        del sat.toa_band_range[5]

    if sr_algorithm:
        sat.sr_algorithm = sr_algorithm.upper()
        if (sat.sr_algorithm.startswith('LASRC')):
            sat.specific_sr_file_name = tuple([
                ('FILE_NAME_QUALITY_L2_AEROSOL', None)])
            sat.specific_sr_data_type = tuple([
                ('DATA_TYPE_QUALITY_L2_AEROSOL', None)])

            # Add the name mapping for the reflectance bands
            sat.specific_name_mapping.update({
                'FILE_NAME_QUALITY_L2_AEROSOL': 'FILE_NAME_QA_AEROSOL',
                'DATA_TYPE_QUALITY_L2_AEROSOL': 'DATA_TYPE_QA_AEROSOL'
            })

        elif (sat.sr_algorithm.startswith('LEDAPS')):
            sat.specific_sr_file_name = tuple([
                ('FILE_NAME_ATMOSPHERIC_OPACITY', None),
                ('FILE_NAME_QUALITY_L2_SURFACE_REFLECTANCE_CLOUD', None)])

            sat.specific_sr_data_type = tuple([
                ('DATA_TYPE_ATMOSPHERIC_OPACITY', None),
                ('DATA_TYPE_QUALITY_L2_SURFACE_REFLECTANCE_CLOUD', None)])

            # Add the name mapping for the reflectance bands
            sat.specific_name_mapping.update({
                'FILE_NAME_QUALITY_L2_SURFACE_REFLECTANCE_CLOUD':
                    'FILE_NAME_CLOUD_QA',
                'DATA_TYPE_QUALITY_L2_SURFACE_REFLECTANCE_CLOUD':
                    'DATA_TYPE_CLOUD_QA',
                'FILE_NAME_ATMOSPHERIC_OPACITY': 'FILE_NAME_ATMOS_OPACITY',
                'DATA_TYPE_ATMOSPHERIC_OPACITY': 'DATA_TYPE_ATMOS_OPACITY'
            })
    else: # sr_algorithm = None
        sat.sr_algorithm = ""
        sat.specific_sr_file_name = tuple()
        sat.specific_sr_data_type = tuple()

    # Initialize dictionaries
    file_name_dict = dict()
    data_type_dict = dict()
    file_name_toa_dict = dict()
    data_type_toa_dict = dict()
    file_name_bt_dict = dict()
    data_type_bt_dict = dict()

    # Add SR bands
    if ("sr" in l2_product_type):
        if sensor == 'OLI_TIRS':
            file_name_dict = dict(zip(range(1, 8),
                get_band_based_group('FILE_NAME_BAND_', 7)))
            data_type_dict = dict(zip(range(1, 8),
                get_band_based_group('DATA_TYPE_BAND_', 7)))

        elif sensor == 'TM':
            # Add bands 1-5, 7
            file_name_dict = dict(zip(range(1, 6),
                get_band_based_group('FILE_NAME_BAND_', 5)))
            file_name_dict[7] = ('FILE_NAME_BAND_7',  None)
            data_type_dict = dict(zip(range(1, 6),
                get_band_based_group('DATA_TYPE_BAND_', 5)))
            data_type_dict[7] = ('DATA_TYPE_BAND_7',  None)

        elif sensor == 'ETM':
            # Add bands 1-5, 7
            file_name_dict = dict(zip(range(1, 6),
                get_band_based_group('FILE_NAME_BAND_', 5)))
            file_name_dict[7] = ('FILE_NAME_BAND_7',  None)
            data_type_dict = dict(zip(range(1, 6),
                get_band_based_group('DATA_TYPE_BAND_', 5)))
            data_type_dict[7] = ('DATA_TYPE_BAND_7',  None)

        # Create the name mapping for the SR bands
        sat.specific_name_mapping.update(
            create_name_mapping_group(
                'FILE_NAME_BAND_', 'FILE_NAME_B', sat.sr_band_range)
        )
        sat.specific_name_mapping.update(
            create_name_mapping_group(
                'DATA_TYPE_BAND_', 'DATA_TYPE_B', sat.sr_band_range)
        )

    # Add ST bands
    if ("st" in l2_product_type):
        if sat.st_algorithm.startswith('SPLIT_WINDOW'):
            # Add the ST SW band to the band dictionaries (assuming band 10
            # for now, since split window is only used for L8-9 TIRS
            file_name_dict[10] = ('FILE_NAME_BAND_ST_SW',  None)
            data_type_dict[10] = ('DATA_TYPE_BAND_ST_SW',  None)
            sat.temp_band = ['_ST_SW']
            sat.st_quality_file_name = tuple()
            sat.st_quality_data_type = tuple()

            # Add the name mapping for the temp bands
            sat.specific_name_mapping.update({
                'FILE_NAME_BAND_ST_SW': 'FILE_NAME_' + sat.st_id,
                'DATA_TYPE_BAND_ST_SW': 'DATA_TYPE_' + sat.st_id,
            })

        elif (sat.st_algorithm.startswith('ST_') or
                sat.st_algorithm.startswith('SINGLE_CHANNEL_')):
            sat.st_quality_file_name = tuple([
                ('FILE_NAME_QUALITY_L2_SURFACE_TEMPERATURE', None)])
            sat.st_quality_data_type = tuple([
                ('DATA_TYPE_QUALITY_L2_SURFACE_TEMPERATURE', None)])

            if sensor == 'OLI_TIRS':
                sat.temp_band = ['_ST_B10']
                # Add the ST Single Channel band to the band dictionaries
                file_name_dict[10] = ('FILE_NAME_BAND_ST_B10',  None)
                data_type_dict[10] = ('DATA_TYPE_BAND_ST_B10',  None)

                sat.specific_name_mapping.update({
                    'FILE_NAME_BAND_ST_B10': 'FILE_NAME_' + sat.st_id,
                    'DATA_TYPE_BAND_ST_B10': 'DATA_TYPE_' + sat.st_id,
                })

            elif (sensor in ('TM', 'ETM')):
                sat.temp_band = ['_ST_B6']
                # Add the ST Single Channel band to the band dictionaries
                file_name_dict[6] = ('FILE_NAME_BAND_ST_B6',  None)
                data_type_dict[6] = ('DATA_TYPE_BAND_ST_B6',  None)

                # Add the name mapping for the temp bands
                sat.specific_name_mapping.update({
                    'FILE_NAME_BAND_ST_B6': 'FILE_NAME_B6',
                    'DATA_TYPE_BAND_ST_B6': 'DATA_TYPE_B6'
                })

    # Add BT bands
    if ("bt" in l2_product_type):
        if sensor == 'OLI_TIRS':
            # Add bands BT 10, 11
            sat.bt_band_range = ['10', '11']
            file_name_bt_dict[1] = ('FILE_NAME_BT_BAND_10', None)
            file_name_bt_dict[2] = ('FILE_NAME_BT_BAND_11', None)
            data_type_bt_dict[1] = ('DATA_TYPE_BT_BAND_10', None)
            data_type_bt_dict[2] = ('DATA_TYPE_BT_BAND_11', None)
        elif sensor == 'TM':
            # Add bands BT 6
            sat.bt_band_range = ['6']
            file_name_bt_dict[1] = ('FILE_NAME_BT_BAND_6', None)
            data_type_bt_dict[1] = ('DATA_TYPE_BT_BAND_6', None)
        elif sensor == 'ETM':
            # Add bands BT 6
            sat.bt_band_range = ['6']
            file_name_bt_dict[1] = ('FILE_NAME_BT_BAND_6', None)
            data_type_bt_dict[1] = ('DATA_TYPE_BT_BAND_6', None)

        # Create the name mapping for the BT bands
        sat.specific_name_mapping.update(
            create_name_mapping_group(
                'FILE_NAME_BT_BAND_', 'FILE_NAME_BT_B', sat.bt_band_range)
        )
        sat.specific_name_mapping.update(
            create_name_mapping_group(
                'DATA_TYPE_BT_BAND_', 'DATA_TYPE_BT_B', sat.bt_band_range)
        )

    # Add TOA bands
    if ("toa" in l2_product_type):
        if sensor == 'OLI_TIRS':
            file_name_toa_dict = dict(zip(range(1, 8),
                get_band_based_group('FILE_NAME_TOA_BAND_', 7)))
            data_type_toa_dict = dict(zip(range(1, 8),
                get_band_based_group('DATA_TYPE_TOA_BAND_', 7)))
            # Add TOA reflectance band-9
            file_name_toa_dict[9] = ('FILE_NAME_TOA_BAND_9',  None)
            data_type_toa_dict[9] = ('DATA_TYPE_TOA_BAND_9',  None)

        elif sensor in ('TM','ETM'):
            # Add bands 1-5, 7
            file_name_toa_dict = dict(zip(range(1, 6),
                get_band_based_group('FILE_NAME_TOA_BAND_', 5)))
            file_name_toa_dict[7] = ('FILE_NAME_TOA_BAND_7', None)
            data_type_toa_dict = dict(zip(range(1, 6),
                get_band_based_group('DATA_TYPE_TOA_BAND_', 5)))
            data_type_toa_dict[7] = ('DATA_TYPE_TOA_BAND_7', None)

        # Create the name mapping for the TOA bands
        sat.specific_name_mapping.update(
            create_name_mapping_group(
                'FILE_NAME_TOA_BAND_', 'FILE_NAME_TOA_B', sat.toa_band_range)
        )
        sat.specific_name_mapping.update(
            create_name_mapping_group(
                'DATA_TYPE_TOA_BAND_', 'DATA_TYPE_TOA_B', sat.toa_band_range)
        )

    # Sort the file name & data type dictionaries
    sat.file_name_bands = tuple(
        [file_name_dict[x] for x in sorted(file_name_dict)])
    sat.data_type_bands = tuple(
        [data_type_dict[x] for x in sorted(data_type_dict)])

    sat.file_name_toa_bands = tuple(
        [file_name_toa_dict[x] for x in sorted(file_name_toa_dict)])
    sat.data_type_toa_bands = tuple(
        [data_type_toa_dict[x] for x in sorted(data_type_toa_dict)])

    sat.file_name_bt_bands = tuple(
        [file_name_bt_dict[x] for x in sorted(file_name_bt_dict)])
    sat.data_type_bt_bands = tuple(
        [data_type_bt_dict[x] for x in sorted(data_type_bt_dict)])

    return sat


def copy_data_to_mtl(groups, data, key_lookup):
    """ Goes through each key in the MTL structure and copies the correct
        value over.

        Loops through each group in the MTL structure. If the key doesn't
        already have a value then it uses the key to lookup the values in
        the passed in data dictionary. Some key names in the data
        dictionary don't match with the respective MTL key name due to name
        changes or how the key was created. The key lookup dictionary that
        is passed in provides conversion between the two. If the key is not
        in the lookup dictionary, then the key is tried directly in the
        data dictionary itself. If the key isn't found an exception is
        raised.

        Args:
            groups (OrderedDict): The OrderedDict MTL structure.
            data (ODL or MetadataDict): Contains the values to copy over.
            key_lookup (dict): Contains the conversions between the key
                names in the MTL structure and the data dictionary.

        Returns:
            Void.

        Raises:
            KeyError if the key is not found.
    """
    for mtl_group in groups:
        group = groups[mtl_group]
        for key in group:
            if group[key] is None:
                try:
                    group[key] = data.get_value(key_lookup[key],
                                                strip_quotes=False)
                except KeyError:
                    pass
                else:
                    # The key was found so move onto the next one
                    continue

                try:
                    group[key] = data.get_value(key, strip_quotes=False)
                except KeyError:
                    # Log error message and re-raise the error
                    logger.error('Error: Unable to locate ' + key)
                    raise


def join_ordered_dict(*args):
    """ Simple helper function that combines OrderedDicts into a single
        one.

        Note:
            Order matters. The last ordered dict will overwrite any
            identical key from the other ordered dicts.

        Args:
            *args: The ordered dicts to join together.

        Returns:
            OrderedDict containing the joined OrderedDicts.
    """
    ordered_dict = OrderedDict([])
    for arg in args:
        ordered_dict.update(arg)
    return ordered_dict


def get_software_version():
    """ Gets the current software version.

        Args:
            None.

        Returns:
            The software version

        Raises:
            OSError or subprocess.CalledProcessError if an error occurs
            while attempting to check the output.
            ValueError if unable to determine processing system name
    """
    version = None
    name = os.getenv('PROCESSING_SYSTEM')
    if name is None:
        logger.error('Error determining system name from $PROCESSING_SYSTEM')
        raise ValueError('Unknown processing system name')

    try:
        version = subprocess.check_output(['ias_version']).strip()
        version = '\"' + name + '_' + str(version.decode('ascii')) + '\"'

    except (OSError, subprocess.CalledProcessError):
        logger.error('Unable to check software version using ias_version')
        raise

    except UnicodeDecodeError:
        logger.error('Failed to format software version value')
        version = '\"' + name + '_' + str(version) + '\"'

    return version


def nested_dict_to_flat(nested_dict):
    """ Converts the ODL nested structure to a flat structure like the XML
        uses.

        The XML file is parsed into a flat OrderedDict while the ODL files
        are parsed into a nested OrderedDict. This is function converts the
        ODL nested OrderedDict to a flat dict. This is so that they can be
        joined without any problems when copying over the values from the
        l2 XML and l1 MTL files. Uses a internally defined recursive
        function to navigate the nested dict.

        Args:
            nested_dict (OrderedDict): The nested OrderedDict to convert.

        Returns:
            The flat version of the nested dict.
    """
    def _convert(current_group, new_dict):
        for key in current_group:
            if isinstance(current_group[key], OrderedDict):
                _convert(current_group[key], new_dict)

            else:
                value = current_group[key]
                new_dict[key] = value

    converted_dict = {}
    _convert(nested_dict, converted_dict)
    return converted_dict


def create_l2_mtl(l2_prod_id, output_dir, product_date, l2_product_type,
                  processing_level, include_angle_bands):
    """ Creates the level 2 MTL file from the level 2 espa _gtif XML, level 1
        MTL, OMF, reanalysis.txt and FORMAT_AND_PACKAGE_L2 file.

        All of the files are parsed into memory and then the portions that are
        required for the level 2 MTL file are copied into an ODL object. Some
        of the names for the keys need to be changed for the ODL file and those
        are converted over when the key value pairs are copied over. Once all
        the key value pairs have been copied over, the ODL object is written to
        disk.

        Args:
            l2_prod_id (str): The level 2 product id to use.
            output_dir (str): The directory to output the completed files too.
            product_date (datetime): A datetime object containing the current
                utc time
            l2_product_type (str): The defined product type. This is either
                    'sr_st_toa_bt'(L2 SR/ST/TOA/BT product),
                     or 'sr_toa_bt'(L2 SR-Only Albers product),
                     or 'sr_st'(L2 SR/ST product) or 'sr'(L2 SR-Only product).
            processing_level: Processing level based on the L2 product type.
                    This is either 'L2SP' or 'L2SR'.
            include_angle_bands: Flag that enables/disables the inclusion of
                    the band 4 angle bands.

        Returns:
            void.
    """
    # Get the level 2 MTL file name
    l2_mtl_filename = l2_prod_id + '_MTL.txt'

    # Get the l2 _gtif.xml file generated by format and package
    l2_gtif_xml_filename = os.path.join(output_dir, l2_prod_id + '_gtif.xml')
    l2_xml = MetadataDict(l2_gtif_xml_filename)

    # Parse now so that we can get the l1 mtl file name
    l2_xml.parse()

    l1_mtl_filename = l2_xml.get_value('lpgs_metadata_file', upper_case=False)

    l1_mtl = ODL(file_name=l1_mtl_filename)

    format_and_pack = ODL(file_name='FORMAT_AND_PACKAGE_L2.odl')

    # Parse now so that we can get the omf file name
    format_and_pack.parse()

    omf_filename = format_and_pack.get_value('WORK_ORDER_ID') + '.omf'
    omf = ODL(file_name=omf_filename)

    for parser in (l1_mtl, omf):
        parser.parse()

    output_format = '\"GEOTIFF\"'
    # On KeyError, the HDF parameters don't exist so leave format as default
    try:
        if format_and_pack.get_value('HDF5_FILENAME') != '':
            output_format = '\"HDF5\"'
    except KeyError:
        pass
    try:
        if format_and_pack.get_value('HDF_FILENAME') != '':
            output_format = '\"HDF4\"'
    except KeyError:
        pass

    # Get the values that are specific to the spacecraft, sensor and map
    # projection
    sat = get_sensor_specific_values(
        l1_mtl.get_value('SENSOR_ID'),
        l1_mtl.get_value('MAP_PROJECTION', group_name='PROJECTION_ATTRIBUTES'),
        l2_xml.get_value('APP_VERSION_B1'), l2_xml, l2_product_type)

    # New name: Old name
    l2_name_changes = {
        'LANDSAT_PRODUCT_ID': 'PRODUCT_ID',
        'FILE_NAME_QUALITY_L2_SURFACE_TEMPERATURE': 'FILE_NAME_QA',
        'FILE_NAME_QUALITY_L1_PIXEL': 'FILE_NAME_QA_PIXEL',
        'FILE_NAME_QUALITY_L1_RADIOMETRIC_SATURATION': 'FILE_NAME_QA_RADSAT',
        'DATA_TYPE_QUALITY_L2_SURFACE_TEMPERATURE': 'DATA_TYPE_QA',
        'DATA_TYPE_QUALITY_L1_PIXEL': 'DATA_TYPE_QA_PIXEL',
        'DATA_TYPE_QUALITY_L1_RADIOMETRIC_SATURATION': 'DATA_TYPE_QA_RADSAT',
        'MAP_PROJECTION': 'PROJECTION_INFORMATION_PROJECTION',
        'DATUM': 'PROJECTION_INFORMATION_DATUM',
        'ALGORITHM_SOURCE_SURFACE_REFLECTANCE': 'APP_VERSION_B1',
    }

    if ("st" in l2_product_type):
        # Get the correct output based of the surface temperature algorithm used
        (st_output_file_names, st_output_data_types, st_output_name_mapping) = (
            get_st_intermediate_bands(sat)
        )
        # Add in satellite specific name mapping
        l2_name_changes.update(st_output_name_mapping)

        st_processing_record = (
            ('ALGORITHM_SOURCE_SURFACE_TEMPERATURE',
             l2_xml.get_value('APP_VERSION_' + sat.st_id, strip_quotes=False)),
            ('DATA_SOURCE_REANALYSIS', '\"' +
             get_data_source_reanalysis(sat.st_id) + '\"')
        )
    else:
        st_output_file_names = tuple()
        sat.st_quality_file_name = tuple()
        st_output_data_types = tuple()
        sat.st_quality_data_type = tuple()
        st_processing_record = tuple()

    # Include the angle bands if requested
    if (include_angle_bands):
        # Initial values for these fields are not populated
        file_name_angle_bands = tuple([
            ('FILE_NAME_ANGLE_SENSOR_AZIMUTH_BAND_4', None),
            ('FILE_NAME_ANGLE_SENSOR_ZENITH_BAND_4', None),
            ('FILE_NAME_ANGLE_SOLAR_AZIMUTH_BAND_4', None),
            ('FILE_NAME_ANGLE_SOLAR_ZENITH_BAND_4', None)])

        data_type_angle_bands = tuple([
            ('DATA_TYPE_ANGLE_SENSOR_AZIMUTH_BAND_4', None),
            ('DATA_TYPE_ANGLE_SENSOR_ZENITH_BAND_4', None),
            ('DATA_TYPE_ANGLE_SOLAR_AZIMUTH_BAND_4', None),
            ('DATA_TYPE_ANGLE_SOLAR_ZENITH_BAND_4', None)])

        # Provide field names from the L2 xml to get the value from
        l2_name_changes.update({
            'FILE_NAME_ANGLE_SENSOR_AZIMUTH_BAND_4': 'FILE_NAME_VAA',
            'FILE_NAME_ANGLE_SENSOR_ZENITH_BAND_4': 'FILE_NAME_VZA',
            'FILE_NAME_ANGLE_SOLAR_AZIMUTH_BAND_4': 'FILE_NAME_SAA',
            'FILE_NAME_ANGLE_SOLAR_ZENITH_BAND_4': 'FILE_NAME_SZA',
            'DATA_TYPE_ANGLE_SENSOR_AZIMUTH_BAND_4': 'DATA_TYPE_VAA',
            'DATA_TYPE_ANGLE_SENSOR_ZENITH_BAND_4': 'DATA_TYPE_VZA',
            'DATA_TYPE_ANGLE_SOLAR_AZIMUTH_BAND_4': 'DATA_TYPE_SAA',
            'DATA_TYPE_ANGLE_SOLAR_ZENITH_BAND_4': 'DATA_TYPE_SZA'
        })

    else:
        file_name_angle_bands = tuple()
        data_type_angle_bands = tuple()

    l2_name_changes.update(sat.specific_name_mapping)

    new_l2_mtl = ODL()

    new_l2_mtl.add_group('PRODUCT_CONTENTS', values=(
            (('ORIGIN', l1_mtl.get_value('ORIGIN', strip_quotes=False)),
            ('DIGITAL_OBJECT_IDENTIFIER', sat.digital_object_id),
            ('LANDSAT_PRODUCT_ID', '\"' + l2_prod_id + '\"'),
            ('PROCESSING_LEVEL', '\"' + processing_level + '\"'),
            ('COLLECTION_NUMBER', l1_mtl.get_value('COLLECTION_NUMBER')),
            ('COLLECTION_CATEGORY', l1_mtl.get_value('COLLECTION_CATEGORY',
                                                    strip_quotes=False)),
            ('OUTPUT_FORMAT', output_format)) +
            sat.file_name_bands +
            sat.file_name_toa_bands +
            sat.file_name_bt_bands +
            st_output_file_names +
            sat.specific_sr_file_name +
            sat.st_quality_file_name +
            (('FILE_NAME_QUALITY_L1_PIXEL', None),
            ('FILE_NAME_QUALITY_L1_RADIOMETRIC_SATURATION', None)) +
            file_name_angle_bands +
            (('FILE_NAME_ANGLE_COEFFICIENT', '\"' + l2_prod_id + '_ANG.txt\"'),
            ('FILE_NAME_METADATA_ODL', '\"' + l2_mtl_filename + '\"'),
            ('FILE_NAME_METADATA_XML', '\"' + l2_prod_id + '_MTL.xml\"')) +
            sat.data_type_bands +
            sat.data_type_toa_bands +
            sat.data_type_bt_bands +
            st_output_data_types +
            sat.specific_sr_data_type +
            sat.st_quality_data_type +
            (('DATA_TYPE_QUALITY_L1_PIXEL', None),
            ('DATA_TYPE_QUALITY_L1_RADIOMETRIC_SATURATION', None)) +
            data_type_angle_bands
    ))

    new_l2_mtl.add_group('IMAGE_ATTRIBUTES', values=(
        l1_mtl.get_group('IMAGE_ATTRIBUTES')
        )
    )

    new_l2_mtl.add_group('PROJECTION_ATTRIBUTES', values=(
        l1_mtl.get_group('PROJECTION_ATTRIBUTES')
        )
    )

    # Remove PROJECTION_ATTRIBUTES fields that aren't needed
    # Level 2 products never have PAN band
    fields_to_delete = [ 'PANCHROMATIC_LINES',
                         'GRID_CELL_SIZE_PANCHROMATIC',
                         'PANCHROMATIC_SAMPLES'
                       ]
    # If ST and BT are not included, also remove thermal band fields
    if ("st" not in l2_product_type and "bt" not in l2_product_type):
        fields_to_delete.extend( [ 'GRID_CELL_SIZE_THERMAL',
                                   'THERMAL_LINES',
                                   'THERMAL_SAMPLES'
                                  ] )

    for field in fields_to_delete:
        try:
            new_l2_mtl.delete_value(field, group_name='PROJECTION_ATTRIBUTES')
        except KeyError:
            # If it was a KeyError, the field didn't exist so skip it
            pass

    new_l2_mtl.add_group('LEVEL2_PROCESSING_RECORD', values=(
            (('ORIGIN', l1_mtl.get_value('ORIGIN', strip_quotes=False)),
            ('DIGITAL_OBJECT_IDENTIFIER', sat.digital_object_id),
            ('REQUEST_ID', omf.get_value('PRODUCT_REQUEST_ID',
                                        strip_quotes=False)),
            ('LANDSAT_PRODUCT_ID', '\"' + l2_prod_id + '\"'),
            ('PROCESSING_LEVEL', '\"' + processing_level + '\"'),
            ('OUTPUT_FORMAT', output_format),
            ('DATE_PRODUCT_GENERATED',
                product_date.strftime('%Y-%m-%dT%H:%M:%SZ')),
            ('PROCESSING_SOFTWARE_VERSION', get_software_version()),
            ('ALGORITHM_SOURCE_SURFACE_REFLECTANCE', None)) +
            get_surface_reflect_algorithm(sat) +
            st_processing_record
    ))

    new_l2_mtl.add_group(
        'LEVEL2_SURFACE_REFLECTANCE_PARAMETERS', values=(
            reflectance_bands(sat.sr_band_range, l2_xml)
        ))

    # Setup for surface temperature band parameters
    values = ()
    if ("st" in l2_product_type):
        for band in sat.temp_band:
            values = values + tuple(surface_temp_bands(band, l2_xml,
                                                       sat.st_id))
        # Add this group when the product includes ST
        new_l2_mtl.add_group('LEVEL2_SURFACE_TEMPERATURE_PARAMETERS',
                             values=values)

    # Setup for brightness temperature bands parameters
    if ("bt" in l2_product_type):
        # Add this group when the product includes BT
        new_l2_mtl.add_group('LEVEL2_BRIGHTNESS_TEMPERATURE_PARAMETERS',
                             values=(bt_bands(sat.bt_band_range, l2_xml)))

    # Setup for TOA reflectance bands parameters
    if ("toa" in l2_product_type):
        # Add this group when the product includes TOA
        new_l2_mtl.add_group('LEVEL2_TOA_REFLECTANCE_PARAMETERS',
                             values=(reflectance_bands(sat.toa_band_range,
                                     l2_xml, True)))

    new_l1_mtl = ODL()

    new_l1_mtl.add_group('LEVEL1_PROCESSING_RECORD', values=(
        l1_mtl.get_group('LEVEL1_PROCESSING_RECORD'))
    )

    new_l1_mtl.add_group('LEVEL1_MIN_MAX_RADIANCE', values=(
        l1_mtl.get_group('LEVEL1_MIN_MAX_RADIANCE'))
    )

    new_l1_mtl.add_group('LEVEL1_MIN_MAX_REFLECTANCE', values=(
        l1_mtl.get_group('LEVEL1_MIN_MAX_REFLECTANCE'))
    )

    new_l1_mtl.add_group('LEVEL1_MIN_MAX_PIXEL_VALUE', values=(
        l1_mtl.get_group('LEVEL1_MIN_MAX_PIXEL_VALUE'))
    )

    new_l1_mtl.add_group('LEVEL1_RADIOMETRIC_RESCALING', values=(
        l1_mtl.get_group('LEVEL1_RADIOMETRIC_RESCALING'))
    )

    new_l1_mtl.add_group('LEVEL1_THERMAL_CONSTANTS', values=(
        l1_mtl.get_group('LEVEL1_THERMAL_CONSTANTS'))
    )

    new_l1_mtl.add_group('LEVEL1_PROJECTION_PARAMETERS', values=(
        l1_mtl.get_group('LEVEL1_PROJECTION_PARAMETERS'))
    )

    if sat.sensor != 'OLI_TIRS':
        new_l1_mtl.add_group('PRODUCT_PARAMETERS', values=(
            l1_mtl.get_group('PRODUCT_PARAMETERS'))
        )

    # Merge l1 MTL and l2 XML data with l2 XML overwriting any common fields
    l2_xml.data = join_ordered_dict(nested_dict_to_flat(l1_mtl.data),
                                    l2_xml.data)

    # Create l2 portion of new MTL file
    copy_data_to_mtl(new_l2_mtl.data, l2_xml, l2_name_changes)

    # Create the path to the new MTL file name
    file_name = os.path.join(output_dir, l2_mtl_filename)

    # Combine the l1 and l2 portions of the new MTL file
    final_mtl = ODL(file_name=file_name, separator='GROUP')
    final_mtl.add_group('LANDSAT_METADATA_FILE', values=(
        join_ordered_dict(new_l2_mtl.data, new_l1_mtl.data)))

    final_mtl.write()

    # Delete the l2 _gtif xml file
    os.unlink(l2_gtif_xml_filename)

    # Create the XML version of the level 2 MTL file
    command = ['convert_odl_metadata_to_xml.pl', file_name]
    subprocess.check_output(command, universal_newlines=True,
                            stderr=subprocess.STDOUT)
