# daily_data_read_format
#
# Read the format from the website of Sardegna Ambiente and
# write out CSV files with a table.
#
# Copyright 2021 Sustainable Sardinia
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pandas as pd
import numpy as np
import datetime


def _reindex_daily(data_series, year):
    """ Re-index a data series and fill missing measurements with nans
    """
    idx = pd.date_range('01-01-' + str(year), '12-31-' + str(year))
    data_series = data_series.reindex(idx, fill_value=np.nan)
    return data_series


def _reformat_monthly_data_by_day(montly_data, month_number, year):
    """ Reformat the data about a month into a time series by day
    """
    import math

    measurements = pd.to_numeric(montly_data.array, errors='coerce')
    day_of_month = pd.to_numeric(montly_data.index)

    new_measurements = []
    new_dates = []
    for meas, day in zip(measurements, day_of_month):
        # Remove the invalid or empty measurements
        if not math.isnan(meas):
            new_measurements.append(meas)
            new_dates.append(datetime.date(year, month_number, day+1))
    return pd.Series(new_measurements, index=new_dates)


def _convert_to_timeseries(daily_pollution_data, year):
    """ Convert the weird format used by the Sardegna Ambiente dataset
    """
    # Exclude the "day of the month" column
    pollution_data = daily_pollution_data.iloc[:, 1:]

    data_series = pd.Series(dtype='float64')

    # Iterate through the months
    month = 1
    for curr_column in pollution_data.columns:
        curr_data = pollution_data[curr_column]
        month_series = _reformat_monthly_data_by_day(curr_data, month, year)
        month = month+1

        data_series = data_series.append(month_series)

    data_series = _reindex_daily(data_series, year)
    return data_series


def _read_data_file(data_file_name, year):
    """ Common logic for reading the data """
    daily_pollution_data = pd.read_csv(
        "./data/daily_emissions/" + data_file_name, comment="#")
    return _convert_to_timeseries(daily_pollution_data, year)


def _convert_website_format_to_timeseries():
    """ Read the weird website format and write to a CSV file """
    data = pd.DataFrame(dtype='float64')

    station_data = [
        {'name': 'CENCA1', 'city': 'Cagliari', 'city_sardinian': 'Casteddu'},
        {'name': 'CENMO1', 'city': 'Monserrato', 'city_sardinian': 'Pauli'},
        {'name': 'CENQU1', 'city': "Quartu Sant'Elena",
            'city_sardinian': "Cuartu Sant' Aleni"},
        {'name': 'CENSA2', 'city': 'Sarroch', 'city_sardinian': 'Sarrocu'}]

    for year in range(2018, 2022):
        for station in station_data:
            station_name = station['name']
            for pollutant in ["PM2.5", "PM10"]:
                data_file_name = station_name + "-Anno-" + \
                    str(year) + "-" + pollutant + ".csv"
                try:
                    series = _read_data_file(data_file_name, year)

                    frame = series.to_frame(name='value') \
                        .assign(station_name=station_name) \
                        .assign(city=station['city']) \
                        .assign(city_sardinian=station['city_sardinian']) \
                        .assign(pollutant=pollutant) \
                        .assign(timestamp=series.index) \
                        .dropna()
                    data = data.append(frame)
                except:
                    pass
    return data.sort_values(by=['timestamp', 'station_name', 'pollutant'])


def _filter_data(data, filter_function):
    data = data.sort_values(['station_name', 'pollutant', 'timestamp'])
    label = data['timestamp'].map(filter_function)
    data = data.assign(label=label)
    data = data.reset_index(drop=True)
    filtered_data = data.loc[data.groupby(
        ['label', 'station_name', 'pollutant'])['value'].idxmax()].drop_duplicates()
    filtered_data = filtered_data.drop('label', 1)
    return filtered_data.sort_values(['timestamp', 'station_name', 'pollutant'])


def _filter_data_weekly(data):
    return _filter_data(data, lambda t: str(t.isocalendar()[0]).zfill(4) + str(t.isocalendar()[1]).zfill(3))


def _filter_data_monthly(data):
    return _filter_data(data, lambda t: str(t.year).zfill(4) + str(t.month).zfill(3))


if __name__ == '__main__':
    data = _convert_website_format_to_timeseries()
    data['timestamp'] = pd.to_datetime(data['timestamp'])
    print(data.head())
    data.to_csv('./data/daily_emissions/daily-emissions.csv', index=False)
    filtered_data = _filter_data_weekly(data)
    print(filtered_data.head())
    filtered_data.to_csv(
        './data/daily_emissions/daily-emissions-weekly-peaks.csv', index=False)
    filtered_data = _filter_data_monthly(data)
    print(filtered_data.head())
    filtered_data.to_csv(
        './data/daily_emissions/daily-emissions-monthly-peaks.csv', index=False)

# daily_data_read_format
#
# Read the format from the website of Sardegna Ambiente and
# write out CSV files with a table.
#
# Copyright 2021 Sustainable Sardinia
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pandas as pd
import numpy as np
import datetime


def _reindex_daily(data_series, year):
    """ Re-index a data series and fill missing measurements with nans
    """
    idx = pd.date_range('01-01-' + str(year), '12-31-' + str(year))
    data_series = data_series.reindex(idx, fill_value=np.nan)
    return data_series


def _reformat_monthly_data_by_day(montly_data, month_number, year):
    """ Reformat the data about a month into a time series by day
    """
    import math

    measurements = pd.to_numeric(montly_data.array, errors='coerce')
    day_of_month = pd.to_numeric(montly_data.index)

    new_measurements = []
    new_dates = []
    for meas, day in zip(measurements, day_of_month):
        # Remove the invalid or empty measurements
        if not math.isnan(meas):
            new_measurements.append(meas)
            new_dates.append(datetime.date(year, month_number, day+1))
    return pd.Series(new_measurements, index=new_dates)


def _convert_to_timeseries(daily_pollution_data, year):
    """ Convert the weird format used by the Sardegna Ambiente dataset
    """
    # Exclude the "day of the month" column
    pollution_data = daily_pollution_data.iloc[:, 1:]

    data_series = pd.Series(dtype='float64')

    # Iterate through the months
    month = 1
    for curr_column in pollution_data.columns:
        curr_data = pollution_data[curr_column]
        month_series = _reformat_monthly_data_by_day(curr_data, month, year)
        month = month+1

        data_series = data_series.append(month_series)

    data_series = _reindex_daily(data_series, year)
    return data_series


def _read_data_file(data_file_name, year):
    """ Common logic for reading the data """
    daily_pollution_data = pd.read_csv(
        "./data/daily_emissions/" + data_file_name, comment="#")
    return _convert_to_timeseries(daily_pollution_data, year)


def _convert_website_format_to_timeseries():
    """ Read the weird website format and write to a CSV file """
    data = pd.DataFrame(dtype='float64')

    station_data = [
        {'name': 'CENCA1', 'city': 'Cagliari', 'city_sardinian': 'Casteddu'},
        {'name': 'CENMO1', 'city': 'Monserrato', 'city_sardinian': 'Pauli'},
        {'name': 'CENQU1', 'city': "Quartu Sant'Elena",
            'city_sardinian': "Cuartu Sant' Aleni"},
        {'name': 'CENSA2', 'city': 'Sarroch', 'city_sardinian': 'Sarrocu'}]

    for year in range(2018, 2022):
        for station in station_data:
            station_name = station['name']
            for pollutant in ["PM2.5", "PM10"]:
                data_file_name = station_name + "-Anno-" + \
                    str(year) + "-" + pollutant + ".csv"
                try:
                    series = _read_data_file(data_file_name, year)

                    frame = series.to_frame(name='value') \
                        .assign(station_name=station_name) \
                        .assign(city=station['city']) \
                        .assign(city_sardinian=station['city_sardinian']) \
                        .assign(pollutant=pollutant) \
                        .assign(timestamp=series.index) \
                        .dropna()
                    data = data.append(frame)
                except:
                    pass
    return data.sort_values(by=['timestamp', 'station_name', 'pollutant'])


def _filter_data(data, filter_function):
    data = data.sort_values(['station_name', 'pollutant', 'timestamp'])
    label = data['timestamp'].map(filter_function)
    data = data.assign(label=label)
    data = data.reset_index(drop=True)
    filtered_data = data.loc[data.groupby(
        ['label', 'station_name', 'pollutant'])['value'].idxmax()].drop_duplicates()
    filtered_data = filtered_data.drop('label', 1)
    return filtered_data.sort_values(['timestamp', 'station_name', 'pollutant'])


def _filter_data_weekly(data):
    return _filter_data(data, lambda t: str(t.isocalendar()[0]).zfill(4) + str(t.isocalendar()[1]).zfill(3))


def _filter_data_monthly(data):
    return _filter_data(data, lambda t: str(t.year).zfill(4) + str(t.month).zfill(3))


if __name__ == '__main__':
    data = _convert_website_format_to_timeseries()
    data['timestamp'] = pd.to_datetime(data['timestamp'])
    print(data.head())
    data.to_csv('./data/daily_emissions/daily-emissions.csv', index=False)
    filtered_data = _filter_data_weekly(data)
    print(filtered_data.head())
    filtered_data.to_csv(
        './data/daily_emissions/daily-emissions-weekly-peaks.csv', index=False)
    filtered_data = _filter_data_monthly(data)
    print(filtered_data.head())
    filtered_data.to_csv(
        './data/daily_emissions/daily-emissions-monthly-peaks.csv', index=False)

# daily_data_read_format # # Read the format from the website of Sardegna Ambiente and # write out CSV files with a table. # # Copyright 2021 Sustainable Sardinia # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. Re-index a data series and fill missing measurements with nans Reformat the data about a month into a time series by day # Remove the invalid or empty measurements Convert the weird format used by the Sardegna Ambiente dataset # Exclude the "day of the month" column # Iterate through the months Common logic for reading the data Read the weird website format and write to a CSV file