# -*- coding:  utf-8 -*-
"""
Created on Thurs 5 27 2021
@name:    USCensus ACS Migration Download Application
@author:  <NAME>

"""

import sys
import os.path
import time
import warnings
import logging
import pandas as pd

MAIN_DIR = os.path.dirname(os.path.realpath(__file__))
ROOT_DIR = os.path.abspath(os.path.join(MAIN_DIR, os.pardir))
SAVE_DIR = os.path.join(ROOT_DIR, "save")
RESOURCE_DIR = os.path.join(ROOT_DIR, "resources")
REPOSITORY_DIR = os.path.join(SAVE_DIR, "uscensus")
REPORT_FILE = os.path.join(REPOSITORY_DIR, "migrate.csv")
AUTHENTICATE_FILE = os.path.join(RESOURCE_DIR, "authenticate.csv")
if ROOT_DIR not in sys.path:
    sys.path.append(ROOT_DIR)

from utilities.iostream import InputParser
from utilities.dataframes import dataframe_fromfile, dataframe_parser
from webscraping.webtimers import WebDelayer
from webscraping.webreaders import WebReader, Retrys
from webscraping.weburl import WebURL
from webscraping.webpages import WebJsonPage
from webscraping.webquerys import WebQuery, WebDatasets
from webscraping.webqueues import WebScheduler, WebQueueable
from webscraping.webdownloaders import WebDownloader, CacheMixin
from webscraping.webvariables import Geography

__version__ = "1.0.0"
__author__ = "<NAME>"
__all__ = ["USCensus_ACSM_WebDelayer", "USCensus_ACSM_WebDownloader", "USCensus_ACSM_WebScheduler"]
__copyright__ = "Copyright 2020, <NAME>"
__license__ = ""


LOGGER = logging.getLogger(__name__)
warnings.filterwarnings("ignore")    


AUTHENTICATE = dataframe_parser(dataframe_fromfile(AUTHENTICATE_FILE, index="website", header=0), parsers={}, default=str).transpose()
TAGS = {"growth": "MOVEDNET", "entering": "MOVEDIN", "exiting": "MOVEDOUT", "targetname": "FULL2_NAME", "targetstate": "STATE2", "targetcounty": "COUNTY2", "name": "FULL1_NAME"}
GEOGRAPHYS = {"state": "state", "county": "county", "subdivision": "county subdivision"}
STATES = {
    "AL": "Alabama", "AK": "Alaska","AZ": "Arizona", "AR": "Arkansas", "CA": "California", "CO": "Colorado", "CT": "Connecticut", "DE": "Delaware", "FL": "Florida", "GA": "Georgia", 
    "HI": "Hawaii", "ID": "Idaho", "IL": "Illinois", "IN": "Indiana", "IA": "Iowa", "KS": "Kansas", "KY": "Kentucky", "LA": "Louisiana", "ME": "Maine", "MD": "Maryland", "MA": "Massachusetts", 
    "MI": "Michigan", "MN": "Minnesota", "MS": "Mississippi", "MO": "Missouri", "MT": "Montana", "NE": "Nebraska", "NV": "Nevada", "NH": "New Hampshire", "NJ": "New Jersey", "NM": "New Mexico", 
    "NY": "New York", "NC": "North Carolina", "ND": "North Dakota", "OH": "Ohio", "OK": "Oklahoma", "OR": "Oregon", "PA": "Pennsylvania", "RI": "Rhode Island", "SC": "South Carolina", "SD": "South Dakota", 
    "TN": "Tennessee", "TX": "Texas", "UT": "Utah", "VT": "Vermont", "VA": "Virginia", "WA": "Washington", "WV": "West Virginia", "WI": "Wisconsin", "WY": "Wyoming"}


_inverted = lambda x: {v: k for k, v in x.items()}
_filter = lambda x: [i for i in x if i is not None]
_range = lambda x: [str(i) for i in range(int(x.split("-")[0]), int(x.split("-")[1])+1)]
_list = lambda x: [str(i) for i in str(x).split("|")]
_state = lambda x: str(x) if str(x) in STATES.values() else STATES[str(x)]
_county = lambda x: " ".join([str(x), "County"]) if not str(x).endswith(" County") else str(x)
_tag = lambda *args: {"get": "{}".format(",".join(list(args)))}
_forgeo = lambda **kwargs: {"for": "{}".format("%20".join([": ".join([key, value]) for key, value in kwargs.items()]))} if kwargs else {}
_ingeo = lambda **kwargs: {"in": "{}".format("%20".join([": ".join([key, value]) for key, value in kwargs.items()]))} if kwargs else {}
_apikey = lambda apikey: {"key": "{}".format(str(apikey))}


class USCensus_ACSM_WebDelayer(WebDelayer): pass
class USCensus_ACSM_WebReader(WebReader, retrys=Retrys(retries=3, backoff=0.3, httpcodes=(500, 502, 504)), authenticate=None): pass


class USCensus_ACSM_WebURL(WebURL, protocol="https", domain="api.census.gov", spaceproxy="%20"):
    def path(self, *args, dataset, date, **kwargs):
        return ["data", str(date), "acs", str(dataset)]

    def parm(self, *args, tags, geography, apikey, **kwargs):
        assert isinstance(geography, Geography)
        forgeo = {GEOGRAPHYS[key].replace(" ", self.spaceproxy): (value.replace(" ", self.spaceproxy) if value is not None else "*") for key, name, value in geography[-1].items()}
        ingeo = {GEOGRAPHYS[key].replace(" ", self.spaceproxy): (value.replace(" ", self.spaceproxy) if value is not None else "*") for key, name, value in geography[:-1].items()}
        return {**_tag(*tags), **_forgeo(**forgeo), **_ingeo(**ingeo), **_apikey(apikey)}


class USCensus_ACSM_WebQuery(WebQuery, WebQueueable, fields=["date", "state", "county"]): pass
class USCensus_ACSM_WebDatasets(WebDatasets, fields=["{}_{}_{}".format("household", "geography", "migration")]): pass


class USCensus_ACSM_WebScheduler(WebScheduler, fields=["date", "state", "county"]):
    @staticmethod
    def date(*args, date=None, dates=[], **kwargs): return [str(item) for item in [date, *dates] if item]
    @staticmethod
    def state(*args, state, **kwargs): return [_state(state)]
    @staticmethod
    def county(*args, county=None, countys=[], **kwargs): return [_county(item) for item in [county, *countys] if item]
    @staticmethod
    def execute(querys, *args, **kwargs): return [USCensus_ACSM_WebQuery(query) for query in querys]


class USCensus_WebPage(WebJsonPage):
    def __init__(self, *args, apikey, **kwargs):
        self.__apikey = apikey
        super().__init__(*args, **kwargs)

    @property
    def apikey(self): return self.__apikey
    def execute(self, *args, **kwargs): pass


class USCensus_ACSMGeography_WebPage(USCensus_WebPage):
    def run(self, *args, state, county, date, **kwargs):
        geography = Geography(keys=["state"], names=[None], values=[None])
        url = USCensus_ACSM_WebURL(dataset="acs5", tags=["NAME"], geography=geography, date=date, apikey=self.apikey)
        self.load(url)
        states = self.parser(self.source)
        geography = Geography(keys=["state", "county"], names=[state, None], values=[states[state], None])
        url = USCensus_ACSM_WebURL(dataset="acs5", tags=["NAME"], geography=geography, date=date, apikey=self.apikey)
        self.load(url)
        countys = self.parser(self.source)
        geography = Geography(keys=["state", "county", "subdivision"], names=[state, county, None], values=[states[state], countys[county], None])
        url = USCensus_ACSM_WebURL(dataset="acs5", tags=["NAME"], geography=geography, date=date, apikey=self.apikey)
        self.load(url)
        subdivisions = self.parser(self.source)
        geographys = [Geography(keys=["state", "county", "subdivision"], names=[state, county, name], values=[states[state], countys[county], value]) for name, value in subdivisions.items()]
        return geographys

    @staticmethod
    def parser(data): return {item[0].split(", ")[0]: item[-1] for item in list(data)[1:]}


class USCensus_ACSMData_WebPage(USCensus_WebPage):
    def run(self, *args, geography, date, **kwargs):
        url = USCensus_ACSM_WebURL(dataset="flows", tags=list(TAGS.values()), geography=geography, date=date, apikey=self.apikey)
        self.load(url)
        dataframe = self.parser(self.source)
        dataframe = self.geography(dataframe)
        dataframe = self.variables(dataframe)
        dataframe["date"] = date
        return dataframe

    @staticmethod
    def parser(data): return pd.DataFrame(data=list(data)[1:], columns=list(data)[0])

    @staticmethod
    def geography(dataframe):
        dataframe.rename(_inverted(GEOGRAPHYS), axis=1, inplace=True)
        function = lambda x, keys, names, values:  str(Geography(keys=keys, names=x.to_dict()[names].split(", ")[::-1], values=[x.to_dict()[value] for value in values]))
        dataframe["geography"] = dataframe.apply(function, result_type="reduce", axis=1, args=(["state", "county", "subdivision"], "name", ["state", "county", "subdivision"]))
        dataframe["target"] = dataframe.apply(function, result_type="reduce", axis=1, args=(["state", "county"], "targetname", ["targetstate", "targetcounty"]))
        return dataframe

    @staticmethod
    def variables(dataframe):
        dataframe.rename(_inverted(TAGS), axis=1, inplace=True)
        dataframe = dataframe[["growth", "entering", "exiting", "geography", "target"]]
        return dataframe


class USCensus_ACSM_WebDownloader(WebDownloader + CacheMixin):
    @staticmethod
    def execute(*args, queue, delayer, apikey, **kwargs):
        with USCensus_ACSM_WebReader() as session:
            uscensus_geography = USCensus_ACSMGeography_WebPage(session, apikey=apikey, delayer=delayer)
            uscensus_data = USCensus_ACSMData_WebPage(session, apikey=apikey, delayer=delayer)
            dataset = "{}_{}_{}".format("household", "geography", "migration")
            with queue:
                for query in iter(queue):
                    with query:
                        for geography in uscensus_geography(session, **query.todict()):
                            data = uscensus_data(geography=geography, date=query["date"])
                            yield query, USCensus_ACSM_WebDatasets({dataset, data})


def main(*args, **kwargs):  
    delayer = USCensus_ACSM_WebDelayer("constant", wait=15)
    scheduler = USCensus_ACSM_WebScheduler(*args, file=REPORT_FILE, **kwargs)
    downloader = USCensus_ACSM_WebDownloader(*args, repository=REPOSITORY_DIR, **kwargs)
    queue = scheduler(*args, **kwargs)
    apikey = AUTHENTICATE.to_dict()["uscensus"]["apikey"]
    downloader(*args, queue=queue, delayer=delayer, apikey=apikey, **kwargs)
    while True:  
        if downloader.off:
            break
        if downloader.error:
            break
        time.sleep(15)
    LOGGER.info(str(downloader))
    for query, results in downloader.results:
        LOGGER.info(str(query))
        LOGGER.info(str(results))
    if not bool(downloader):
        raise downloader.error


if __name__ == "__main__":
    sys.argv += ["state=CA", "county=Kern", "date=2019"]
    logging.basicConfig(level="INFO", format="[%(levelname)s, %(threadName)s]:   %(message)s", handlers=[logging.StreamHandler(sys.stdout)])
    parsers = {"dates": _range, "state": lambda x: _state(x), "county": lambda x: _county(x), "countys": lambda x: [_county(i) for i in _list(x)]}
    inputparser = InputParser(proxys={"assign": "=", "space": "_"}, parsers=parsers, default=str)
    inputparser(*sys.argv[1:])
    main(*inputparser.arguments, **inputparser.parameters)
    

    
# -*- coding:  utf-8 -*-
"""
Created on Thurs 5 27 2021
@name:    USCensus ACS Migration Download Application
@author:  <NAME>

"""

import sys
import os.path
import time
import warnings
import logging
import pandas as pd

MAIN_DIR = os.path.dirname(os.path.realpath(__file__))
ROOT_DIR = os.path.abspath(os.path.join(MAIN_DIR, os.pardir))
SAVE_DIR = os.path.join(ROOT_DIR, "save")
RESOURCE_DIR = os.path.join(ROOT_DIR, "resources")
REPOSITORY_DIR = os.path.join(SAVE_DIR, "uscensus")
REPORT_FILE = os.path.join(REPOSITORY_DIR, "migrate.csv")
AUTHENTICATE_FILE = os.path.join(RESOURCE_DIR, "authenticate.csv")
if ROOT_DIR not in sys.path:
    sys.path.append(ROOT_DIR)

from utilities.iostream import InputParser
from utilities.dataframes import dataframe_fromfile, dataframe_parser
from webscraping.webtimers import WebDelayer
from webscraping.webreaders import WebReader, Retrys
from webscraping.weburl import WebURL
from webscraping.webpages import WebJsonPage
from webscraping.webquerys import WebQuery, WebDatasets
from webscraping.webqueues import WebScheduler, WebQueueable
from webscraping.webdownloaders import WebDownloader, CacheMixin
from webscraping.webvariables import Geography

__version__ = "1.0.0"
__author__ = "<NAME>"
__all__ = ["USCensus_ACSM_WebDelayer", "USCensus_ACSM_WebDownloader", "USCensus_ACSM_WebScheduler"]
__copyright__ = "Copyright 2020, <NAME>"
__license__ = ""


LOGGER = logging.getLogger(__name__)
warnings.filterwarnings("ignore")    


AUTHENTICATE = dataframe_parser(dataframe_fromfile(AUTHENTICATE_FILE, index="website", header=0), parsers={}, default=str).transpose()
TAGS = {"growth": "MOVEDNET", "entering": "MOVEDIN", "exiting": "MOVEDOUT", "targetname": "FULL2_NAME", "targetstate": "STATE2", "targetcounty": "COUNTY2", "name": "FULL1_NAME"}
GEOGRAPHYS = {"state": "state", "county": "county", "subdivision": "county subdivision"}
STATES = {
    "AL": "Alabama", "AK": "Alaska","AZ": "Arizona", "AR": "Arkansas", "CA": "California", "CO": "Colorado", "CT": "Connecticut", "DE": "Delaware", "FL": "Florida", "GA": "Georgia", 
    "HI": "Hawaii", "ID": "Idaho", "IL": "Illinois", "IN": "Indiana", "IA": "Iowa", "KS": "Kansas", "KY": "Kentucky", "LA": "Louisiana", "ME": "Maine", "MD": "Maryland", "MA": "Massachusetts", 
    "MI": "Michigan", "MN": "Minnesota", "MS": "Mississippi", "MO": "Missouri", "MT": "Montana", "NE": "Nebraska", "NV": "Nevada", "NH": "New Hampshire", "NJ": "New Jersey", "NM": "New Mexico", 
    "NY": "New York", "NC": "North Carolina", "ND": "North Dakota", "OH": "Ohio", "OK": "Oklahoma", "OR": "Oregon", "PA": "Pennsylvania", "RI": "Rhode Island", "SC": "South Carolina", "SD": "South Dakota", 
    "TN": "Tennessee", "TX": "Texas", "UT": "Utah", "VT": "Vermont", "VA": "Virginia", "WA": "Washington", "WV": "West Virginia", "WI": "Wisconsin", "WY": "Wyoming"}


_inverted = lambda x: {v: k for k, v in x.items()}
_filter = lambda x: [i for i in x if i is not None]
_range = lambda x: [str(i) for i in range(int(x.split("-")[0]), int(x.split("-")[1])+1)]
_list = lambda x: [str(i) for i in str(x).split("|")]
_state = lambda x: str(x) if str(x) in STATES.values() else STATES[str(x)]
_county = lambda x: " ".join([str(x), "County"]) if not str(x).endswith(" County") else str(x)
_tag = lambda *args: {"get": "{}".format(",".join(list(args)))}
_forgeo = lambda **kwargs: {"for": "{}".format("%20".join([": ".join([key, value]) for key, value in kwargs.items()]))} if kwargs else {}
_ingeo = lambda **kwargs: {"in": "{}".format("%20".join([": ".join([key, value]) for key, value in kwargs.items()]))} if kwargs else {}
_apikey = lambda apikey: {"key": "{}".format(str(apikey))}


class USCensus_ACSM_WebDelayer(WebDelayer): pass
class USCensus_ACSM_WebReader(WebReader, retrys=Retrys(retries=3, backoff=0.3, httpcodes=(500, 502, 504)), authenticate=None): pass


class USCensus_ACSM_WebURL(WebURL, protocol="https", domain="api.census.gov", spaceproxy="%20"):
    def path(self, *args, dataset, date, **kwargs):
        return ["data", str(date), "acs", str(dataset)]

    def parm(self, *args, tags, geography, apikey, **kwargs):
        assert isinstance(geography, Geography)
        forgeo = {GEOGRAPHYS[key].replace(" ", self.spaceproxy): (value.replace(" ", self.spaceproxy) if value is not None else "*") for key, name, value in geography[-1].items()}
        ingeo = {GEOGRAPHYS[key].replace(" ", self.spaceproxy): (value.replace(" ", self.spaceproxy) if value is not None else "*") for key, name, value in geography[:-1].items()}
        return {**_tag(*tags), **_forgeo(**forgeo), **_ingeo(**ingeo), **_apikey(apikey)}


class USCensus_ACSM_WebQuery(WebQuery, WebQueueable, fields=["date", "state", "county"]): pass
class USCensus_ACSM_WebDatasets(WebDatasets, fields=["{}_{}_{}".format("household", "geography", "migration")]): pass


class USCensus_ACSM_WebScheduler(WebScheduler, fields=["date", "state", "county"]):
    @staticmethod
    def date(*args, date=None, dates=[], **kwargs): return [str(item) for item in [date, *dates] if item]
    @staticmethod
    def state(*args, state, **kwargs): return [_state(state)]
    @staticmethod
    def county(*args, county=None, countys=[], **kwargs): return [_county(item) for item in [county, *countys] if item]
    @staticmethod
    def execute(querys, *args, **kwargs): return [USCensus_ACSM_WebQuery(query) for query in querys]


class USCensus_WebPage(WebJsonPage):
    def __init__(self, *args, apikey, **kwargs):
        self.__apikey = apikey
        super().__init__(*args, **kwargs)

    @property
    def apikey(self): return self.__apikey
    def execute(self, *args, **kwargs): pass


class USCensus_ACSMGeography_WebPage(USCensus_WebPage):
    def run(self, *args, state, county, date, **kwargs):
        geography = Geography(keys=["state"], names=[None], values=[None])
        url = USCensus_ACSM_WebURL(dataset="acs5", tags=["NAME"], geography=geography, date=date, apikey=self.apikey)
        self.load(url)
        states = self.parser(self.source)
        geography = Geography(keys=["state", "county"], names=[state, None], values=[states[state], None])
        url = USCensus_ACSM_WebURL(dataset="acs5", tags=["NAME"], geography=geography, date=date, apikey=self.apikey)
        self.load(url)
        countys = self.parser(self.source)
        geography = Geography(keys=["state", "county", "subdivision"], names=[state, county, None], values=[states[state], countys[county], None])
        url = USCensus_ACSM_WebURL(dataset="acs5", tags=["NAME"], geography=geography, date=date, apikey=self.apikey)
        self.load(url)
        subdivisions = self.parser(self.source)
        geographys = [Geography(keys=["state", "county", "subdivision"], names=[state, county, name], values=[states[state], countys[county], value]) for name, value in subdivisions.items()]
        return geographys

    @staticmethod
    def parser(data): return {item[0].split(", ")[0]: item[-1] for item in list(data)[1:]}


class USCensus_ACSMData_WebPage(USCensus_WebPage):
    def run(self, *args, geography, date, **kwargs):
        url = USCensus_ACSM_WebURL(dataset="flows", tags=list(TAGS.values()), geography=geography, date=date, apikey=self.apikey)
        self.load(url)
        dataframe = self.parser(self.source)
        dataframe = self.geography(dataframe)
        dataframe = self.variables(dataframe)
        dataframe["date"] = date
        return dataframe

    @staticmethod
    def parser(data): return pd.DataFrame(data=list(data)[1:], columns=list(data)[0])

    @staticmethod
    def geography(dataframe):
        dataframe.rename(_inverted(GEOGRAPHYS), axis=1, inplace=True)
        function = lambda x, keys, names, values:  str(Geography(keys=keys, names=x.to_dict()[names].split(", ")[::-1], values=[x.to_dict()[value] for value in values]))
        dataframe["geography"] = dataframe.apply(function, result_type="reduce", axis=1, args=(["state", "county", "subdivision"], "name", ["state", "county", "subdivision"]))
        dataframe["target"] = dataframe.apply(function, result_type="reduce", axis=1, args=(["state", "county"], "targetname", ["targetstate", "targetcounty"]))
        return dataframe

    @staticmethod
    def variables(dataframe):
        dataframe.rename(_inverted(TAGS), axis=1, inplace=True)
        dataframe = dataframe[["growth", "entering", "exiting", "geography", "target"]]
        return dataframe


class USCensus_ACSM_WebDownloader(WebDownloader + CacheMixin):
    @staticmethod
    def execute(*args, queue, delayer, apikey, **kwargs):
        with USCensus_ACSM_WebReader() as session:
            uscensus_geography = USCensus_ACSMGeography_WebPage(session, apikey=apikey, delayer=delayer)
            uscensus_data = USCensus_ACSMData_WebPage(session, apikey=apikey, delayer=delayer)
            dataset = "{}_{}_{}".format("household", "geography", "migration")
            with queue:
                for query in iter(queue):
                    with query:
                        for geography in uscensus_geography(session, **query.todict()):
                            data = uscensus_data(geography=geography, date=query["date"])
                            yield query, USCensus_ACSM_WebDatasets({dataset, data})


def main(*args, **kwargs):  
    delayer = USCensus_ACSM_WebDelayer("constant", wait=15)
    scheduler = USCensus_ACSM_WebScheduler(*args, file=REPORT_FILE, **kwargs)
    downloader = USCensus_ACSM_WebDownloader(*args, repository=REPOSITORY_DIR, **kwargs)
    queue = scheduler(*args, **kwargs)
    apikey = AUTHENTICATE.to_dict()["uscensus"]["apikey"]
    downloader(*args, queue=queue, delayer=delayer, apikey=apikey, **kwargs)
    while True:  
        if downloader.off:
            break
        if downloader.error:
            break
        time.sleep(15)
    LOGGER.info(str(downloader))
    for query, results in downloader.results:
        LOGGER.info(str(query))
        LOGGER.info(str(results))
    if not bool(downloader):
        raise downloader.error


if __name__ == "__main__":
    sys.argv += ["state=CA", "county=Kern", "date=2019"]
    logging.basicConfig(level="INFO", format="[%(levelname)s, %(threadName)s]:   %(message)s", handlers=[logging.StreamHandler(sys.stdout)])
    parsers = {"dates": _range, "state": lambda x: _state(x), "county": lambda x: _county(x), "countys": lambda x: [_county(i) for i in _list(x)]}
    inputparser = InputParser(proxys={"assign": "=", "space": "_"}, parsers=parsers, default=str)
    inputparser(*sys.argv[1:])
    main(*inputparser.arguments, **inputparser.parameters)
    

    
# -*- coding:  utf-8 -*- Created on Thurs 5 27 2021 @name:    USCensus ACS Migration Download Application @author:  <NAME>