# AUTOGENERATED! DO NOT EDIT! File to edit: 00_loadarticles.ipynb (unless otherwise specified).

__all__ = ['article_holder', 'article_holder', 'CoverageTrendsLoader', 'dailysourcepermalinksLoader', 'article_holder',
           'article_holder']

# Cell
import pandas as pd
import os, datetime, re
from nltk.stem.snowball import SnowballStemmer
from sklearn.feature_extraction import text
import nltk
import json

# Cell
class article_holder():
    "Basic unit to keep load and analze my articles"

    def __init__(self):
        self.articleDir = None
        self.df = None

        try:
            stopwords = text.ENGLISH_STOP_WORDS
        except:
            print("stopwords not found, downloading")
            nltk.download('stopwords')



    extra_stopwords = ["news", "say", "said", "told", "tell", "day", "video", "week", "state", "new", "york", "times"]
    stopwords = text.ENGLISH_STOP_WORDS.union(extra_stopwords)

    subclass="article_holder"

# Cell
class article_holder(article_holder):

    def set_articleDir(self, path): self.articleDir = path
    def get_articleDir(self): return self.articleDir



# Cell
def CoverageTrendsLoader(publications:[str] = [], path=".", dateStart:str=None, dateEnd:str=None, lastN:int=None, verbose=False, **kwargs) -> []:

    """
    Turns CSV of scraped headlines from CoverageTrends into a Pandas Dataframe.
    Expects that CoverageTrends (https://github.com/brockmanmatt/CoverageTrends) is cloned into ../CoverageTrends

    Parameters

    publications: list of publications to try to pull from CoverageTrends CSV, all if []

    dateStart: String YYYYMMDD for first date of CSV to load for each publication

    dateEnd: String YYYYMMDD for last date of CSV to load for each publication

    lastN: get max (available days, lastN) days

    """

    "Engine to load articles from CoverageTrends GitHub repo"
    if "archived_links" not in os.listdir(path):
        missingCoverageTrends="CoverageTrends engine requires CoverageTrends"
        missingCoverageTrends+="\nPlease clone https://github.com/brockmanmatt/CoverageTrends to use this option"
        raise Exception(missingCoverageTrends)

    "Make list of publications that have scraped lists"
    availablePublications = [x for x in os.listdir("{}/archived_links".format(path)) if x.find(".") ==-1]

    "If publications are limited, then only go with those"
    if len(publications) > 0:
        availablePublications = [x for x in publications if x in availablePublications]

    loaded_articles = []

    "Loop through each publisher in CoverageTrends and load each day"
    for publisher in availablePublications:

        csvPaths = []

        pubPath = "{}/{}".format("{}/archived_links".format(path), publisher)
        for month in os.listdir(pubPath):
            if month.find(".") > -1:
                continue
            monthPath = "{}/{}".format(pubPath, month)
            for day in os.listdir(monthPath):
                if dateStart != None:
                    if int(day.split("_")[1][:-4]) < int(dateStart):
                        continue
                if dateEnd != None:
                    if int(day.split("_")[1][:-4]) > int(dateEnd):
                        continue
                csvPaths.append("{}/{}".format(monthPath, day))

        csvPaths = sorted(csvPaths)

        if lastN != None:
            csvPaths = csvPaths[-lastN:]

        csvPaths = pd.concat([pd.read_csv(x) for x in csvPaths], ignore_index=True)
        csvPaths["source"] = publisher
        loaded_articles.append(csvPaths)

    return pd.concat(loaded_articles).fillna("")


# Cell
def dailysourcepermalinksLoader(path="", filename="", **kwargs):
    "returns df of a json file which is date indexed with publishers for columns"

    df = pd.read_json("{}/{}".format(path, filename)).T
    df["date"] = df.index
    df = df.melt(id_vars = ["date"], var_name="Source", value_name="text") #make big column
    return df

# Cell
class article_holder(article_holder):
    "gives article_holder ability to load articles for publication between dateStart and dateEnd"

    def load_articles(self, engine=CoverageTrendsLoader, publications:[str] = [], dateStart:str=None, dateEnd:str=None, lastN:int=None, verbose=False, **kwargs) -> []:
        if self.articleDir == None:
            raise Exception("holder missing path")

        self.df = engine(path=self.articleDir, publications=publications, dateStart=dateStart, dateEnd=dateEnd, lastN=lastN, verbose=verbose, **kwargs)

        tmp = pd.DataFrame([self.df.text.unique()]).T
        tmp.columns=["text"]
        stemmer = SnowballStemmer("english", ignore_stopwords=True)
        tmp["quickReplace"] = tmp["text"].fillna("").apply(lambda x: re.sub('[^a-z]+', " ", x.lower()))
        tmp["tokens"] = tmp["quickReplace"].apply(lambda x: [stemmer.stem(y) for y in x.split() if len (y) > 0])
        tmp["quickReplace"] = tmp["tokens"].apply(lambda x: " ".join(x))

        self.df = self.df.merge(tmp)

# Cell
class article_holder(article_holder):
    "article_holders should be able to load dfs from other article_holders"

    def load_article_holder(self, other_article_holder:article_holder):
        errorMessage = "Bad format"

        try:
            missing = []
            otherColumns = other_article_holder.df.columns
            if "quickReplace" not in otherColumns:
                missing.append("quickReplace")
            if "tokens" not in otherColumns:
                missing.append("tokens")
            if "text" not in otherColumns:
                missing.append("text")
            if "source" not in otherColumns:
                missing.append("source")
            if len(missing) > 0:
                errorMessage = "Missing {}".format(";".join(missing))
                raise Exception("error")

        except:
            raise Exception("Passed article holder structure not recognized: {}".format(errorMessage))

        self.df = other_article_holder.df.copy()
# AUTOGENERATED! DO NOT EDIT! File to edit: 00_loadarticles.ipynb (unless otherwise specified).

__all__ = ['article_holder', 'article_holder', 'CoverageTrendsLoader', 'dailysourcepermalinksLoader', 'article_holder',
           'article_holder']

# Cell
import pandas as pd
import os, datetime, re
from nltk.stem.snowball import SnowballStemmer
from sklearn.feature_extraction import text
import nltk
import json

# Cell
class article_holder():
    "Basic unit to keep load and analze my articles"

    def __init__(self):
        self.articleDir = None
        self.df = None

        try:
            stopwords = text.ENGLISH_STOP_WORDS
        except:
            print("stopwords not found, downloading")
            nltk.download('stopwords')



    extra_stopwords = ["news", "say", "said", "told", "tell", "day", "video", "week", "state", "new", "york", "times"]
    stopwords = text.ENGLISH_STOP_WORDS.union(extra_stopwords)

    subclass="article_holder"

# Cell
class article_holder(article_holder):

    def set_articleDir(self, path): self.articleDir = path
    def get_articleDir(self): return self.articleDir



# Cell
def CoverageTrendsLoader(publications:[str] = [], path=".", dateStart:str=None, dateEnd:str=None, lastN:int=None, verbose=False, **kwargs) -> []:

    """
    Turns CSV of scraped headlines from CoverageTrends into a Pandas Dataframe.
    Expects that CoverageTrends (https://github.com/brockmanmatt/CoverageTrends) is cloned into ../CoverageTrends

    Parameters

    publications: list of publications to try to pull from CoverageTrends CSV, all if []

    dateStart: String YYYYMMDD for first date of CSV to load for each publication

    dateEnd: String YYYYMMDD for last date of CSV to load for each publication

    lastN: get max (available days, lastN) days

    """

    "Engine to load articles from CoverageTrends GitHub repo"
    if "archived_links" not in os.listdir(path):
        missingCoverageTrends="CoverageTrends engine requires CoverageTrends"
        missingCoverageTrends+="\nPlease clone https://github.com/brockmanmatt/CoverageTrends to use this option"
        raise Exception(missingCoverageTrends)

    "Make list of publications that have scraped lists"
    availablePublications = [x for x in os.listdir("{}/archived_links".format(path)) if x.find(".") ==-1]

    "If publications are limited, then only go with those"
    if len(publications) > 0:
        availablePublications = [x for x in publications if x in availablePublications]

    loaded_articles = []

    "Loop through each publisher in CoverageTrends and load each day"
    for publisher in availablePublications:

        csvPaths = []

        pubPath = "{}/{}".format("{}/archived_links".format(path), publisher)
        for month in os.listdir(pubPath):
            if month.find(".") > -1:
                continue
            monthPath = "{}/{}".format(pubPath, month)
            for day in os.listdir(monthPath):
                if dateStart != None:
                    if int(day.split("_")[1][:-4]) < int(dateStart):
                        continue
                if dateEnd != None:
                    if int(day.split("_")[1][:-4]) > int(dateEnd):
                        continue
                csvPaths.append("{}/{}".format(monthPath, day))

        csvPaths = sorted(csvPaths)

        if lastN != None:
            csvPaths = csvPaths[-lastN:]

        csvPaths = pd.concat([pd.read_csv(x) for x in csvPaths], ignore_index=True)
        csvPaths["source"] = publisher
        loaded_articles.append(csvPaths)

    return pd.concat(loaded_articles).fillna("")


# Cell
def dailysourcepermalinksLoader(path="", filename="", **kwargs):
    "returns df of a json file which is date indexed with publishers for columns"

    df = pd.read_json("{}/{}".format(path, filename)).T
    df["date"] = df.index
    df = df.melt(id_vars = ["date"], var_name="Source", value_name="text") #make big column
    return df

# Cell
class article_holder(article_holder):
    "gives article_holder ability to load articles for publication between dateStart and dateEnd"

    def load_articles(self, engine=CoverageTrendsLoader, publications:[str] = [], dateStart:str=None, dateEnd:str=None, lastN:int=None, verbose=False, **kwargs) -> []:
        if self.articleDir == None:
            raise Exception("holder missing path")

        self.df = engine(path=self.articleDir, publications=publications, dateStart=dateStart, dateEnd=dateEnd, lastN=lastN, verbose=verbose, **kwargs)

        tmp = pd.DataFrame([self.df.text.unique()]).T
        tmp.columns=["text"]
        stemmer = SnowballStemmer("english", ignore_stopwords=True)
        tmp["quickReplace"] = tmp["text"].fillna("").apply(lambda x: re.sub('[^a-z]+', " ", x.lower()))
        tmp["tokens"] = tmp["quickReplace"].apply(lambda x: [stemmer.stem(y) for y in x.split() if len (y) > 0])
        tmp["quickReplace"] = tmp["tokens"].apply(lambda x: " ".join(x))

        self.df = self.df.merge(tmp)

# Cell
class article_holder(article_holder):
    "article_holders should be able to load dfs from other article_holders"

    def load_article_holder(self, other_article_holder:article_holder):
        errorMessage = "Bad format"

        try:
            missing = []
            otherColumns = other_article_holder.df.columns
            if "quickReplace" not in otherColumns:
                missing.append("quickReplace")
            if "tokens" not in otherColumns:
                missing.append("tokens")
            if "text" not in otherColumns:
                missing.append("text")
            if "source" not in otherColumns:
                missing.append("source")
            if len(missing) > 0:
                errorMessage = "Missing {}".format(";".join(missing))
                raise Exception("error")

        except:
            raise Exception("Passed article holder structure not recognized: {}".format(errorMessage))

        self.df = other_article_holder.df.copy()
# AUTOGENERATED! DO NOT EDIT! File to edit: 00_loadarticles.ipynb (unless otherwise specified). # Cell # Cell # Cell # Cell Turns CSV of scraped headlines from CoverageTrends into a Pandas Dataframe.     Expects that CoverageTrends (https://github.com/brockmanmatt/CoverageTrends) is cloned into ../CoverageTrends      Parameters      publications: list of publications to try to pull from CoverageTrends CSV, all if []      dateStart: String YYYYMMDD for first date of CSV to load for each publication      dateEnd: String YYYYMMDD for last date of CSV to load for each publication      lastN: get max (available days, lastN) days # Cell #make big column # Cell # Cell