import newspaper
import feedparser
import urllib.parse

import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore

from article_parser import Article
from config import KEYFILE

# If less than 100 tokens retry parsing the article not cleaning dom
retry_article_parse_tokens = 100


def download(url, clean_doc=True):
    """
    Tries to download + parse the article using newspaper
    :param url:
    :return:
    """

    article = Article(url)
    is_valid = True

    try:
        article.download()
        article.parse(clean_doc=clean_doc)
    except newspaper.article.ArticleException:
        print("Download error: {0}".format(url))
        is_valid = False

    return article, is_valid


def scrape_article(url):
    """
    Downloads the article from URL
    :param url:
    :return:
    """

    article, is_valid = download(url)

    if is_valid:

        # Retry downloading article without cleaning
        if len(article.text.split()) < retry_article_parse_tokens:
            article, is_valid = download(url, clean_doc=False)

        title = article.title
        text = article.text
        date = article.publish_date
        img = article.top_image

        return {
            'text': text,
            'title': title,
            'date': str(date),
            'img_url': img,
            'url': url,
        }


def rss_to_article_dicts(rss_link, max_articles=5):
    """
    Takes the rss link, parses articles from it and gets the article dict for each link in the rss feed
    :param rss_link:
    :return:
    """

    feed = feedparser.parse(rss_link)
    article_dicts = []

    for entry in feed['entries'][:max_articles]:
        article = scrape_article(entry['links'][0]['href'])

        # First n sentences + remove newlines
        article['summary'] = ". ".join(article['text'].split('.')[:6]).replace("\n", " ")
        article_dicts.append(article)

    return article_dicts


def store_article_dicts_from_rss(rss_link):
    """
    Parses the rss_link and saves the individual articles in the DB
    :param rss_link:
    :return:
    """

    try:
        # Check if app already initialized will fail if apps with same name initialized twice
        if (not len(firebase_admin._apps)):
            cred = credentials.Certificate(KEYFILE)
            firebase_admin.initialize_app(cred)

        db = firestore.client()

        article_dicts = rss_to_article_dicts(rss_link)

        if len(article_dicts) > 0:
            # Create a document with URL encoded RSS link as the key
            encoded = urllib.parse.quote_plus(rss_link)

            rss_docs = db.collection("rss_feeds").document(encoded)
            # Need to add rss_feed property so that rss_feeds collection can be queried for rss_feeds, with no feild (only document) the collection comes back empty
            # Becuase with no feilds considered "virtual document", "This document does not exist and will not appear in queries or snapshots, but identically structured document works"
            rss_docs.set({
                "rss_feed": rss_link
            })

            # Get old titles to delete later
            old_articles = db.collection("rss_feeds").document(encoded).collection('articles').stream()
            old_titles = [article.to_dict()['title'] for article in old_articles]
            new_titles = [article['title'] for article in article_dicts]

            # Add each article_dict as a document in the 'articles' collection with the title as the key
            for article in article_dicts:
                rss_docs.collection('articles').document(article['title']).set(article)

            # Only delete after adding new articles so api will never be queried when DB empty, otherwise if called when refreshing content no articles my come back.
            for old in old_titles:
                if old not in new_titles:
                    rss_docs.collection('articles').document(old).delete()


    except Exception:
        return None

    # If documents are successfully saved return the article dict
    return article_dicts


def get_article_dicts_from_rss(rss_link):
    """
    Checks the DB for the rss_link key and returns the list of article_dicts if found
    :param rss_link:
    :return:
    """

    try:
        # Check if app already initialized will fail if apps with same name initialized twice
        if (not len(firebase_admin._apps)):
            cred = credentials.Certificate(KEYFILE)
            firebase_admin.initialize_app(cred)

        db = firestore.client()
        articles = db.collection("rss_feeds").document(urllib.parse.quote_plus(rss_link)).collection(
            'articles').stream()
        article_dicts = [article.to_dict() for article in articles]

    except KeyError:
        return None

    # If documents are successfully saved return the article dict
    return article_dicts

import newspaper
import feedparser
import urllib.parse

import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore

from article_parser import Article
from config import KEYFILE

# If less than 100 tokens retry parsing the article not cleaning dom
retry_article_parse_tokens = 100


def download(url, clean_doc=True):
    """
    Tries to download + parse the article using newspaper
    :param url:
    :return:
    """

    article = Article(url)
    is_valid = True

    try:
        article.download()
        article.parse(clean_doc=clean_doc)
    except newspaper.article.ArticleException:
        print("Download error: {0}".format(url))
        is_valid = False

    return article, is_valid


def scrape_article(url):
    """
    Downloads the article from URL
    :param url:
    :return:
    """

    article, is_valid = download(url)

    if is_valid:

        # Retry downloading article without cleaning
        if len(article.text.split()) < retry_article_parse_tokens:
            article, is_valid = download(url, clean_doc=False)

        title = article.title
        text = article.text
        date = article.publish_date
        img = article.top_image

        return {
            'text': text,
            'title': title,
            'date': str(date),
            'img_url': img,
            'url': url,
        }


def rss_to_article_dicts(rss_link, max_articles=5):
    """
    Takes the rss link, parses articles from it and gets the article dict for each link in the rss feed
    :param rss_link:
    :return:
    """

    feed = feedparser.parse(rss_link)
    article_dicts = []

    for entry in feed['entries'][:max_articles]:
        article = scrape_article(entry['links'][0]['href'])

        # First n sentences + remove newlines
        article['summary'] = ". ".join(article['text'].split('.')[:6]).replace("\n", " ")
        article_dicts.append(article)

    return article_dicts


def store_article_dicts_from_rss(rss_link):
    """
    Parses the rss_link and saves the individual articles in the DB
    :param rss_link:
    :return:
    """

    try:
        # Check if app already initialized will fail if apps with same name initialized twice
        if (not len(firebase_admin._apps)):
            cred = credentials.Certificate(KEYFILE)
            firebase_admin.initialize_app(cred)

        db = firestore.client()

        article_dicts = rss_to_article_dicts(rss_link)

        if len(article_dicts) > 0:
            # Create a document with URL encoded RSS link as the key
            encoded = urllib.parse.quote_plus(rss_link)

            rss_docs = db.collection("rss_feeds").document(encoded)
            # Need to add rss_feed property so that rss_feeds collection can be queried for rss_feeds, with no feild (only document) the collection comes back empty
            # Becuase with no feilds considered "virtual document", "This document does not exist and will not appear in queries or snapshots, but identically structured document works"
            rss_docs.set({
                "rss_feed": rss_link
            })

            # Get old titles to delete later
            old_articles = db.collection("rss_feeds").document(encoded).collection('articles').stream()
            old_titles = [article.to_dict()['title'] for article in old_articles]
            new_titles = [article['title'] for article in article_dicts]

            # Add each article_dict as a document in the 'articles' collection with the title as the key
            for article in article_dicts:
                rss_docs.collection('articles').document(article['title']).set(article)

            # Only delete after adding new articles so api will never be queried when DB empty, otherwise if called when refreshing content no articles my come back.
            for old in old_titles:
                if old not in new_titles:
                    rss_docs.collection('articles').document(old).delete()


    except Exception:
        return None

    # If documents are successfully saved return the article dict
    return article_dicts


def get_article_dicts_from_rss(rss_link):
    """
    Checks the DB for the rss_link key and returns the list of article_dicts if found
    :param rss_link:
    :return:
    """

    try:
        # Check if app already initialized will fail if apps with same name initialized twice
        if (not len(firebase_admin._apps)):
            cred = credentials.Certificate(KEYFILE)
            firebase_admin.initialize_app(cred)

        db = firestore.client()
        articles = db.collection("rss_feeds").document(urllib.parse.quote_plus(rss_link)).collection(
            'articles').stream()
        article_dicts = [article.to_dict() for article in articles]

    except KeyError:
        return None

    # If documents are successfully saved return the article dict
    return article_dicts

# If less than 100 tokens retry parsing the article not cleaning dom Tries to download + parse the article using newspaper     :param url:     :return: Downloads the article from URL     :param url:     :return: # Retry downloading article without cleaning Takes the rss link, parses articles from it and gets the article dict for each link in the rss feed     :param rss_link:     :return: # First n sentences + remove newlines Parses the rss_link and saves the individual articles in the DB     :param rss_link:     :return: # Check if app already initialized will fail if apps with same name initialized twice # Create a document with URL encoded RSS link as the key # Need to add rss_feed property so that rss_feeds collection can be queried for rss_feeds, with no feild (only document) the collection comes back empty # Becuase with no feilds considered "virtual document", "This document does not exist and will not appear in queries or snapshots, but identically structured document works" # Get old titles to delete later # Add each article_dict as a document in the 'articles' collection with the title as the key # Only delete after adding new articles so api will never be queried when DB empty, otherwise if called when refreshing content no articles my come back. # If documents are successfully saved return the article dict Checks the DB for the rss_link key and returns the list of article_dicts if found     :param rss_link:     :return: # Check if app already initialized will fail if apps with same name initialized twice # If documents are successfully saved return the article dict