def discard_unwanted(tree):
    '''Delete unwanted sections of an HTML document and return them as a list'''
    my_discarded = []
    for subtree in tree.xpath(DISCARD_EXPRESSIONS):
        my_discarded.append(subtree)
        subtree.getparent().remove(subtree)
    return tree, my_discarded def extract_url_date(testurl, outputformat):
    """Extract the date out of an URL string complying with the Y-M-D format"""
    match = COMPLETE_URL.search(testurl)
    if match:
        dateresult = match.group(0)
        LOGGER.debug('found date in URL: %s', dateresult)
        try:
            dateobject = datetime.datetime(int(match.group(1)),
                                           int(match.group(2)),
                                           int(match.group(3)))
            if date_validator(dateobject, outputformat) is True:
                return dateobject.strftime(outputformat)
        except ValueError as err:
            LOGGER.debug('conversion error: %s %s', dateresult, err)
    return None def extract_partial_url_date(testurl, outputformat):
    """Extract an approximate date out of an URL string in Y-M format"""
    match = PARTIAL_URL.search(testurl)
    if match:
        dateresult = match.group(0) + '/01'
        LOGGER.debug('found partial date in URL: %s', dateresult)
        try:
            dateobject = datetime.datetime(int(match.group(1)),
                                           int(match.group(2)),
                                           1)
            if date_validator(dateobject, outputformat) is True:
                return dateobject.strftime(outputformat)
        except ValueError as err:
            LOGGER.debug('conversion error: %s %s', dateresult, err)
    return None def idiosyncrasies_search(htmlstring, outputformat, min_date, max_date):
    '''Look for author-written dates throughout the web page'''
    result = None
    # DE
    result = extract_idiosyncrasy(DE_PATTERNS, htmlstring, outputformat, min_date, max_date)
    # EN
    if result is None:
        result = extract_idiosyncrasy(EN_PATTERNS, htmlstring, outputformat, min_date, max_date)
    # TR
    if result is None:
        result = extract_idiosyncrasy(TR_PATTERNS, htmlstring, outputformat, min_date, max_date)
    return result def img_search(tree, outputformat, min_date, max_date):
    '''Skim through image elements'''
    element = tree.find('.//meta[@property="og:image"]')
    if element is not None and 'content' in element.attrib:
        result = extract_url_date(element.get('content'), outputformat)
        if date_validator(result, outputformat, earliest=min_date, latest=max_date) is True:
            return result
    return None def json_search(tree, outputformat, original_date, min_date, max_date):
    '''Look for JSON time patterns in JSON sections of the tree'''
    # determine pattern
    if original_date is True:
        json_pattern = JSON_PATTERN_PUBLISHED
    else:
        json_pattern = JSON_PATTERN_MODIFIED
    # look throughout the HTML tree
    for elem in tree.xpath('.//script[@type="application/ld+json"]|//script[@type="application/settings+json"]'):
        if not elem.text or '"date' not in elem.text:
            continue
        json_match = json_pattern.search(elem.text)
        if json_match and date_validator(json_match.group(1), '%Y-%m-%d', earliest=min_date, latest=max_date):
            LOGGER.debug('JSON time found: %s', json_match.group(0))
            return convert_date(json_match.group(1), '%Y-%m-%d', outputformat)
    return None def timestamp_search(htmlstring, outputformat, min_date, max_date):
    '''Look for timestamps throughout the web page'''
    tstamp_match = TIMESTAMP_PATTERN.search(htmlstring)
    if tstamp_match and date_validator(tstamp_match.group(1), '%Y-%m-%d', earliest=min_date, latest=max_date):
        LOGGER.debug('time regex found: %s', tstamp_match.group(0))
        return convert_date(tstamp_match.group(1), '%Y-%m-%d', outputformat)
    return None @lru_cache(maxsize=CACHE_SIZE)
def try_ymd_date(string, outputformat, extensive_search, min_date, max_date):
    """Use a series of heuristics and rules to parse a potential date expression"""
    # if string less than 6 chars, stop
    if not string or len(string) < 6:
        return None

    # count how many digit number in this string
    if not 4 <= len([c for c in string if c.isdigit()]) <= 18:
        return None

    # check if string only contains time/single year or digits and not a date
    if not TEXT_DATE_PATTERN.search(string) or NO_TEXT_DATE_PATTERN.match(string):
        return None

    # try to parse using the faster method
    customresult = custom_parse(string, outputformat, extensive_search, min_date, max_date)
    if customresult is not None:
        return customresult

    # use slow but extensive search
    if extensive_search is True:
        # send to date parser
        dateparser_result = external_date_parser(string, outputformat)
        if date_validator(
            dateparser_result, outputformat, earliest=min_date, latest=max_date
        ):
            return dateparser_result

    return None ADDITIONAL_EXPRESSIONS = ".//*[contains(@class, 'fecha') or contains(@class, 'parution')]" DATE_EXPRESSIONS = """
    .//*[contains(@id, 'date') or contains(@id, 'Date') or
    contains(@id, 'datum') or contains(@id, 'Datum') or contains(@id, 'time')
    or contains(@class, 'post-meta-time')]
    |
    .//*[contains(@class, 'date') or contains(@class, 'Date')
    or contains(@class, 'datum') or contains(@class, 'Datum')]
    |
    .//*[contains(@class, 'postmeta') or contains(@class, 'post-meta')
    or contains(@class, 'entry-meta') or contains(@class, 'entry-date') or contains(@class, 'postMeta')
    or contains(@class, 'post_meta') or contains(@class, 'post__meta') or
    contains(@class, 'article__date') or contains(@class, 'post_detail') or @class='meta'
    or @class='meta-before' or @class='asset-meta' or
    contains(@id, 'article-metadata') or contains(@class, 'article-metadata')
    or contains(@class, 'block-content') or contains(@class, 'byline') or
    contains(@class, 'dateline') or contains(@class, 'subline')
    or contains(@class, 'published') or contains(@class, 'posted') or
    contains(@class, 'submitted') or contains(@class, 'updated') or contains(@class, 'created-post')
    or contains(@id, 'post-timestamp') or contains(@class, 'post-timestamp')]
    |
    .//*[contains(@id, 'lastmod') or contains(@itemprop, 'date') or
    contains(@class, 'time') or contains(@id, 'metadata') or contains(@id, 'publish')]
    |
    .//footer
    |
    .//*[@class='post-footer' or @class='footer' or @id='footer']
    |
    .//small
    |
    .//*[contains(@class, 'author') or contains(@class, 'autor') or
    contains(@class, 'field-content') or @class='meta' or
    contains(@class, 'info') or contains(@class, 'fa-clock-o') or contains(@class, 'fa-calendar') or
    contains(@class, 'publication')]""" YEAR_PATTERN = re.compile(r'^\D?(199[0-9]|20[0-9]{2})') YMD_PATTERN = re.compile(r'(?:\D|^)(\d{4})[\-/.](\d{1,2})[\-/.](\d{1,2})(?:\D|$)') COPYRIGHT_PATTERN = re.compile(r'(?:©|\&copy;|Copyright|\(c\))\D*(?:[12][0-9]{3}-)?([12][0-9]{3})\D') THREE_PATTERN = re.compile(r'/([0-9]{4}/[0-9]{2}/[0-9]{2})[01/]') THREE_CATCH = re.compile(r'([0-9]{4})/([0-9]{2})/([0-9]{2})') THREE_LOOSE_PATTERN = re.compile(r'\D([0-9]{4}[/.-][0-9]{2}[/.-][0-9]{2})\D') THREE_LOOSE_CATCH = re.compile(r'([0-9]{4})[/.-]([0-9]{2})[/.-]([0-9]{2})') SELECT_YMD_PATTERN = re.compile(r'\D([0-3]?[0-9][/.-][01]?[0-9][/.-][0-9]{4})\D') SELECT_YMD_YEAR = re.compile(r'(19[0-9]{2}|20[0-9]{2})\D?$') YMD_YEAR = re.compile(r'^([0-9]{4})') DATESTRINGS_PATTERN = re.compile(r'(\D19[0-9]{2}[01][0-9][0-3][0-9]\D|\D20[0-9]{2}[01][0-9][0-3][0-9]\D)') DATESTRINGS_CATCH = re.compile(r'([12][0-9]{3})([01][0-9])([0-3][0-9])') SLASHES_PATTERN = re.compile(r'\D([0-3]?[0-9][/.][01]?[0-9][/.][0129][0-9])\D') SLASHES_YEAR = re.compile(r'([0-9]{2})$') YYYYMM_PATTERN = re.compile(r'\D([12][0-9]{3}[/.-][01][0-9])\D') YYYYMM_CATCH = re.compile(r'([12][0-9]{3})[/.-]([01][0-9])') MMYYYY_PATTERN = re.compile(r'\D([01]?[0-9][/.-][12][0-9]{3})\D') MMYYYY_YEAR = re.compile(r'([12][0-9]{3})\D?$') SIMPLE_PATTERN = re.compile(r'\D(199[0-9]|20[0-9]{2})\D') THREE_COMP_REGEX_A = re.compile(r'([0-3]?[0-9])[/.-]([01]?[0-9])[/.-]([0-9]{4})') THREE_COMP_REGEX_B = re.compile(r'([0-3]?[0-9])[/.-]([01]?[0-9])[/.-]([0-9]{2})') TWO_COMP_REGEX = re.compile(r'([0-3]?[0-9])[/.-]([0-9]{4})') CACHE_SIZE = 8192 HTML_CLEANER = Cleaner() MAX_POSSIBLE_CANDIDATES = 150 def load_html(htmlobject):
    """Load object given as input and validate its type
    (accepted: LXML tree, bytestring and string)
    """
    # use tree directly
    if isinstance(htmlobject, (etree._ElementTree, html.HtmlElement)):
        return htmlobject
    # do not accept any other type after this point
    if not isinstance(htmlobject, (bytes, str)):
        raise TypeError('incompatible input type', type(htmlobject))
    # the string is a URL, download it
    if isinstance(htmlobject, str) and htmlobject.startswith('http'):
        htmltext = None
        if re.match(r'https?://[^ ]+$', htmlobject):
            LOGGER.info('URL detected, downloading: %s', htmlobject)
            htmltext = fetch_url(htmlobject)
            if htmltext is not None:
                htmlobject = htmltext
        # log the error and quit
        if htmltext is None:
            raise ValueError("URL couldn't be processed: %s", htmlobject)
    # start processing
    tree = None
    # try to guess encoding and decode file: if None then keep original
    htmlobject = decode_file(htmlobject)
    # sanity check
    check_flag = is_dubious_html(htmlobject)
    # use Unicode string
    try:
        tree = html.fromstring(htmlobject, parser=HTML_PARSER)
    except ValueError:
        # "Unicode strings with encoding declaration are not supported."
        try:
            tree = html.fromstring(htmlobject.encode('utf8'), parser=HTML_PARSER)
        except Exception as err:
            LOGGER.error('lxml parser bytestring %s', err)
    except Exception as err:
        LOGGER.error('lxml parsing failed: %s', err)
    # rejection test: is it (well-formed) HTML at all?
    if tree is not None and check_flag is True and len(tree) < 2:
        LOGGER.error('parsed tree length: %s, wrong data type or not valid HTML', len(tree))
        tree = None
    return tree def check_extracted_reference(reference, outputformat, min_date, max_date):
    '''Test if the extracted reference date can be returned'''
    if reference > 0:
        dateobject = datetime.datetime.fromtimestamp(reference)
        converted = dateobject.strftime(outputformat)
        if date_validator(converted, outputformat, earliest=min_date, latest=max_date) is True:
            return converted
    return None def compare_values(reference, attempt, outputformat, original_date):
    """Compare the date expression to a reference"""
    timestamp = time.mktime(datetime.datetime.strptime(attempt, outputformat).timetuple())
    if original_date is True and (reference == 0 or timestamp < reference):
        reference = timestamp
    elif original_date is False and timestamp > reference:
        reference = timestamp
    return reference def convert_date(datestring, inputformat, outputformat):
    """Parse date and return string in desired format"""
    # speed-up (%Y-%m-%d)
    if inputformat == outputformat:
        return str(datestring)
    # date object (speedup)
    if isinstance(datestring, datetime.date):
        return datestring.strftime(outputformat)
    # normal
    dateobject = datetime.datetime.strptime(datestring, inputformat)
    return dateobject.strftime(outputformat) @lru_cache(maxsize=CACHE_SIZE)
def date_validator(date_input, outputformat, earliest=MIN_DATE, latest=LATEST_POSSIBLE):
    """Validate a string w.r.t. the chosen outputformat and basic heuristics"""
    # safety check
    if date_input is None:
        return False
    # try if date can be parsed using chosen outputformat
    if not isinstance(date_input, datetime.date):
        # speed-up
        try:
            if outputformat == '%Y-%m-%d':
                dateobject = datetime.datetime(int(date_input[:4]),
                                               int(date_input[5:7]),
                                               int(date_input[8:10]))
            # default
            else:
                dateobject = datetime.datetime.strptime(date_input, outputformat)
        except ValueError:
            return False
    else:
        dateobject = date_input
    # basic year validation
    year = int(datetime.date.strftime(dateobject, '%Y'))
    if MIN_YEAR <= year <= MAX_YEAR:
        # not newer than today or stored variable
        try:
            if earliest <= dateobject.date() <= latest:
                return True
        except AttributeError:
            if earliest <= dateobject <= latest:
                return True
    LOGGER.debug('date not valid: %s', date_input)
    return False @lru_cache(maxsize=CACHE_SIZE)
def filter_ymd_candidate(bestmatch, pattern, original_date, copyear, outputformat, min_date, max_date):
    """Filter free text candidates in the YMD format"""
    if bestmatch is not None:
        pagedate = '-'.join([bestmatch.group(1), bestmatch.group(2), bestmatch.group(3)])
        if date_validator(
            pagedate, '%Y-%m-%d', earliest=min_date, latest=max_date
        ) is True and (copyear == 0 or int(bestmatch.group(1)) >= copyear):
            LOGGER.debug('date found for pattern "%s": %s', pattern, pagedate)
            return convert_date(pagedate, '%Y-%m-%d', outputformat)
            ## TODO: test and improve
            #if original_date is True:
            #    if copyear == 0 or int(bestmatch.group(1)) <= copyear:
            #        LOGGER.debug('date found for pattern "%s": %s', pattern, pagedate)
            #        return convert_date(pagedate, '%Y-%m-%d', outputformat)
            #else:
            #    if copyear == 0 or int(bestmatch.group(1)) >= copyear:
            #        LOGGER.debug('date found for pattern "%s": %s', pattern, pagedate)
            #        return convert_date(pagedate, '%Y-%m-%d', outputformat)
    return None def get_min_date(min_date):
    '''Validates the minimum date and/or defaults to earliest plausible date'''
    if min_date is not None:
        try:
            # internal conversion from Y-M-D format
            min_date = datetime.date(int(min_date[:4]),
                                     int(min_date[5:7]),
                                     int(min_date[8:10]))
        except ValueError:
            min_date = MIN_DATE
    else:
        min_date = MIN_DATE
    return min_date def get_max_date(max_date):
    '''Validates the maximum date and/or defaults to latest plausible date'''
    if max_date is not None:
        try:
            # internal conversion from Y-M-D format
            max_date = datetime.date(int(max_date[:4]),
                                     int(max_date[5:7]),
                                     int(max_date[8:10]))
        except ValueError:
            max_date = LATEST_POSSIBLE
    else:
        max_date = LATEST_POSSIBLE
    return max_date def output_format_validator(outputformat):
    """Validate the output format in the settings"""
    # test in abstracto
    if not isinstance(outputformat, str) or '%' not in outputformat:
        LOGGER.error('malformed output format: %s', outputformat)
        return False
    # test with date object
    dateobject = datetime.datetime(2017, 9, 1, 0, 0)
    try:
        dateobject.strftime(outputformat)
    # Python < 3.7 only
    except (NameError, TypeError, UnicodeError, ValueError) as err:
        LOGGER.error('wrong output format or type: %s %s', outputformat, err)
        return False
    return True @lru_cache(maxsize=CACHE_SIZE)
def plausible_year_filter(htmlstring, pattern, yearpat, tocomplete=False):
    """Filter the date patterns to find plausible years only"""
    # slow!
    allmatches = pattern.findall(htmlstring)
    occurrences = Counter(allmatches)
    toremove = set()
    # LOGGER.debug('occurrences: %s', occurrences)
    for item in occurrences.keys():
        # scrap implausible dates
        try:
            if tocomplete is False:
                potential_year = int(yearpat.search(item).group(1))
            else:
                lastdigits = yearpat.search(item).group(1)
                if lastdigits[0] == '9':
                    potential_year = int('19' + lastdigits)
                else:
                    potential_year = int('20' + lastdigits)
        except AttributeError:
            LOGGER.debug('not a year pattern: %s', item)
            toremove.add(item)
        else:
            if potential_year < MIN_YEAR or potential_year > MAX_YEAR:
                LOGGER.debug('no potential year: %s', item)
                toremove.add(item)
            # occurrences.remove(item)
            # continue
    # preventing dictionary changed size during iteration error
    for item in toremove:
        del occurrences[item]
    return occurrences
import logging
import re
from collections import Counter
from copy import deepcopy
from functools import lru_cache, partial
from lxml import etree, html
from .extractors import (discard_unwanted, extract_url_date,
                         extract_partial_url_date, idiosyncrasies_search,
                         img_search, json_search, timestamp_search, try_ymd_date,
                         ADDITIONAL_EXPRESSIONS, DATE_EXPRESSIONS,
                         YEAR_PATTERN, YMD_PATTERN, COPYRIGHT_PATTERN,
                         THREE_PATTERN, THREE_CATCH,
                         THREE_LOOSE_PATTERN, THREE_LOOSE_CATCH,
                         SELECT_YMD_PATTERN, SELECT_YMD_YEAR, YMD_YEAR,
                         DATESTRINGS_PATTERN, DATESTRINGS_CATCH,
                         SLASHES_PATTERN, SLASHES_YEAR,
                         YYYYMM_PATTERN, YYYYMM_CATCH, MMYYYY_PATTERN,
                         MMYYYY_YEAR, SIMPLE_PATTERN,
                         THREE_COMP_REGEX_A, THREE_COMP_REGEX_B, TWO_COMP_REGEX)
from .settings import CACHE_SIZE, HTML_CLEANER, MAX_POSSIBLE_CANDIDATES
from .utils import load_html
from .validators import (check_extracted_reference, compare_values,
                         convert_date, date_validator, filter_ymd_candidate,
                         get_min_date, get_max_date, output_format_validator,
                         plausible_year_filter)
    elements = tree.findall('.//time')
    if elements is not None and len(elements) < MAX_POSSIBLE_CANDIDATES:
        # scan all the tags and look for the newest one
        reference = 0
        for elem in elements:
            shortcut_flag = False
            # go for datetime
            if 'datetime' in elem.attrib and len(elem.get('datetime')) > 6:
                # shortcut: time pubdate
                if 'pubdate' in elem.attrib and elem.get('pubdate') == 'pubdate':
                    if original_date is True:
                        shortcut_flag = True
                    LOGGER.debug('time pubdate found: %s', elem.get('datetime'))
                # first choice: entry-date + datetime attribute
                elif 'class' in elem.attrib:
                    if elem.get('class').startswith('entry-date') or elem.get('class').startswith('entry-time'):
                        # shortcut
                        if original_date is True:
                            shortcut_flag = True
                        LOGGER.debug('time/datetime found: %s', elem.get('datetime'))
                    # updated time
                    elif elem.get('class') == 'updated' and original_date is False:
                        LOGGER.debug('updated time/datetime found: %s', elem.get('datetime'))
                # datetime attribute
                else:
                    LOGGER.debug('time/datetime found: %s', elem.get('datetime'))
                # analyze attribute
                if shortcut_flag:
                    attempt = try_ymd_date(elem.get('datetime'), outputformat, extensive_search, min_date, max_date)
                    if attempt is not None:
                        return attempt
                else:
                    reference = compare_reference(reference, elem.get('datetime'), outputformat, extensive_search, original_date, min_date, max_date)
                    if reference > 0:
                        break
            # bare text in element
            elif elem.text is not None and len(elem.text) > 6:
                LOGGER.debug('time/datetime found: %s', elem.text)
                reference = compare_reference(reference, elem.text, outputformat, extensive_search, original_date, min_date, max_date)
            # else...?
        # return
        converted = check_extracted_reference(reference, outputformat, min_date, max_date)
        if converted is not None:
            return converted
    return None


def normalize_match(match):
    '''Normalize string output by adding "0" if necessary.'''
    day = match.group(1)
    if len(day) == 1:
        day = '0' + day
    month = match.group(2)
    if len(month) == 1:
        month = '0' + month
    return day, month


def search_page(htmlstring, outputformat, original_date, min_date, max_date):
    """
    Opportunistically search the HTML text for common text patterns

    :param htmlstring:
        The HTML document in string format, potentially cleaned and stripped to
        the core (much faster)
    :type htmlstring: string
    :param outputformat:
        Provide a valid datetime format for the returned string
        (see datetime.strftime())
    :type outputformat: string
    :param original_date:
        Look for original date (e.g. publication date) instead of most recent
        one (e.g. last modified, updated time)
    :type original_date: boolean
    :return: Returns a valid date expression as a string, or None

    """

    # copyright symbol
    LOGGER.debug('looking for copyright/footer information')
    copyear = 0
    bestmatch = search_pattern(htmlstring, COPYRIGHT_PATTERN, YEAR_PATTERN, YEAR_PATTERN, original_date, min_date, max_date)
    if bestmatch is not None:
        LOGGER.debug('Copyright detected: %s', bestmatch.group(0))
        if date_validator(bestmatch.group(0), '%Y', latest=max_date) is True:
            LOGGER.debug('copyright year/footer pattern found: %s', bestmatch.group(0))
            copyear = int(bestmatch.group(0))

    # 3 components
    LOGGER.debug('3 components')
    # target URL characteristics
    bestmatch = search_pattern(htmlstring, THREE_PATTERN, THREE_CATCH, YEAR_PATTERN, original_date, min_date, max_date)
    result = filter_ymd_candidate(bestmatch, THREE_PATTERN, original_date, copyear, outputformat, min_date, max_date)
    if result is not None:
        return result

    # more loosely structured data
    bestmatch = search_pattern(htmlstring, THREE_LOOSE_PATTERN, THREE_LOOSE_CATCH, YEAR_PATTERN, original_date, min_date, max_date)
    result = filter_ymd_candidate(bestmatch, THREE_LOOSE_PATTERN, original_date, copyear, outputformat, min_date, max_date)
    if result is not None:
        return result

    # YYYY-MM-DD/DD-MM-YYYY
    candidates = plausible_year_filter(htmlstring, SELECT_YMD_PATTERN, SELECT_YMD_YEAR)
    # revert DD-MM-YYYY patterns before sorting
    replacement = {}
    for item in candidates:
        match = THREE_COMP_REGEX_A.match(item)
        day, month = normalize_match(match)
        candidate = '-'.join([match.group(3), month, day])
        replacement[candidate] = candidates[item]
    candidates = Counter(replacement)
    # select
    bestmatch = select_candidate(candidates, YMD_PATTERN, YMD_YEAR, original_date, min_date, max_date)
    result = filter_ymd_candidate(bestmatch, SELECT_YMD_PATTERN, original_date, copyear, outputformat, min_date, max_date)
    if result is not None:
        return result

    # valid dates strings
    bestmatch = search_pattern(htmlstring, DATESTRINGS_PATTERN, DATESTRINGS_CATCH, YEAR_PATTERN, original_date, min_date, max_date)

    result = filter_ymd_candidate(bestmatch, DATESTRINGS_PATTERN, original_date, copyear, outputformat, min_date, max_date)