def examine(htmlstring, extensive_bool=True, original_date=False,
            verbose_flag=False, mindate=None, maxdate=None):
    """ Generic safeguards and triggers """
    # safety check
    if htmlstring is None:
        sys.stderr.write('# ERROR: empty document\n')
    elif len(htmlstring) > MAX_FILE_SIZE:
        sys.stderr.write('# ERROR: file too large\n')
    elif len(htmlstring) < MIN_FILE_SIZE:
        sys.stderr.write('# ERROR: file too small\n')
    else:
        return find_date(htmlstring, extensive_search=extensive_bool,
                           original_date=original_date, verbose=verbose_flag,
                           min_date=mindate, max_date=maxdate)
    return None def main():
    """ Run as a command-line utility. """
    # arguments
    args = parse_args(sys.argv[1:])
    # process input on STDIN
    process_args(args) def parse_args(args):
    """Define parser for command-line arguments"""
    argsparser = argparse.ArgumentParser()
    argsparser.add_argument("-f", "--fast",
                            help="fast mode: disable extensive search",
                            action="store_false")
    argsparser.add_argument("-i", "--inputfile",
                            help="""name of input file for batch processing
                            (similar to wget -i)""",
                            type=str)
    argsparser.add_argument("--original",
                            help="original date prioritized",
                            action="store_true")
    argsparser.add_argument("-min", "--mindate",
                            help="earliest acceptable date (YYYY-MM-DD)",
                            type=str)
    argsparser.add_argument("-max", "--maxdate",
                            help="latest acceptable date (YYYY-MM-DD)",
                            type=str)
    argsparser.add_argument("-u", "--URL",
                            help="custom URL download",
                            type=str)
    argsparser.add_argument("-v", "--verbose",
                            help="increase output verbosity",
                            action="store_true")
    argsparser.add_argument("--version",
                            help="show version information and exit",
                            action="version",
                            version="Htmldate {} - Python {}".format(
                            __version__, python_version()
                            ),)
    return argsparser.parse_args() def process_args(args):
    """Process the arguments passed on the command-line."""
    if not args.inputfile:
        # URL as input
        if args.URL:
            htmlstring = fetch_url(args.URL)
            if htmlstring is None:
                sys.exit('# ERROR no valid result for url: ' + args.URL + '\n')
        # unicode check
        else:
            try:
                htmlstring = sys.stdin.read()
            except UnicodeDecodeError as err:
                sys.exit('# ERROR system/buffer encoding: ' + str(err) + '\n')
        result = examine(htmlstring, extensive_bool=args.fast,
                         original_date=args.original,
                         verbose_flag=args.verbose,
                         maxdate=args.maxdate)
        if result is not None:
            sys.stdout.write(result + '\n')

    # process input file line by line
    else:
        with open(args.inputfile, mode='r', encoding='utf-8') as inputfile:
            for line in inputfile:
                htmltext = fetch_url(line.strip())
                result = examine(htmltext, extensive_bool=args.fast,
                                 original_date=args.original,
                                 verbose_flag=args.verbose,
                                 mindate=args.mindate, maxdate=args.maxdate)
                if result is None:
                    result = 'None'
                sys.stdout.write(line.strip() + '\t' + result + '\n') LOGGER = logging.getLogger(__name__)
DATE_ATTRIBUTES = {
                  'article.created', 'article_date_original',
                  'article.published', 'article:published_time',
                  'bt:pubdate', 'citation_date', 'citation_publication_date',
                  'created', 'cxenseparse:recs:publishtime',
                  'date', 'date_published',
                  'datecreated', 'dateposted', 'datepublished',
                  # Dublin Core: https://wiki.whatwg.org/wiki/MetaExtensions
                  'dc.date', 'dc.created', 'dc.date.created',
                  'dc.date.issued', 'dc.date.publication',
                  'dcterms.created', 'dcterms.date',
                  'dcterms.issued', 'dc:created', 'dc:date',
                  'gentime',
                  # Open Graph: https://opengraphprotocol.org/
                  'og:published_time', 'og:article:published_time',
                  'originalpublicationdate', 'parsely-pub-date',
                  'pubdate', 'publishdate', 'publish_date',
                  'published-date', 'publication_date',  'rnews:datepublished',
                  'sailthru.date', 'shareaholic:article_published_time', 'timestamp'
                  }
PROPERTY_MODIFIED = {
                    'article:modified_time', 'datemodified',
                    'dc.modified', 'dcterms.modified', 'modified_time',
                    'og:article:modified_time', 'og:updated_time',
                    'release_date', 'updated_time'
                    }
TRIM_REGEX = re.compile(r'[\n\r\s\t]+')
NON_DIGITS_REGEX = re.compile(r'\D+$')
GER_STRIP_REGEX = re.compile(r'^am ')
LAST_MODIFIED = {'lastmodified', 'last-modified'}
ITEMPROP_ATTRS_ORIGINAL = {'datecreated', 'datepublished', 'pubyear'}
ITEMPROP_ATTRS_MODIFIED = {'datemodified', 'dateupdate'}
ITEMPROP_ATTRS = ITEMPROP_ATTRS_ORIGINAL.union(ITEMPROP_ATTRS_MODIFIED)
CLASS_ATTRS = {'date-published', 'published', 'time published'}
def logstring(element):
def examine_date_elements(tree, expression, outputformat, extensive_search, min_date, max_date):
def examine_header(tree, outputformat, extensive_search, original_date, min_date, max_date):
def select_candidate(occurrences, catch, yearpat, original_date, min_date, max_date):
def search_pattern(htmlstring, pattern, catch, yearpat, original_date, min_date, max_date):
def try_expression(expression, outputformat, extensive_search, min_date, max_date):
def compare_reference(reference, expression, outputformat, extensive_search, original_date, min_date, max_date):
def examine_abbr_elements(tree, outputformat, extensive_search, original_date, min_date, max_date):
def examine_time_elements(tree, outputformat, extensive_search, original_date, min_date, max_date):
def normalize_match(match):
def search_page(htmlstring, outputformat, original_date, min_date, max_date):
def find_date(htmlobject, extensive_search=True, original_date=False, outputformat='%Y-%m-%d', url=None, verbose=False, min_date=None, max_date=None): def custom_parse(string, outputformat, extensive_search, min_date, max_date):
    """Try to bypass the slow dateparser"""
    LOGGER.debug('custom parse test: %s', string)

    # 1. '201709011234' not covered by dateparser, and regex too slow
    if string[0:8].isdigit():
        try:
            candidate = datetime.date(int(string[:4]),
                                      int(string[4:6]),
                                      int(string[6:8]))
        except ValueError:
            return None
        if date_validator(candidate, '%Y-%m-%d') is True:
            LOGGER.debug('ymd match: %s', candidate)
            return convert_date(candidate, '%Y-%m-%d', outputformat)

    # 2. shortcut, much faster
    if string[0:4].isdigit():
        # try speedup with ciso8601 (if installed)
        try:
            if extensive_search is True:
                result = parse_datetime(string)
            # speed-up by ignoring time zone info if ciso8601 is installed
            else:
                result = parse_datetime_as_naive(string)
            if date_validator(result, outputformat, earliest=min_date, latest=max_date) is True:
                LOGGER.debug('parsing result: %s', result)
                return result.strftime(outputformat)
        except (OverflowError, TypeError, ValueError):
            LOGGER.debug('parsing error: %s', string)

    # 3. Try YYYYMMDD, use regex
    match = YMD_NO_SEP_PATTERN.search(string)
    if match:
        try:
            year, month, day = int(match.group(1)[:4]), int(match.group(1)[4:6]), int(match.group(1)[6:8])
            candidate = datetime.date(year, month, day)
        except ValueError:
            LOGGER.debug('YYYYMMDD value error: %s', match.group(0))
        else:
            if date_validator(candidate, '%Y-%m-%d') is True:
                LOGGER.debug('YYYYMMDD match: %s', candidate)
                return convert_date(candidate, '%Y-%m-%d', outputformat)

    # 4. Try Y-M-D pattern since it's the one used in ISO-8601
    # see also fromisoformat() in Python >= 3.7
    match = YMD_PATTERN.search(string)
    if match:
        try:
            day, month, year = int(match.group(3)), int(match.group(2)), int(match.group(1))
            candidate = datetime.date(year, month, day)
        except ValueError:
            LOGGER.debug('Y-M-D value error: %s', match.group(0))
        else:
            if date_validator(candidate, '%Y-%m-%d') is True:
                LOGGER.debug('Y-M-D match: %s', candidate)
                return convert_date(candidate, '%Y-%m-%d', outputformat)

    # 5. Try the D-M-Y pattern since it's the most common date format in the world
    match = DMY_PATTERN.search(string)
    if match:
        try:
            day, month, year = int(match.group(1)), int(match.group(2)), int(match.group(3))
            year = correct_year(year)
            day, month = try_swap_values(day, month)
            candidate = datetime.date(year, month, day)
        except ValueError:
            LOGGER.debug('D-M-Y value error: %s', match.group(0))
        else:
            if date_validator(candidate, '%Y-%m-%d') is True:
                LOGGER.debug('D-M-Y match: %s', candidate)
                return convert_date(candidate, '%Y-%m-%d', outputformat)

    # 6. Try the Y-M pattern
    match = YM_PATTERN.search(string)
    if match:
        try:
            year, month = int(match.group(1)), int(match.group(2))
            candidate = datetime.date(year, month, 1)
        except ValueError:
            LOGGER.debug('Y-M value error: %s', match.group(0))
        else:
            if date_validator(candidate, '%Y-%m-%d') is True:
                LOGGER.debug('Y-M match: %s', candidate)
                return convert_date(candidate, '%Y-%m-%d', outputformat)

    # 7. Try the other regex pattern
    dateobject = regex_parse(string)
    if date_validator(dateobject, outputformat) is True:
        try:
            LOGGER.debug('custom parse result: %s', dateobject)
            return dateobject.strftime(outputformat)
        except ValueError as err:
            LOGGER.debug('value error during conversion: %s %s', string, err)

    return None def external_date_parser(string, outputformat):
    """Use dateutil parser or dateparser module according to system settings"""
    LOGGER.debug('send to external parser: %s', string)
    try:
        # dateparser installed or not
        #if EXTERNAL_PARSER is not None:
        target = EXTERNAL_PARSER.get_date_data(string)['date_obj']
        #else:
        #    target = FULL_PARSE(string, **DEFAULT_PARSER_PARAMS)
    # 2 types of errors possible
    except (OverflowError, ValueError):
        target = None
    # issue with data type
    if target is not None:
        return datetime.date.strftime(target, outputformat)
    return None def extract_partial_url_date(testurl, outputformat):
    """Extract an approximate date out of an URL string in Y-M format"""
    match = PARTIAL_URL.search(testurl)
    if match:
        dateresult = match.group(0) + '/01'
        LOGGER.debug('found partial date in URL: %s', dateresult)
        try:
            dateobject = datetime.datetime(int(match.group(1)),
                                           int(match.group(2)),
                                           1)
            if date_validator(dateobject, outputformat) is True:
                return dateobject.strftime(outputformat)
        except ValueError as err:
            LOGGER.debug('conversion error: %s %s', dateresult, err)
    return None def regex_parse(string):
    """Full-text parse using a series of regular expressions"""
    dateobject = regex_parse_de(string)
    if dateobject is None:
        dateobject = regex_parse_multilingual(string)
    return dateobject MIN_DATE = datetime.date(1995, 1, 1) LATEST_POSSIBLE = datetime.date.today() def decode_response(response):
    """Read the urllib3 object corresponding to the server response, then
       try to guess its encoding and decode it to return a unicode string"""
    # urllib3 response object / bytes switch
    if isinstance(response, urllib3.response.HTTPResponse) or hasattr(response, 'data'):
        resp_content = response.data
    else:
        resp_content = response
    return decode_file(resp_content) def detect_encoding(bytesobject):
    """Read all input or first chunk and return a list of encodings"""
    # alternatives: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
    # unicode-test
    if isutf8(bytesobject):
        return ['utf-8']
    guesses = []
    # additional module
    if cchardet_detect is not None:
        cchardet_guess = cchardet_detect(bytesobject)['encoding']
        if cchardet_guess is not None:
            guesses.append(cchardet_guess.lower())
    # try charset_normalizer on first part, fallback on full document
    detection_results = from_bytes(bytesobject[:15000]) or from_bytes(bytesobject)
    # return alternatives
    if len(detection_results) > 0:
        guesses.extend([r.encoding for r in detection_results])
    # it cannot be utf-8 (tested above)
    return [g for g in guesses if g not in UNICODE_ALIASES] def fetch_url(url):
    """Fetches page using urllib3 and decodes the response.

    Args:
        url: URL of the page to fetch.

    Returns:
        HTML code as string, or Urllib3 response object (headers + body), or empty string in case
        the result is invalid, or None if there was a problem with the network.

    """
    # send
    try:
        # read by streaming chunks (stream=True, iter_content=xx)
        # so we can stop downloading as soon as MAX_FILE_SIZE is reached
        response = HTTP_POOL.request('GET', url, timeout=30)
    except urllib3.exceptions.NewConnectionError as err:
        LOGGER.error('connection refused: %s %s', url, err)
    except urllib3.exceptions.MaxRetryError as err:
        LOGGER.error('retries/redirects: %s %s', url, err)
    except urllib3.exceptions.TimeoutError as err:
        LOGGER.error('connection timeout: %s %s', url, err)
    except Exception as err:
        LOGGER.error('unknown error: %s %s', url, err)  # sys.exc_info()[0]
    else:
        # safety checks
        if response.status != 200:
            LOGGER.error('not a 200 response: %s for URL %s', response.status, url)
        elif response.data is None or \
            len(response.data) < MIN_FILE_SIZE or len(response.data) > MAX_FILE_SIZE:
            LOGGER.error('incorrect input data for URL %s', url)
        else:
            return decode_response(response.data)
    return None def load_html(htmlobject):
    """Load object given as input and validate its type
    (accepted: LXML tree, bytestring and string)
    """
    # use tree directly
    if isinstance(htmlobject, (etree._ElementTree, html.HtmlElement)):
        return htmlobject
    # do not accept any other type after this point
    if not isinstance(htmlobject, (bytes, str)):
        raise TypeError('incompatible input type', type(htmlobject))
    # the string is a URL, download it
    if isinstance(htmlobject, str) and htmlobject.startswith('http'):
        htmltext = None
        if re.match(r'https?://[^ ]+$', htmlobject):
            LOGGER.info('URL detected, downloading: %s', htmlobject)
            htmltext = fetch_url(htmlobject)
            if htmltext is not None:
                htmlobject = htmltext
        # log the error and quit
        if htmltext is None:
            raise ValueError("URL couldn't be processed: %s", htmlobject)
    # start processing
    tree = None
    # try to guess encoding and decode file: if None then keep original
    htmlobject = decode_file(htmlobject)
    # sanity check
    check_flag = is_dubious_html(htmlobject)
    # use Unicode string
    try:
        tree = html.fromstring(htmlobject, parser=HTML_PARSER)
    except ValueError:
        # "Unicode strings with encoding declaration are not supported."
        try:
            tree = html.fromstring(htmlobject.encode('utf8'), parser=HTML_PARSER)
        except Exception as err:
            LOGGER.error('lxml parser bytestring %s', err)
    except Exception as err:
        LOGGER.error('lxml parsing failed: %s', err)
    # rejection test: is it (well-formed) HTML at all?
    if tree is not None and check_flag is True and len(tree) < 2:
        LOGGER.error('parsed tree length: %s, wrong data type or not valid HTML', len(tree))
        tree = None
    return tree def is_dubious_html(htmlobject):
    "Assess if the object is proper HTML (with a corresponding declaration)."
    if isinstance(htmlobject, bytes):
        if 'html' not in htmlobject[:50].decode(encoding='ascii', errors='ignore').lower():
            return True
    elif isinstance(htmlobject, str):
        if 'html' not in htmlobject[:50].lower():
            return True
    return False def convert_date(datestring, inputformat, outputformat):
    """Parse date and return string in desired format"""
    # speed-up (%Y-%m-%d)
    if inputformat == outputformat:
        return str(datestring)
    # date object (speedup)
    if isinstance(datestring, datetime.date):
        return datestring.strftime(outputformat)
    # normal
    dateobject = datetime.datetime.strptime(datestring, inputformat)
    return dateobject.strftime(outputformat) @lru_cache(maxsize=CACHE_SIZE)
def date_validator(date_input, outputformat, earliest=MIN_DATE, latest=LATEST_POSSIBLE):
    """Validate a string w.r.t. the chosen outputformat and basic heuristics"""
    # safety check
    if date_input is None:
        return False
    # try if date can be parsed using chosen outputformat
    if not isinstance(date_input, datetime.date):
        # speed-up
        try:
            if outputformat == '%Y-%m-%d':
                dateobject = datetime.datetime(int(date_input[:4]),
                                               int(date_input[5:7]),
                                               int(date_input[8:10]))
            # default
            else:
                dateobject = datetime.datetime.strptime(date_input, outputformat)
        except ValueError:
            return False
    else:
        dateobject = date_input
    # basic year validation
    year = int(datetime.date.strftime(dateobject, '%Y'))
    if MIN_YEAR <= year <= MAX_YEAR:
        # not newer than today or stored variable
        try:
            if earliest <= dateobject.date() <= latest:
                return True
        except AttributeError:
            if earliest <= dateobject <= latest:
                return True
    LOGGER.debug('date not valid: %s', date_input)
    return False def get_max_date(max_date):
    '''Validates the maximum date and/or defaults to latest plausible date'''
    if max_date is not None:
        try:
            # internal conversion from Y-M-D format
            max_date = datetime.date(int(max_date[:4]),
                                     int(max_date[5:7]),
                                     int(max_date[8:10]))
        except ValueError:
            max_date = LATEST_POSSIBLE
    else:
        max_date = LATEST_POSSIBLE
    return max_date def get_min_date(min_date):
    '''Validates the minimum date and/or defaults to earliest plausible date'''
    if min_date is not None:
        try:
            # internal conversion from Y-M-D format
            min_date = datetime.date(int(min_date[:4]),
                                     int(min_date[5:7]),
                                     int(min_date[8:10]))
        except ValueError:
            min_date = MIN_DATE
    else:
        min_date = MIN_DATE
    return min_date def output_format_validator(outputformat):
    """Validate the output format in the settings"""
    # test in abstracto
    if not isinstance(outputformat, str) or '%' not in outputformat:
        LOGGER.error('malformed output format: %s', outputformat)
        return False
    # test with date object
    dateobject = datetime.datetime(2017, 9, 1, 0, 0)
    try:
        dateobject.strftime(outputformat)
    # Python < 3.7 only
    except (NameError, TypeError, UnicodeError, ValueError) as err:
        LOGGER.error('wrong output format or type: %s %s', outputformat, err)
        return False
    return True
import datetime
import io
import logging
import os
import re
import sys
import pytest
    import dateparser
from collections import Counter
from contextlib import redirect_stdout
from unittest.mock import Mock, patch
from lxml import html
from htmldate.cli import examine, main, parse_args, process_args
from htmldate.core import compare_reference, examine_date_elements, find_date, search_page, search_pattern, select_candidate, try_ymd_date
from htmldate.extractors import custom_parse, external_date_parser, extract_partial_url_date, regex_parse
from htmldate.settings import MIN_DATE, LATEST_POSSIBLE
from htmldate.utils import decode_response, detect_encoding, fetch_url, load_html, is_dubious_html
from htmldate.validators import convert_date, date_validator, get_max_date, get_min_date, output_format_validator
# pylint:disable-msg=W1401
"""
Unit tests for the htmldate library.
"""





try:
    EXT_PARSER = True
    PARSER = dateparser.DateDataParser(languages=['de', 'en'], settings={'PREFER_DAY_OF_MONTH': 'first', 'PREFER_DATES_FROM': 'past', 'DATE_ORDER': 'DMY'}) # allow_redetect_language=False,
except ImportError:
    EXT_PARSER = False




TEST_DIR = os.path.abspath(os.path.dirname(__file__))
OUTPUTFORMAT = '%Y-%m-%d'

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

MOCK_PAGES = {
'http://blog.kinra.de/?p=959/': 'kinra.de.html',
'http://blog.python.org/2016/12/python-360-is-now-available.html': 'blog.python.org.html',
'http://blog.todamax.net/2018/midp-emulator-kemulator-und-brick-challenge/': 'blog.todamax.net.html',
'http://carta.info/der-neue-trend-muss-statt-wunschkoalition/': 'carta.info.html',
'https://500px.com/photo/26034451/spring-in-china-by-alexey-kruglov': '500px.com.spring.html',
'https://bayern.de/': 'bayern.de.html',
'https://creativecommons.org/about/': 'creativecommons.org.html',
'https://die-partei.net/sh/': 'die-partei.net.sh.html',
'https://en.blog.wordpress.com/': 'blog.wordpress.com.html',
'https://en.support.wordpress.com/': 'support.wordpress.com.html',
'https://futurezone.at/digital-life/wie-creativecommons-richtig-genutzt-wird/24.600.504': 'futurezone.at.cc.html',
'https://github.com/adbar/htmldate': 'github.com.html',
'https://netzpolitik.org/2016/die-cider-connection-abmahnungen-gegen-nutzer-von-creative-commons-bildern/': 'netzpolitik.org.abmahnungen.html',
'https://pixabay.com/en/service/terms/': 'pixabay.com.tos.html',
'https://www.austria.info/': 'austria.info.html',
'https://www.befifty.de/home/2017/7/12/unter-uns-montauk': 'befifty.montauk.html',
'https://www.beltz.de/fachmedien/paedagogik/didacta_2019_in_koeln_19_23_februar/beltz_veranstaltungen_didacta_2016/veranstaltung.html?tx_news_pi1%5Bnews%5D=14392&tx_news_pi1%5Bcontroller%5D=News&tx_news_pi1%5Baction%5D=detail&cHash=10b1a32fb5b2b05360bdac257b01c8fa': 'beltz.de.didakta.html',
'https://www.channelpartner.de/a/sieben-berufe-die-zukunft-haben,3050673': 'channelpartner.de.berufe.html',
'https://www.creativecommons.at/faircoin-hackathon': 'creativecommons.at.faircoin.html',
'https://www.deutschland.de/en': 'deutschland.de.en.html',
'https://www.eff.org/files/annual-report/2015/index.html': 'eff.org.2015.html',
'https://www.facebook.com/visitaustria/': 'facebook.com.visitaustria.html',
'https://www.gnu.org/licenses/gpl-3.0.en.html': 'gnu.org.gpl.html',
'https://www.goodform.ch/blog/schattiges_plaetzchen': 'goodform.ch.blog.html',
'https://www.horizont.net/marketing/kommentare/influencer-marketing-was-sich-nach-dem-vreni-frost-urteil-aendert-und-aendern-muss-172529': 'horizont.net.html',
'https://www.intel.com/content/www/us/en/legal/terms-of-use.html': 'intel.com.tos.html',
'https://www.pferde-fuer-unsere-kinder.de/unsere-projekte/': 'pferde.projekte.de.html',
'https://www.rosneft.com/business/Upstream/Licensing/': 'rosneft.com.licensing.html',
'https://www.scs78.de/news/items/warm-war-es-schoen-war-es.html': 'scs78.de.html',
'https://www.tagesausblick.de/Analyse/USA/DOW-Jones-Jahresendrally-ade__601.html': 'tagesausblick.de.dow.html',
'https://www.transgen.de/aktuell/2687.afrikanische-schweinepest-genome-editing.html': 'transgen.de.aktuell.html',
'https://www.weltwoche.ch/ausgaben/2019-4/artikel/forbes-die-weltwoche-ausgabe-4-2019.html': 'weltwoche.ch.html',
'https://www.wunderweib.de/manuela-reimann-hochzeitsueberraschung-in-bayern-107930.html': 'wunderweib.html',
'http://unexpecteduser.blogspot.de/2011/': 'unexpecteduser.2011.html',
'http://viehbacher.com/de/spezialisierung/internationale-forderungsbeitreibung': 'viehbacher.com.forderungsbetreibung.html',
'http://www.eza.gv.at/das-ministerium/presse/aussendungen/2018/07/aussenministerin-karin-kneissl-beim-treffen-der-deutschsprachigen-aussenminister-in-luxemburg/': 'eza.gv.at.html',
'http://www.freundeskreis-videoclips.de/waehlen-sie-car-player-tipps-zur-auswahl-der-besten-car-cd-player/': 'freundeskreis-videoclips.de.html',
'http://www.greenpeace.org/international/en/campaigns/forests/asia-pacific/': 'greenpeace.org.forests.html',
'http://www.heimicke.de/chronik/zahlen-und-daten/': 'heimicke.de.zahlen.html',
'http://www.hobby-werkstatt-blog.de/arduino/424-eine-arduino-virtual-wall-fuer-den-irobot-roomba.php': 'hobby-werkstatt-blog.de.roomba.html',
'http://www.hundeverein-kreisunna.de/termine.html': 'hundeverein-kreisunna.de.html',
'http://www.hundeverein-querfurt.de/index.php?option=com_content&view=article&id=54&Itemid=50': 'hundeverein-querfurt.de.html',
'http://www.jovelstefan.de/2012/05/11/parken-in-paris/': 'jovelstefan.de.parken.html',
'http://www.klimawandel-global.de/klimaschutz/energie-sparen/elektromobilitat-der-neue-trend/': 'klimawandel-global.de.html',
'http://www.medef.com/en/content/alternative-dispute-resolution-for-antitrust-damages': 'medef.fr.dispute.html',
'http://www.pbrunst.de/news/2011/12/kein-cyberterrorismus-diesmal/': 'pbrunst.de.html',
'http://www.stuttgart.de/': 'stuttgart.de.html',
'https://paris-luttes.info/quand-on-comprend-que-les-grenades-12355': 'paris-luttes.info.html',
'https://www.brigitte.de/aktuell/riverdale--so-ehrt-die-serie-luke-perry-in-staffel-vier-11602344.html': 'brigitte.de.riverdale.html',
'https://www.ldt.de/ldtblog/fall-in-love-with-black/': 'ldt.de.fallinlove.html',
'http://www.loldf.org/spip.php?article717': 'loldf.org.html',
'https://www.beltz.de/sachbuch_ratgeber/buecher/produkt_produktdetails/37219-12_wege_zu_guter_pflege.html': 'beltz.de.12wege.html',
'https://www.oberstdorf-resort.de/interaktiv/blog/unser-kraeutergarten-wannenkopfhuette.html': 'oberstdorfresort.de.kraeuter.html',
'https://www.wienbadminton.at/news/119843/Come-Together': 'wienbadminton.at.html',
'https://blog.wikimedia.org/2018/06/28/interactive-maps-now-in-your-language/': 'blog.wikimedia.interactivemaps.html',
'https://blogs.mediapart.fr/elba/blog/260619/violences-policieres-bombe-retardement-mediatique': 'mediapart.fr.violences.html',
'https://verfassungsblog.de/the-first-decade/': 'verfassungsblog.de.decade.html',
'https://cric-grenoble.info/infos-locales/article/putsh-en-cours-a-radio-kaleidoscope-1145': 'cric-grenoble.info.radio.html',
'https://www.sebastian-kurz.at/magazin/wasserstoff-als-schluesseltechnologie': 'kurz.at.wasserstoff.html',
'https://la-bas.org/la-bas-magazine/chroniques/Didier-Porte-souhaite-la-Sante-a-Balkany': 'la-bas.org.porte.html',
'https://exporo.de/wiki/europaeische-zentralbank-ezb/': 'exporo.de.ezb.html',
'https://www.revolutionpermanente.fr/Antonin-Bernanos-en-prison-depuis-pres-de-deux-mois-en-raison-de-son-militantisme': 'revolutionpermanente.fr.antonin.html',
'http://www.wara-enforcement.org/guinee-un-braconnier-delephant-interpelle-et-condamne-a-la-peine-maximale/': 'wara-enforcement.org.guinee.html',
'https://ebene11.com/die-arbeit-mit-fremden-dwg-dateien-in-autocad': 'ebene11.com.autocad.html',
'https://www.acredis.com/schoenheitsoperationen/augenlidstraffung/': 'acredis.com.augenlidstraffung.html',
'https://www.hertie-school.org/en/debate/detail/content/whats-on-the-cards-for-von-der-leyen/': 'hertie-school.org.leyen.html',
'https://www.adac.de/rund-ums-fahrzeug/tests/kindersicherheit/kindersitztest-2018/': 'adac.de.kindersitztest.html',
'http://web.archive.org/web/20210916140120/https://www.kath.ch/die-insel-der-klosterzoeglinge/': 'archive.org.kath.ch.html',
}

MEDIACLOUD_PAGES = {
    'pagename' : 'thing.html'
}


def load_mock_page(url, dirname='cache', dictname=MOCK_PAGES):
    '''load mock page from samples'''
    with open(os.path.join(TEST_DIR, dirname, dictname[url]), 'rb') as inputf:
        return inputf.read()


def load_mediacloud_page(url):
    '''load mediacloud page from samples'''
    return load_mock_page(url, dirname='test_set', dictname=MEDIACLOUD_PAGES)


def test_input():
    '''test if loaded strings/trees are handled properly'''
    assert is_dubious_html('This is a string.') is True
    assert is_dubious_html(b'This is a string.') is True
    with pytest.raises(TypeError) as err:

        assert load_html(123) is None