#!/usr/bin/python
# -*- coding: utf-8  -*-
import re
import mwparserfromhell as wparser
import wikidataStuff.wdqsLookup as lookup


def remove_multiple_spaces(text):
    return re.sub(' +', ' ', text)


def remove_markup(text):
    remove_br = re.compile('<br.*?>\W*', re.I)
    text = remove_br.sub(' ', text)
    text = " ".join(text.split())
    if "[" in text or "''" in text:
        text = wparser.parse(text)
        text = text.strip_code()
    return remove_multiple_spaces(text.strip())


def get_wd_items_using_prop(prop):
    """
    Get WD items that already have some value of a unique ID.

    Even if there are none before we start working,
    it's still useful to have in case an upload is interrupted
    and has to be restarted, or if we later want to enhance
    some items. When matching, these should take predecence
    over any hardcoded matching files.

    The output is a dictionary of ID's and items
    that looks like this:
    {'4420': 'Q28936211', '2041': 'Q28933898'}
    """
    items = {}
    print("WILL NOW DOWNLOAD WD ITEMS THAT USE " + prop)
    query = "SELECT DISTINCT ?item ?value  WHERE {?item p:" + \
        prop + "?statement. OPTIONAL { ?item wdt:" + prop + " ?value. }}"
    data = lookup.make_simple_wdqs_query(query, verbose=False)
    for x in data:
        key = lookup.sanitize_wdqs_result(x['item'])
        value = x['value']
        items[value] = key
    print("FOUND {} WD ITEMS WITH PROP {}".format(len(items), prop))
    return items

#!/usr/bin/python
# -*- coding: utf-8  -*-
import re
import mwparserfromhell as wparser
import wikidataStuff.wdqsLookup as lookup


def remove_multiple_spaces(text):
    return re.sub(' +', ' ', text)


def remove_markup(text):
    remove_br = re.compile('<br.*?>\W*', re.I)
    text = remove_br.sub(' ', text)
    text = " ".join(text.split())
    if "[" in text or "''" in text:
        text = wparser.parse(text)
        text = text.strip_code()
    return remove_multiple_spaces(text.strip())


def get_wd_items_using_prop(prop):
    """
    Get WD items that already have some value of a unique ID.

    Even if there are none before we start working,
    it's still useful to have in case an upload is interrupted
    and has to be restarted, or if we later want to enhance
    some items. When matching, these should take predecence
    over any hardcoded matching files.

    The output is a dictionary of ID's and items
    that looks like this:
    {'4420': 'Q28936211', '2041': 'Q28933898'}
    """
    items = {}
    print("WILL NOW DOWNLOAD WD ITEMS THAT USE " + prop)
    query = "SELECT DISTINCT ?item ?value  WHERE {?item p:" + \
        prop + "?statement. OPTIONAL { ?item wdt:" + prop + " ?value. }}"
    data = lookup.make_simple_wdqs_query(query, verbose=False)
    for x in data:
        key = lookup.sanitize_wdqs_result(x['item'])
        value = x['value']
        items[value] = key
    print("FOUND {} WD ITEMS WITH PROP {}".format(len(items), prop))
    return items

#!/usr/bin/python # -*- coding: utf-8  -*- Get WD items that already have some value of a unique ID.      Even if there are none before we start working,     it's still useful to have in case an upload is interrupted     and has to be restarted, or if we later want to enhance     some items. When matching, these should take predecence     over any hardcoded matching files.      The output is a dictionary of ID's and items     that looks like this:     {'4420': 'Q28936211', '2041': 'Q28933898'}