<reponame>johannesemme/ESA
# String data in a MARISA-trie may take up to 50x-100x less memory than in a standard Python dict; 
# the raw lookup speed is comparable; trie also provides fast advanced methods like prefix search.
from marisa_trie import Trie 

import logging
from logging import basicConfig
import re
import click
from typing import List
import numpy as np

KEY_RULE = re.compile("^(.*):([^:]+)$")

logger = logging.getLogger(__name__)


@click.command()
@click.argument("wikidata_dump_file", type=click.Path(exists=True))
@click.argument("out_file", type=click.Path())
def build_interwiki_db(wikidata_dump_file: str, out_file: List[str] = None):
    logging.basicConfig(level=logging.INFO)
    interwiki_db = WikiDB.build(wikidata_dump_file)
    interwiki_db.save(out_file)


class WikIDB(object):
    def __init__(self, title_trie: Trie, data: np.ndarray, indptr: np.ndarray, title_indices: np.ndarray):
        self._title_trie = title_trie
        self._data = data
        self._indptr = indptr
        self._title_indices = title_indices
# String data in a MARISA-trie may take up to 50x-100x less memory than in a standard Python dict; 
# the raw lookup speed is comparable; trie also provides fast advanced methods like prefix search.
from marisa_trie import Trie 

import logging
from logging import basicConfig
import re
import click
from typing import List
import numpy as np

KEY_RULE = re.compile("^(.*):([^:]+)$")

logger = logging.getLogger(__name__)


@click.command()
@click.argument("wikidata_dump_file", type=click.Path(exists=True))
@click.argument("out_file", type=click.Path())
def build_interwiki_db(wikidata_dump_file: str, out_file: List[str] = None):
    logging.basicConfig(level=logging.INFO)
    interwiki_db = WikiDB.build(wikidata_dump_file)
    interwiki_db.save(out_file)


class WikIDB(object):
    def __init__(self, title_trie: Trie, data: np.ndarray, indptr: np.ndarray, title_indices: np.ndarray):
        self._title_trie = title_trie
        self._data = data
        self._indptr = indptr
        self._title_indices = title_indices
# String data in a MARISA-trie may take up to 50x-100x less memory than in a standard Python dict; # the raw lookup speed is comparable; trie also provides fast advanced methods like prefix search.