

def extract_data(invoicefile, templates=None, debug=False):
    if debug:
        logging.basicConfig(level=logging.DEBUG)
    output = {
        
    }
    if (templates is None):
        templates = read_templates(pkg_resources.resource_filename('invoice2data', 'templates'))
    extracted_str = pdftotext.to_text(invoicefile).decode('utf-8')
    charcount = len(extracted_str)
    logger.debug('number of char in pdf2text extract: %d', charcount)
    logger.debug('START pdftotext result ===========================')
    logger.debug(extracted_str)
    logger.debug('END pdftotext result =============================')
    logger.debug('Testing {} template files'.format(len(templates)))
    for t in templates:
        run_options = OPTIONS_DEFAULT.copy()
        if ('options' in t):
            run_options.update(t['options'])
        if run_options['remove_whitespace']:
            optimized_str = re.sub(' +', '', extracted_str)
        else:
            optimized_str = extracted_str
        if run_options['remove_accents']:
            optimized_str = unidecode(optimized_str)
        for replace in run_options['replace']:
            assert (len(replace) == 2), 'A replace should be a list of 2 items'
            optimized_str = optimized_str.replace(replace[0], replace[1])
        if all([(keyword in optimized_str) for keyword in t['keywords']]):
            logger.debug('Matched template %s', t['template_name'])
            logger.debug('START optimized_str ========================')
            logger.debug(optimized_str)
            logger.debug('END optimized_str ==========================')
            date_formats = run_options['date_formats']
            languages = run_options['languages']
            decimal_sep = run_options['decimal_separator']
            for lang in languages:
                assert (len(lang) == 2), 'lang code must have 2 letters'
            logger.debug('Date parsing: languages=%s date_formats=%s', languages, date_formats)
            logger.debug('Float parsing: decimal separator=%s', decimal_sep)
            logger.debug('keywords=%s', t['keywords'])
            logger.debug(run_options)
            for (k, v) in t['fields'].items():
                if k.startswith('static_'):
                    logger.debug('field=%s | static value=%s', k, v)
                    output[k.replace('static_', '')] = v
                else:
                    logger.debug('field=%s | regexp=%s', k, v)
                    if (type(v) is list):
                        for v_option in v:
                            res_find = re.findall(v_option, optimized_str)
                            if res_find:
                                break
                    else:
                        res_find = re.findall(v, optimized_str)
                    if res_find:
                        logger.debug('res_find=%s', res_find)
                        if k.startswith('date'):
                            raw_date = res_find[0]
                            output[k] = dateparser.parse(raw_date, date_formats=date_formats, languages=languages)
                            logger.debug('result of date parsing=%s', output[k])
                            if (not output[k]):
                                logger.error("Date parsing failed on date '%s'", raw_date)
                                return None
                        elif k.startswith('amount'):
                            assert (res_find[0].count(decimal_sep) < 2), 'Decimal separator cannot be present several times'
                            amount_pipe = res_find[0].replace(decimal_sep, '|')
                            amount_pipe_no_thousand_sep = re.sub('[.,\\s]', '', amount_pipe)
                            amount_regular = amount_pipe_no_thousand_sep.replace('|', '.')
                            output[k] = float(amount_regular)
                        else:
                            output[k] = res_find[0]
                    else:
                        logger.warning("regexp for field %s didn't match", k)
            if ('issuer' not in t.keys()):
                identifier = t['keywords'][0]
            else:
                identifier = t['issuer']
            output['currency'] = run_options['currency']
            if (len(output.keys()) >= 4):
                output['desc'] = ('Invoice %s from %s' % (output['invoice_number'], identifier))
                logger.debug(output)
                return output
            else:
                logger.error('Missing some fields for file %s', invoicefile)
                logger.error(output)
                return None
    logger.error('No template for %s', invoicefile)
    logger.debug(output)
    return False

dataset/ETHPy150Open m3nu/invoice2data/invoice2data/main.py extract_data/original