<reponame>akitenkrad/show-us-the-data
from typing import List, Dict
import os
from codecs import ignore_errors
from pathlib import Path
import urllib.request
from glob import glob
from tqdm import tqdm
from concurrent.futures import ProcessPoolExecutor
import re
import csv
import json
import numpy as np
import pandas as pd
from collections import namedtuple
from nltk.tokenize import sent_tokenize
import click

def clean_text(txt):
    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()

@click.group()
def cli():
    pass

@cli.command()
@click.option('--out-path', type=click.Path(), help='path to output.csv')
def get_external_dataset(out_path):
    out_path = Path(out_path)
    assert out_path.suffix == '.csv'

    api_list = [
        'http://www.usda.gov/data.json',
        'http://www.commerce.gov/data.json',
        'http://www.ed.gov/data.json',
        'https://healthdata.gov/data.json',
        'http://www.dol.gov/data.json',
        'https://www.doi.gov/sites/doi.gov/files/data.json',
        'http://www.energy.gov/data.json',
        'http://www.dhs.gov/data.json',
        'http://www.ustreas.gov/data.json',
        'http://www.dot.gov/data.json',
        'http://www.va.gov/data.json',
        'http://www.gsa.gov/data.json',
        'http://www.nsf.gov/data.json',
        'http://www.sba.gov/data.json',
        # 'http://www.ssa.gov/data.json',
        'http://www.usaid.gov/data.json'
    ]

    df = []
    for site in api_list:
        print(site)
        with urllib.request.urlopen(site) as url:
            data = json.loads(url.read().decode())
            df.append(pd.DataFrame(pd.json_normalize(data, 'dataset')))
    df = pd.concat(df, ignore_index=True).title
    df = df.apply(clean_text)
    df = pd.Series(df.sort_values().unique())
    out_path.parent.mkdir(parents=True, exist_ok=True)
    df.to_csv(str(out_path), index=False)
    print('----->', str(out_path.resolve().absolute()))


def exec_func(data):
    total, Id, cleaned_labels, ds_path, ext_datasets = data

    data_all = json.load(open(ds_path / 'train' / (Id + '.json')))
    out_file = ds_path / 'train_by_sent' / (Id + '.json')
    cur_cnt = int(np.array([1 for _ in glob(str(out_file.parent / '*.json'))]).sum())
    out_data = []
    for data in tqdm(data_all, desc='{: 5d}/{: 5d}: {}'.format(cur_cnt, total, Id), leave=False):
        texts = sent_tokenize(data['text'])
        item = {'section_text': data['text'], 'sentences': []}
        for text in texts:
            cleaned_text = clean_text(text)
            ds_labels = [label for label in (cleaned_labels+ext_datasets) if label.strip() in cleaned_text]
            item['sentences'].append({'text': text, 'cleaned_labels': ds_labels})
        out_data.append(item)

    out_file.parent.mkdir(parents=True, exist_ok=True)
    json.dump(out_data, open(out_file, 'w', encoding='utf-8'), indent=2, ensure_ascii=False)

@cli.command()
@click.option('--ds-path', type=click.Path(exists=True), help='path to dataset dir')
@click.option('--ext-ds', type=click.Path(exists=True), help='path to external dataset csv', default='')
def split_to_sentence(ds_path, ext_ds):
    ds_path = Path(ds_path)
    
    # load external dataset
    ext_datasets = []
    if ext_ds != '':
        ext_ds = Path(ext_ds)
        assert ext_ds.suffix == '.csv'
        ext_ds_list = csv.reader(open(ext_ds))
        ext_datasets = [ds for ds in ext_ds_list]

    # split original dataset into sentences
    ids, cleaned_labels = [], []
    train_csv = csv.reader(open(ds_path / 'train.csv'))
    # skip header
    next(train_csv)
    for line in train_csv:
        Id, _, _, _, cleaned_label = line
        ids.append(Id)
        cleaned_labels.append(cleaned_label)
    ids = sorted(list(set(ids)))
    cleaned_labels = sorted(list(set(cleaned_labels)))
    total = len(ids)

    data_all = [(total, Id, cleaned_labels, ds_path, ext_datasets) for Id in ids]
    with ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:
        executor.map(exec_func, data_all, chunksize=128)

if __name__ == '__main__':
    cli()

from typing import List, Dict
import os
from codecs import ignore_errors
from pathlib import Path
import urllib.request
from glob import glob
from tqdm import tqdm
from concurrent.futures import ProcessPoolExecutor
import re
import csv
import json
import numpy as np
import pandas as pd
from collections import namedtuple
from nltk.tokenize import sent_tokenize
import click

def clean_text(txt):
    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()

@click.group()
def cli():
    pass

@cli.command()
@click.option('--out-path', type=click.Path(), help='path to output.csv')
def get_external_dataset(out_path):
    out_path = Path(out_path)
    assert out_path.suffix == '.csv'

    api_list = [
        'http://www.usda.gov/data.json',
        'http://www.commerce.gov/data.json',
        'http://www.ed.gov/data.json',
        'https://healthdata.gov/data.json',
        'http://www.dol.gov/data.json',
        'https://www.doi.gov/sites/doi.gov/files/data.json',
        'http://www.energy.gov/data.json',
        'http://www.dhs.gov/data.json',
        'http://www.ustreas.gov/data.json',
        'http://www.dot.gov/data.json',
        'http://www.va.gov/data.json',
        'http://www.gsa.gov/data.json',
        'http://www.nsf.gov/data.json',
        'http://www.sba.gov/data.json',
        # 'http://www.ssa.gov/data.json',
        'http://www.usaid.gov/data.json'
    ]

    df = []
    for site in api_list:
        print(site)
        with urllib.request.urlopen(site) as url:
            data = json.loads(url.read().decode())
            df.append(pd.DataFrame(pd.json_normalize(data, 'dataset')))
    df = pd.concat(df, ignore_index=True).title
    df = df.apply(clean_text)
    df = pd.Series(df.sort_values().unique())
    out_path.parent.mkdir(parents=True, exist_ok=True)
    df.to_csv(str(out_path), index=False)
    print('----->', str(out_path.resolve().absolute()))


def exec_func(data):
    total, Id, cleaned_labels, ds_path, ext_datasets = data

    data_all = json.load(open(ds_path / 'train' / (Id + '.json')))
    out_file = ds_path / 'train_by_sent' / (Id + '.json')
    cur_cnt = int(np.array([1 for _ in glob(str(out_file.parent / '*.json'))]).sum())
    out_data = []
    for data in tqdm(data_all, desc='{: 5d}/{: 5d}: {}'.format(cur_cnt, total, Id), leave=False):
        texts = sent_tokenize(data['text'])
        item = {'section_text': data['text'], 'sentences': []}
        for text in texts:
            cleaned_text = clean_text(text)
            ds_labels = [label for label in (cleaned_labels+ext_datasets) if label.strip() in cleaned_text]
            item['sentences'].append({'text': text, 'cleaned_labels': ds_labels})
        out_data.append(item)

    out_file.parent.mkdir(parents=True, exist_ok=True)
    json.dump(out_data, open(out_file, 'w', encoding='utf-8'), indent=2, ensure_ascii=False)

@cli.command()
@click.option('--ds-path', type=click.Path(exists=True), help='path to dataset dir')
@click.option('--ext-ds', type=click.Path(exists=True), help='path to external dataset csv', default='')
def split_to_sentence(ds_path, ext_ds):
    ds_path = Path(ds_path)
    
    # load external dataset
    ext_datasets = []
    if ext_ds != '':
        ext_ds = Path(ext_ds)
        assert ext_ds.suffix == '.csv'
        ext_ds_list = csv.reader(open(ext_ds))
        ext_datasets = [ds for ds in ext_ds_list]

    # split original dataset into sentences
    ids, cleaned_labels = [], []
    train_csv = csv.reader(open(ds_path / 'train.csv'))
    # skip header
    next(train_csv)
    for line in train_csv:
        Id, _, _, _, cleaned_label = line
        ids.append(Id)
        cleaned_labels.append(cleaned_label)
    ids = sorted(list(set(ids)))
    cleaned_labels = sorted(list(set(cleaned_labels)))
    total = len(ids)

    data_all = [(total, Id, cleaned_labels, ds_path, ext_datasets) for Id in ids]
    with ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:
        executor.map(exec_func, data_all, chunksize=128)

if __name__ == '__main__':
    cli()
# 'http://www.ssa.gov/data.json', # load external dataset # split original dataset into sentences # skip header