{"action":"created","comment":{"url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/comments/166829148","pull_request_review_id":94953054,"id":166829148,"diff_hunk":"@@ -0,0 +1,465 @@\n+import boto3\n+import csv\n+import os\n+import json\n+import pandas as pd\n+import pytz\n+import subprocess\n+import tempfile\n+\n+from collections import defaultdict\n+from datetime import datetime\n+from django.db import connection\n+from elasticsearch import helpers\n+from time import perf_counter, sleep\n+\n+AWARD_DESC_CATEGORIES = {\n+    'loans': 'loans',\n+    'grant': 'grants',\n+    'insurance': 'other',\n+    'other': 'other',\n+    'contract': 'contracts',\n+    'direct payment': 'directpayments'\n+}\n+\n+UNIVERSAL_TRANSACTION_ID_NAME = 'generated_unique_transaction_id'\n+\n+\n+class DataJob:\n+    def __init__(self, *args):\n+        self.name = args[0]\n+        self.index = args[1]\n+        self.fy = args[2]\n+        self.category = args[3]\n+        self.csv = args[4]\n+        self.count = None\n+\n+\n+def configure_sql_strings(config, filename, deleted_ids):\n+    '''\n+    Populates the formatted strings defined globally in this file to create the desired SQL\n+    '''\n+    update_date_str = UPDATE_DATE_SQL.format(config['starting_date'].strftime('%Y-%m-%d'))\n+    award_type_str = ''\n+    if config['award_category']:\n+        if config['award_category'] == 'contract':\n+            award_type_str = CONTRACTS_IDV_SQL.format(config['award_category'])\n+        else:\n+            award_type_str = CATEGORY_SQL.format(config['award_category'])\n+\n+    copy_sql = COPY_SQL.format(\n+        fy=config['fiscal_year'],\n+        update_date=update_date_str,\n+        award_category=award_type_str,\n+        filename=filename)\n+\n+    count_sql = COUNT_SQL.format(\n+        fy=config['fiscal_year'],\n+        update_date=update_date_str,\n+        award_category=award_type_str,\n+    )\n+\n+    if deleted_ids and config['provide_deleted']:\n+        id_list = ','.join([\"('{}')\".format(x) for x in deleted_ids.keys()])\n+        id_sql = CHECK_IDS_SQL.format(id_list=id_list, fy=config['fiscal_year'])\n+    else:\n+        id_sql = None\n+\n+    return copy_sql, id_sql, count_sql\n+\n+\n+def execute_sql_statement(cmd, results=False, verbose=False):\n+    ''' Simple function to execute SQL using the Django DB connection'''\n+    rows = None\n+    if verbose:\n+        print(cmd)\n+    with connection.cursor() as cursor:\n+        cursor.execute(cmd)\n+        if results:\n+            rows = db_rows_to_dict(cursor)\n+    return rows\n+\n+\n+def db_rows_to_dict(cursor):\n+    ''' Return a dictionary of all row results from a database connection cursor '''\n+    columns = [col[0] for col in cursor.description]\n+    return [\n+        dict(zip(columns, row))\n+        for row in cursor.fetchall()\n+    ]\n+\n+\n+def download_db_records(fetch_jobs, done_jobs, config):\n+    while not fetch_jobs.empty():\n+        if done_jobs.full():\n+            printf({'msg': 'Waiting 60s reduce temporary disk space used', 'f': 'Download'})\n+            sleep(60)\n+        else:\n+            start = perf_counter()\n+            job = fetch_jobs.get_nowait()\n+            printf({'msg': 'Preparing to download \"{}\"'.format(job.csv), 'job': job.name, 'f': 'Download'})\n+\n+            sql_config = {\n+                'starting_date': config['starting_date'],\n+                'fiscal_year': job.fy,\n+                'award_category': job.category,\n+                'provide_deleted': config['provide_deleted']\n+            }\n+            copy_sql, _, count_sql = configure_sql_strings(sql_config, job.csv, [])\n+\n+            if os.path.isfile(job.csv):\n+                os.remove(job.csv)\n+\n+            job.count = download_csv(count_sql, copy_sql, job.csv, job.name, config['verbose'])\n+            done_jobs.put(job)\n+            printf({\n+                'msg': 'CSV \"{}\" copy took {} seconds'.format(job.csv, perf_counter() - start),\n+                'job': job.name,\n+                'f': 'Download'\n+            })\n+            sleep(1)\n+\n+    done_jobs.put(DataJob(None, None, None, None, None))\n+    printf({'msg': 'All downloads from Postgres completed', 'f': 'Download'})\n+    return\n+\n+\n+def download_csv(count_sql, copy_sql, filename, job_id, verbose):\n+    count = execute_sql_statement(count_sql, True, verbose)[0]['count']\n+    printf({'msg': 'Writing {} transactions to this file: {}'.format(count, filename), 'job': job_id, 'f': 'Download'})\n+    # It is preferable to not use shell=True, but this command works. Limited user-input so risk is low\n+    subprocess.Popen('psql \"${{DATABASE_URL}}\" -c {}'.format(copy_sql), shell=True).wait()\n+\n+    download_count = csv_row_count(filename)\n+    if count != download_count:\n+        msg = 'Mismatch between CSV and DB rows! Expected: {} | Actual {} in: {}'\n+        printf({\n+            'msg': msg.format(count, download_count, filename),\n+            'job': job_id,\n+            'f': 'Download'})\n+\n+    return count\n+\n+\n+def csv_chunk_gen(filename, chunksize, job_id):\n+    printf({'msg': 'Opening {} (batch size = {})'.format(filename, chunksize), 'job': job_id, 'f': 'ES Ingest'})\n+    # Panda's data type guessing causes issues for Elasticsearch. Explicitly cast using dictionary\n+    dtype = {k: str for k in VIEW_COLUMNS}\n+    for file_df in pd.read_csv(filename, dtype=dtype, header=0, chunksize=chunksize):\n+        file_df = file_df.where(cond=(pd.notnull(file_df)), other=None)\n+        yield file_df.to_dict(orient='records')\n+\n+\n+def es_data_loader(client, fetch_jobs, done_jobs, config):\n+    while True:\n+        if not done_jobs.empty():\n+            job = done_jobs.get_nowait()\n+            if job.name is None:\n+                break\n+\n+            printf({'msg': 'Starting new job', 'job': job.name, 'f': 'ES Ingest'})\n+            post_to_elasticsearch(client, job, config)\n+            if os.path.exists(job.csv) and not config['keep']:\n+                os.remove(job.csv)\n+        else:\n+            printf({'msg': 'No Job :-( Sleeping 15s', 'f': 'ES Ingest'})\n+            sleep(15)\n+\n+    printf({'msg': 'Completed Elasticsearch data load', 'f': 'ES Ingest'})\n+    return\n+\n+\n+def streaming_post_to_es(client, chunk, index_name, job_id=None):\n+    success, failed = 0, 0\n+    try:\n+        for ok, item in helpers.streaming_bulk(client, chunk, index=index_name, doc_type='transaction_mapping'):\n+            success = [success, success + 1][ok]\n+            failed = [failed + 1, failed][ok]\n+\n+    except Exception as e:\n+        print('MASSIVE FAIL!!!\\n\\n{}\\n\\n{}'.format(str(e)[:5000], '*' * 80))\n+        raise SystemExit\n+\n+    printf({'msg': 'Success: {}, Fails: {}'.format(success, failed), 'job': job_id, 'f': 'ES Ingest'})\n+    return success, failed\n+\n+\n+def post_to_elasticsearch(client, job, config, chunksize=250000):\n+    printf({'msg': 'Populating ES Index \"{}\"'.format(job.index), 'job': job.name, 'f': 'ES Ingest'})\n+    start = perf_counter()\n+    try:\n+        does_index_exist = client.indices.exists(job.index)\n+    except Exception as e:\n+        print(e)\n+        raise SystemExit\n+    if not does_index_exist:\n+        printf({'msg': 'Creating index \"{}\"'.format(job.index), 'job': job.name, 'f': 'ES Ingest'})\n+        client.indices.create(index=job.index)  # removed body paramter since behavior wasn't reliable\n+    elif does_index_exist and config['recreate']:\n+        printf({'msg': 'Deleting existing index \"{}\"'.format(job.index), 'job': job.name, 'f': 'ES Ingest'})\n+        client.indices.delete(job.index)\n+\n+    csv_generator = csv_chunk_gen(job.csv, chunksize, job.name)\n+    for count, chunk in enumerate(csv_generator):\n+        if len(chunk) == 0:\n+            printf({'msg': 'No documents to add/delete for chunk #{}'.format(count), 'f': 'ES Ingest', 'job': job.name})\n+            continue\n+        iteration = perf_counter()\n+        if config['recreate'] is False:\n+            id_list = [{'key': c[UNIVERSAL_TRANSACTION_ID_NAME], 'col': UNIVERSAL_TRANSACTION_ID_NAME} for c in chunk]\n+            delete_transactions_from_es(client, id_list, job.name, config, job.index)\n+\n+        current_rows = '({}-{})'.format(count * chunksize + 1, count * chunksize + len(chunk))\n+        printf({\n+            'msg': 'Streaming to ES #{} rows [{}/{}]'.format(count, current_rows, job.count),\n+            'job': job.name,\n+            'f': 'ES Ingest'\n+        })\n+        streaming_post_to_es(client, chunk, job.index, job.name)\n+        printf({\n+            'msg': 'Iteration group #{} took {}s'.format(count, perf_counter() - iteration),\n+            'job': job.name,\n+            'f': 'ES Ingest'\n+        })\n+    printf({\n+        'msg': 'Elasticsearch Index loading took {}s'.format(perf_counter() - start),\n+        'job': job.name,\n+        'f': 'ES Ingest'\n+    })\n+\n+\n+def deleted_transactions(client, config):\n+    deleted_ids = gather_deleted_ids(config)\n+    id_list = [{'key': c, 'col': UNIVERSAL_TRANSACTION_ID_NAME} for c in deleted_ids]\n+    delete_transactions_from_es(client, id_list, None, config, None)\n+\n+\n+def gather_deleted_ids(config):\n+    '''\n+    Connect to S3 and gather all of the transaction ids stored in CSV files\n+    generated by the broker when transactions are removed from the DB.\n+    '''\n+\n+    if not config['provide_deleted']:\n+        printf({'msg': 'Skipping the S3 CSV fetch for deleted transactions'})\n+        return\n+    printf({'msg': 'Gathering all deleted transactions from S3'})\n+    start = perf_counter()\n+    try:\n+        s3 = boto3.resource('s3', region_name=config['aws_region'])\n+        bucket = s3.Bucket(config['s3_bucket'])\n+        bucket_objects = list(bucket.objects.all())\n+    except Exception as e:\n+        print('\\n[ERROR]\\n')\n+        print('Verify settings.CSV_AWS_REGION and settings.DELETED_TRANSACTIONS_S3_BUCKET_NAME are correct')\n+        print('  or is using env variables: CSV_AWS_REGION and DELETED_TRANSACTIONS_S3_BUCKET_NAME')\n+        print('\\n{}\\n'.format(e))\n+        raise SystemExit\n+\n+    if config['verbose']:\n+        printf({'msg': 'CSV data from {} to now'.format(config['starting_date'])})\n+\n+    to_datetime = datetime.combine(config['starting_date'], datetime.min.time(), tzinfo=pytz.UTC)\n+    filtered_csv_list = [\n+        x for x in bucket_objects\n+        if (\n+            x.key.endswith('.csv') and\n+            not x.key.startswith('staging') and\n+            x.last_modified >= to_datetime\n+        )\n+    ]\n+\n+    if config['verbose']:\n+        printf({'msg': 'Found {} csv files'.format(len(filtered_csv_list))})\n+\n+    deleted_ids = {}\n+\n+    for obj in filtered_csv_list:\n+        # Use temporary files to facilitate date moving from csv files on S3 into pands\n+        (file, file_path) = tempfile.mkstemp()\n+        bucket.download_file(obj.key, file_path)\n+\n+        # Ingests the CSV into a dataframe. pandas thinks some ids are dates, so disable parsing\n+        data = pd.DataFrame.from_csv(file_path, parse_dates=False)\n+\n+        if 'detached_award_proc_unique' in data:\n+            new_ids = ['cont_tx_' + x for x in data['detached_award_proc_unique'].values]\n+        elif 'afa_generated_unique' in data:\n+            new_ids = ['asst_tx_' + x for x in data['afa_generated_unique'].values]\n+        else:\n+            printf({'msg': '  [Missing valid col] in {}'.format(obj.key)})\n+\n+        # Next statements are ugly, but properly handle the temp files\n+        os.close(file)\n+        os.remove(file_path)\n+\n+        for uid in new_ids:\n+            if uid in deleted_ids:\n+                if deleted_ids[uid]['timestamp'] < obj.last_modified:\n+                    deleted_ids[uid]['timestamp'] = obj.last_modified\n+            else:\n+                deleted_ids[uid] = {'timestamp': obj.last_modified}\n+\n+    if config['verbose']:\n+        for k, v in deleted_ids.items():\n+            printf({'msg': 'id: {} last modified: {}'.format(k, str(v['timestamp']))})\n+\n+    printf({'msg': 'Gathering {} deleted transactions took {}s'.format(len(deleted_ids), perf_counter() - start)})\n+    return deleted_ids\n+\n+\n+def filter_query(column, values, query_type=\"match_phrase\"):\n+    queries = [{query_type: {column: str(i)}} for i in values]\n+    return {\n+        \"query\": {\n+            \"bool\": {\n+                \"should\": [\n+                    queries\n+                ]\n+            }\n+        }\n+    }\n+\n+\n+def delete_query(response):\n+    return {\n+        \"query\": {\n+            \"ids\": {\n+                \"type\": \"transaction_mapping\",\n+                \"values\": [i['_id'] for i in response['hits']['hits']]\n+            }\n+        }\n+    }\n+\n+\n+def chunks(l, n):\n+    \"\"\"Yield successive n-sized chunks from l.\"\"\"\n+    for i in range(0, len(l), n):\n+        yield l[i:i + n]\n+\n+\n+def delete_transactions_from_es(client, id_list, job_id, config, index=None, size=50000):\n+    '''\n+    id_list = [{key:'key1',col:'tranaction_id'},\n+               {key:'key2',col:'generated_unique_transaction_id'}],\n+               ...]\n+    '''\n+    start = perf_counter()\n+\n+    printf({\n+        'msg': 'Deleting up to {} document(s)'.format(len(id_list)),\n+        'f': 'ES Delete',\n+        'job': job_id})\n+\n+    if index is None:\n+        index = '{}-*'.format(config['root_index'])\n+    start_ = client.search(index=index)['hits']['total']\n+    printf({\n+        'msg': 'Starting amount of indices ----- {}'.format(start_),\n+        'f': 'ES Delete',\n+        'job': job_id})\n+    col_to_items_dict = defaultdict(list)\n+    for l in id_list:\n+        col_to_items_dict[l['col']].append(l['key'])\n+\n+    for column, values in col_to_items_dict.items():\n+        printf({'msg': 'Deleting {} of \"{}\"'.format(len(values), column), 'f': 'ES Delete', 'job': job_id})\n+        values_generator = chunks(values, 1000)\n+        for v in values_generator:\n+            body = filter_query(column, v)\n+            response = client.search(index=index, body=json.dumps(body), size=size)\n+            delete_body = delete_query(response)\n+            try:\n+                client.delete_by_query(index=index, body=json.dumps(delete_body), size=size)\n+            except Exception as e:\n+                printf({'msg': '[ERROR][ERROR][ERROR]\\n{}'.format(str(e)), 'f': 'ES Delete', 'job': job_id})\n+    end_ = client.search(index=index)['hits']['total']\n+\n+    t = perf_counter() - start\n+    total = str(start_ - end_)\n+    printf({'msg': 'ES Deletes took {}s. Deleted {} records'.format(t, total), 'f': 'ES Delete', 'job': job_id})\n+    return\n+\n+\n+def printf(items):\n+    t = datetime.utcnow().strftime('%H:%M:%S.%f')\n+    msg = items['msg']\n+    if 'error' in items:\n+        template = '[{time}] [ERROR] {msg}'\n+        print_msg = template.format(time=t, msg=msg)\n+    else:\n+        template = '[{time}] {complex:<20} | {msg}'\n+        func = '[' + items.get('f', 'main') + ']'\n+        job = items.get('job', None)\n+        j = ''\n+        if job:\n+            j = ' (#{})'.format(job)\n+    print_msg = template.format(time=t, complex=func + j, msg=msg)\n+\n+    print(print_msg)\n+\n+\n+def csv_row_count(filename, has_header=True):","path":"usaspending_api/etl/es_etl_helpers.py","position":402,"original_position":402,"commit_id":"c5b13acb2faa09bb11b2e3507cc1413764809c7d","original_commit_id":"e06017bcae6f4161aeb1ec309f4024f148b4381b","user":{"login":"tony-sappe","id":22781949,"avatar_url":"https://avatars1.githubusercontent.com/u/22781949?v=4","gravatar_id":"","url":"https://api.github.com/users/tony-sappe","html_url":"https://github.com/tony-sappe","followers_url":"https://api.github.com/users/tony-sappe/followers","following_url":"https://api.github.com/users/tony-sappe/following{/other_user}","gists_url":"https://api.github.com/users/tony-sappe/gists{/gist_id}","starred_url":"https://api.github.com/users/tony-sappe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tony-sappe/subscriptions","organizations_url":"https://api.github.com/users/tony-sappe/orgs","repos_url":"https://api.github.com/users/tony-sappe/repos","events_url":"https://api.github.com/users/tony-sappe/events{/privacy}","received_events_url":"https://api.github.com/users/tony-sappe/received_events","type":"User","site_admin":false},"body":"Yeah, I had the same exact reaction. I thought there had to be a better way","created_at":"2018-02-08T04:15:46Z","updated_at":"2018-02-08T04:15:46Z","html_url":"https://github.com/fedspendingtransparency/usaspending-api/pull/946#discussion_r166829148","pull_request_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/946","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/comments/166829148"},"html":{"href":"https://github.com/fedspendingtransparency/usaspending-api/pull/946#discussion_r166829148"},"pull_request":{"href":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/946"}},"in_reply_to_id":166704258},"pull_request":{"url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/946","id":167528308,"html_url":"https://github.com/fedspendingtransparency/usaspending-api/pull/946","diff_url":"https://github.com/fedspendingtransparency/usaspending-api/pull/946.diff","patch_url":"https://github.com/fedspendingtransparency/usaspending-api/pull/946.patch","issue_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues/946","number":946,"state":"open","locked":false,"title":"New elasticsearch data loader","user":{"login":"tony-sappe","id":22781949,"avatar_url":"https://avatars1.githubusercontent.com/u/22781949?v=4","gravatar_id":"","url":"https://api.github.com/users/tony-sappe","html_url":"https://github.com/tony-sappe","followers_url":"https://api.github.com/users/tony-sappe/followers","following_url":"https://api.github.com/users/tony-sappe/following{/other_user}","gists_url":"https://api.github.com/users/tony-sappe/gists{/gist_id}","starred_url":"https://api.github.com/users/tony-sappe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tony-sappe/subscriptions","organizations_url":"https://api.github.com/users/tony-sappe/orgs","repos_url":"https://api.github.com/users/tony-sappe/repos","events_url":"https://api.github.com/users/tony-sappe/events{/privacy}","received_events_url":"https://api.github.com/users/tony-sappe/received_events","type":"User","site_admin":false},"body":"`es_rapidloader` is a new loading script which uses multiprocessing to download Postgres data and upload to Elasticsearch in separate processes.\r\n\r\n## This script relies on an index template for the index datatypes and other mapping info\r\n\r\nModified `es_configure` to create the new template and apply desired settings","created_at":"2018-02-06T21:02:40Z","updated_at":"2018-02-08T04:15:46Z","closed_at":null,"merged_at":null,"merge_commit_sha":"75375709453ad4a901a51132308096e23b53ece1","assignee":{"login":"dpb-bah","id":29313570,"avatar_url":"https://avatars2.githubusercontent.com/u/29313570?v=4","gravatar_id":"","url":"https://api.github.com/users/dpb-bah","html_url":"https://github.com/dpb-bah","followers_url":"https://api.github.com/users/dpb-bah/followers","following_url":"https://api.github.com/users/dpb-bah/following{/other_user}","gists_url":"https://api.github.com/users/dpb-bah/gists{/gist_id}","starred_url":"https://api.github.com/users/dpb-bah/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dpb-bah/subscriptions","organizations_url":"https://api.github.com/users/dpb-bah/orgs","repos_url":"https://api.github.com/users/dpb-bah/repos","events_url":"https://api.github.com/users/dpb-bah/events{/privacy}","received_events_url":"https://api.github.com/users/dpb-bah/received_events","type":"User","site_admin":false},"assignees":[{"login":"dpb-bah","id":29313570,"avatar_url":"https://avatars2.githubusercontent.com/u/29313570?v=4","gravatar_id":"","url":"https://api.github.com/users/dpb-bah","html_url":"https://github.com/dpb-bah","followers_url":"https://api.github.com/users/dpb-bah/followers","following_url":"https://api.github.com/users/dpb-bah/following{/other_user}","gists_url":"https://api.github.com/users/dpb-bah/gists{/gist_id}","starred_url":"https://api.github.com/users/dpb-bah/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dpb-bah/subscriptions","organizations_url":"https://api.github.com/users/dpb-bah/orgs","repos_url":"https://api.github.com/users/dpb-bah/repos","events_url":"https://api.github.com/users/dpb-bah/events{/privacy}","received_events_url":"https://api.github.com/users/dpb-bah/received_events","type":"User","site_admin":false}],"requested_reviewers":[{"login":"miahunsicker","id":8854444,"avatar_url":"https://avatars2.githubusercontent.com/u/8854444?v=4","gravatar_id":"","url":"https://api.github.com/users/miahunsicker","html_url":"https://github.com/miahunsicker","followers_url":"https://api.github.com/users/miahunsicker/followers","following_url":"https://api.github.com/users/miahunsicker/following{/other_user}","gists_url":"https://api.github.com/users/miahunsicker/gists{/gist_id}","starred_url":"https://api.github.com/users/miahunsicker/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/miahunsicker/subscriptions","organizations_url":"https://api.github.com/users/miahunsicker/orgs","repos_url":"https://api.github.com/users/miahunsicker/repos","events_url":"https://api.github.com/users/miahunsicker/events{/privacy}","received_events_url":"https://api.github.com/users/miahunsicker/received_events","type":"User","site_admin":false}],"requested_teams":[],"milestone":null,"commits_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/946/commits","review_comments_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/946/comments","review_comment_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/comments{/number}","comments_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues/946/comments","statuses_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/statuses/c5b13acb2faa09bb11b2e3507cc1413764809c7d","head":{"label":"fedspendingtransparency:update-elastic-data-and-config","ref":"update-elastic-data-and-config","sha":"c5b13acb2faa09bb11b2e3507cc1413764809c7d","user":{"login":"fedspendingtransparency","id":8397762,"avatar_url":"https://avatars0.githubusercontent.com/u/8397762?v=4","gravatar_id":"","url":"https://api.github.com/users/fedspendingtransparency","html_url":"https://github.com/fedspendingtransparency","followers_url":"https://api.github.com/users/fedspendingtransparency/followers","following_url":"https://api.github.com/users/fedspendingtransparency/following{/other_user}","gists_url":"https://api.github.com/users/fedspendingtransparency/gists{/gist_id}","starred_url":"https://api.github.com/users/fedspendingtransparency/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fedspendingtransparency/subscriptions","organizations_url":"https://api.github.com/users/fedspendingtransparency/orgs","repos_url":"https://api.github.com/users/fedspendingtransparency/repos","events_url":"https://api.github.com/users/fedspendingtransparency/events{/privacy}","received_events_url":"https://api.github.com/users/fedspendingtransparency/received_events","type":"Organization","site_admin":false},"repo":{"id":65394827,"name":"usaspending-api","full_name":"fedspendingtransparency/usaspending-api","owner":{"login":"fedspendingtransparency","id":8397762,"avatar_url":"https://avatars0.githubusercontent.com/u/8397762?v=4","gravatar_id":"","url":"https://api.github.com/users/fedspendingtransparency","html_url":"https://github.com/fedspendingtransparency","followers_url":"https://api.github.com/users/fedspendingtransparency/followers","following_url":"https://api.github.com/users/fedspendingtransparency/following{/other_user}","gists_url":"https://api.github.com/users/fedspendingtransparency/gists{/gist_id}","starred_url":"https://api.github.com/users/fedspendingtransparency/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fedspendingtransparency/subscriptions","organizations_url":"https://api.github.com/users/fedspendingtransparency/orgs","repos_url":"https://api.github.com/users/fedspendingtransparency/repos","events_url":"https://api.github.com/users/fedspendingtransparency/events{/privacy}","received_events_url":"https://api.github.com/users/fedspendingtransparency/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/fedspendingtransparency/usaspending-api","description":"An RESTful API for U.S. federal spending data.","fork":false,"url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api","forks_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/forks","keys_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/teams","hooks_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/hooks","issue_events_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues/events{/number}","events_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/events","assignees_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/assignees{/user}","branches_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/branches{/branch}","tags_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/tags","blobs_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/refs{/sha}","trees_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/statuses/{sha}","languages_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/languages","stargazers_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/stargazers","contributors_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/contributors","subscribers_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/subscribers","subscription_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/subscription","commits_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/commits{/sha}","git_commits_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/commits{/sha}","comments_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/comments{/number}","issue_comment_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues/comments{/number}","contents_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/contents/{+path}","compare_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/merges","archive_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/downloads","issues_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues{/number}","pulls_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls{/number}","milestones_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/milestones{/number}","notifications_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/labels{/name}","releases_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/releases{/id}","deployments_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/deployments","created_at":"2016-08-10T15:39:45Z","updated_at":"2018-01-29T17:30:32Z","pushed_at":"2018-02-08T03:53:38Z","git_url":"git://github.com/fedspendingtransparency/usaspending-api.git","ssh_url":"git@github.com:fedspendingtransparency/usaspending-api.git","clone_url":"https://github.com/fedspendingtransparency/usaspending-api.git","svn_url":"https://github.com/fedspendingtransparency/usaspending-api","homepage":"","size":47858,"stargazers_count":52,"watchers_count":52,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":14,"mirror_url":null,"archived":false,"open_issues_count":9,"license":null,"forks":14,"open_issues":9,"watchers":52,"default_branch":"dev"}},"base":{"label":"fedspendingtransparency:dev","ref":"dev","sha":"109f008f5c52ee72f605b1faf7215d9b97b567c0","user":{"login":"fedspendingtransparency","id":8397762,"avatar_url":"https://avatars0.githubusercontent.com/u/8397762?v=4","gravatar_id":"","url":"https://api.github.com/users/fedspendingtransparency","html_url":"https://github.com/fedspendingtransparency","followers_url":"https://api.github.com/users/fedspendingtransparency/followers","following_url":"https://api.github.com/users/fedspendingtransparency/following{/other_user}","gists_url":"https://api.github.com/users/fedspendingtransparency/gists{/gist_id}","starred_url":"https://api.github.com/users/fedspendingtransparency/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fedspendingtransparency/subscriptions","organizations_url":"https://api.github.com/users/fedspendingtransparency/orgs","repos_url":"https://api.github.com/users/fedspendingtransparency/repos","events_url":"https://api.github.com/users/fedspendingtransparency/events{/privacy}","received_events_url":"https://api.github.com/users/fedspendingtransparency/received_events","type":"Organization","site_admin":false},"repo":{"id":65394827,"name":"usaspending-api","full_name":"fedspendingtransparency/usaspending-api","owner":{"login":"fedspendingtransparency","id":8397762,"avatar_url":"https://avatars0.githubusercontent.com/u/8397762?v=4","gravatar_id":"","url":"https://api.github.com/users/fedspendingtransparency","html_url":"https://github.com/fedspendingtransparency","followers_url":"https://api.github.com/users/fedspendingtransparency/followers","following_url":"https://api.github.com/users/fedspendingtransparency/following{/other_user}","gists_url":"https://api.github.com/users/fedspendingtransparency/gists{/gist_id}","starred_url":"https://api.github.com/users/fedspendingtransparency/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fedspendingtransparency/subscriptions","organizations_url":"https://api.github.com/users/fedspendingtransparency/orgs","repos_url":"https://api.github.com/users/fedspendingtransparency/repos","events_url":"https://api.github.com/users/fedspendingtransparency/events{/privacy}","received_events_url":"https://api.github.com/users/fedspendingtransparency/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/fedspendingtransparency/usaspending-api","description":"An RESTful API for U.S. federal spending data.","fork":false,"url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api","forks_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/forks","keys_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/teams","hooks_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/hooks","issue_events_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues/events{/number}","events_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/events","assignees_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/assignees{/user}","branches_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/branches{/branch}","tags_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/tags","blobs_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/refs{/sha}","trees_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/statuses/{sha}","languages_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/languages","stargazers_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/stargazers","contributors_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/contributors","subscribers_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/subscribers","subscription_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/subscription","commits_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/commits{/sha}","git_commits_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/git/commits{/sha}","comments_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/comments{/number}","issue_comment_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues/comments{/number}","contents_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/contents/{+path}","compare_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/merges","archive_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/downloads","issues_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues{/number}","pulls_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls{/number}","milestones_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/milestones{/number}","notifications_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/labels{/name}","releases_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/releases{/id}","deployments_url":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/deployments","created_at":"2016-08-10T15:39:45Z","updated_at":"2018-01-29T17:30:32Z","pushed_at":"2018-02-08T03:53:38Z","git_url":"git://github.com/fedspendingtransparency/usaspending-api.git","ssh_url":"git@github.com:fedspendingtransparency/usaspending-api.git","clone_url":"https://github.com/fedspendingtransparency/usaspending-api.git","svn_url":"https://github.com/fedspendingtransparency/usaspending-api","homepage":"","size":47858,"stargazers_count":52,"watchers_count":52,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":14,"mirror_url":null,"archived":false,"open_issues_count":9,"license":null,"forks":14,"open_issues":9,"watchers":52,"default_branch":"dev"}},"_links":{"self":{"href":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/946"},"html":{"href":"https://github.com/fedspendingtransparency/usaspending-api/pull/946"},"issue":{"href":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues/946"},"comments":{"href":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/issues/946/comments"},"review_comments":{"href":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/946/comments"},"review_comment":{"href":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/pulls/946/commits"},"statuses":{"href":"https://api.github.com/repos/fedspendingtransparency/usaspending-api/statuses/c5b13acb2faa09bb11b2e3507cc1413764809c7d"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"tony-sappe"}}