

def add_census_data(output):
    census_fields = {
        '2010/sf1': [('P0010001', int, 'count'), ('P0180001', int, 'count'), ('P0180002', int, 'count'), ('H0040001', int, 'count'), ('H0050001', int, 'count')],
        '2011/acs5': [('B01003_001E', int, 'count'), ('B01002_001E', float, 'median'), ('B07001_001E', int, 'count'), ('B07001_017E', int, 'count'), ('B07001_033E', int, 'count'), ('B07001_065E', int, 'count'), ('B07001_081E', int, 'count'), ('B19019_001E', float, 'median'), ('B16002_001E', int, 'count')],
    }
    census_data = {
        
    }
    for division in ('tract', 'blockgroup'):
        dat = json.load(open(('data/smd-%s.json' % division)))
        for intsect in dat:
            id = intsect[division]['id']
            if (id in census_data):
                continue
            census_data[id] = {
                
            }
            (state_fips, county_fips, tract, bg_num) = (id[0:2], id[2:5], id[5:11], (id + '?')[11:12])
            for census_table in census_fields:
                field_list = ','.join((f[0] for f in census_fields[census_table]))
                if (division == 'tract'):
                    url = ('http://api.census.gov/data/%s?key=%s&get=%s&in=state:%s+county:%s&for=tract:%s' % (census_table, census_api_key, field_list, state_fips, county_fips, tract))
                else:
                    url = ('http://api.census.gov/data/%s?key=%s&get=%s&in=state:%s+county:%s+tract:%s&for=blockgroup:%s' % (census_table, census_api_key, field_list, state_fips, county_fips, tract, bg_num))
                data_from_api = json.load(urlopen(url))
                for (i, (fieldname, datatype, sum_mode)) in enumerate(census_fields[census_table]):
                    v = data_from_api[1][i]
                    if (v is not None):
                        v = datatype(v)
                    census_data[id][fieldname] = v
    for ward in output.values():
        for anc in ward['ancs'].values():
            anc['census'] = {
                'by-blockgroup': {
                    
                },
                'by-tract': {
                    
                },
            }
            for smd in anc['smds'].values():
                smd['census'] = {
                    'by-blockgroup': {
                        
                    },
                    'by-tract': {
                        
                    },
                }
    for division1 in ('smd', 'anc'):
        for division2 in ('tract', 'blockgroup'):
            dat = json.load(open(('data/%s-%s.json' % (division1, division2))))
            for ix in dat:
                ward = ix[division1]['id'][0]
                anc = ix[division1]['id'][1]
                if (division1 == 'anc'):
                    feature = output[ward]['ancs'][anc]
                else:
                    smd = ix[division1]['id'][2:]
                    feature = output[ward]['ancs'][anc]['smds'][smd]
                feature = feature['census'][('by-' + division2)]
                census_id = ix[division2]['id']
                for census_table in census_fields:
                    for (fieldname, datatype, summode) in census_fields[census_table]:
                        if (summode == 'count'):
                            if (census_data[census_id][fieldname] is None):
                                feature[fieldname] = 'missing-data'
                                continue
                            if (feature.get(fieldname) == 'missing-data'):
                                continue
                            feature.setdefault(fieldname, {
                                'value': 0,
                                'type': summode,
                            })
                            feature[fieldname]['value'] += (census_data[census_id][fieldname] * ix[division2]['ratio'])
                        elif (summode == 'median'):
                            feature.setdefault(fieldname, {
                                'value': 0.0,
                                'type': summode,
                                'weight': 0.0,
                                'missing_weight': 0.0,
                            })
                            w = (census_data[census_id]['B01003_001E'] * ix[division2]['ratio'])
                            if (census_data[census_id][fieldname] is None):
                                feature[fieldname]['missing_weight'] += w
                                continue
                            feature[fieldname]['value'] += (census_data[census_id][fieldname] * w)
                            feature[fieldname]['weight'] += w

    def clean_up(f):
        for division in ('tract', 'blockgroup'):
            for (k, v) in list(f[('by-' + division)].items()):
                if (v == 'missing-data'):
                    continue
                if (v['type'] == 'count'):
                    f[k] = v
                    f[k]['source'] = division
                    v['value'] = int(round(v['value']))
                elif ((v['type'] == 'median') and (v['weight'] > 0)):
                    v['missing_pct'] = (v['missing_weight'] / (v['weight'] + v['missing_weight']))
                    if ((v['missing_pct'] > 0.33) or ((k in f) and (f[k]['missing_pct'] < v['missing_pct']))):
                        continue
                    f[k] = v
                    f[k]['source'] = division
                    v['value'] = round((v['value'] / v['weight']))
                    del v['weight']
                    del v['missing_weight']
                del v['type']
            del f[('by-' + division)]
    for ward in output.values():
        for anc in ward['ancs'].values():
            clean_up(anc['census'])
            for smd in anc['smds'].values():
                clean_up(smd['census'])

dataset/ETHPy150Open codefordc/ancfinder/scripts/update_anc_database.py add_census_data/original