<gh_stars>10-100
"""
Scraper for City of Chicago crime reports.
Main page:
http://data.cityofchicago.org/Public-Safety/Crimes-2001-to-present/ijzp-q8t2
Metadata:
http://data.cityofchicago.org/views/ijzp-q8t2/columns.json
"""

import datetime
import re
import json
from scraper import Scraper

DATA_URL = 'http://data.cityofchicago.org/api/views/INLINE/rows.json?method=index'
INLINE_QUERY_TEMPLATE = """{
    "originalViewId": "ijzp-q8t2",
    "name": "Inline Filter",
    "query" : {
        "filterCondition" : {
            "type" : "operator",
            "value" : "GREATER_THAN",
            "children": [
                {
                    "columnFieldName" : "date",
                    "type" : "column"
                },
                {
                    "type" : "literal",
                    "value" : "%(start_date)s"
                }
            ]
        }
    }
}"""

class CrimeScraper(Scraper):
    schema = 'crime'
    primary_key = ('case_number',)
    #title_format = u'{secondary_type.name}'

    def data(self):
        # Build an inline query for crimes occurring within the past 180 days:
        start_date = datetime.date.today() - datetime.timedelta(days=10)
        inline_query = INLINE_QUERY_TEMPLATE % {
            'start_date': start_date.strftime('%Y-%m-%dT00:00:00'),
        }
        json_string = self.get(DATA_URL, inline_query, headers={'Content-Type': 'application/json'})
        for record in self.parse_socrata(json_string):
            yield self.convert_record(record)

    def convert_record(self, record):
        "Convert Socrata record into a record that we can insert as a newsitem."
        if record['longitude'] is None or record['latitude'] is None:
            location = None
        else:
            location = self.point(float(record['longitude']), float(record['latitude']))

        if record['x_coordinate'] is None or record['y_coordinate'] is None:
            xy = None
        else:
            xy = '%s;%s' % (record['x_coordinate'], record['y_coordinate'])

        number, street_name = record['block'].split(' ', 1)
        number = int(number.replace('X', '0'))
        location_name = '%s block %s' % (number, self.clean_address(street_name))

        dt = self.datetime(record['date'], '%Y-%m-%dT%H:%M:%S')

        return dict(
            title=record['description'],
            item_date=dt.date(),
            location=location,
            location_name=location_name,

            is_outdated=False,
            primary_type=record['primary_type'],
            secondary_type=record['iucr'],
            secondary_type__name=record['description'],
            case_number=record['case_number'],
            place=record['location_description'] or '',
            crime_time=dt.time(),
            beat=record['beat'],
            domestic=record['domestic'],
            xy=xy,
            real_address=location_name,
        )

if __name__ == "__main__":
    CrimeScraper().run()

"""
Scraper for City of Chicago crime reports.
Main page:
http://data.cityofchicago.org/Public-Safety/Crimes-2001-to-present/ijzp-q8t2
Metadata:
http://data.cityofchicago.org/views/ijzp-q8t2/columns.json
"""

import datetime
import re
import json
from scraper import Scraper

DATA_URL = 'http://data.cityofchicago.org/api/views/INLINE/rows.json?method=index'
INLINE_QUERY_TEMPLATE = """{
    "originalViewId": "ijzp-q8t2",
    "name": "Inline Filter",
    "query" : {
        "filterCondition" : {
            "type" : "operator",
            "value" : "GREATER_THAN",
            "children": [
                {
                    "columnFieldName" : "date",
                    "type" : "column"
                },
                {
                    "type" : "literal",
                    "value" : "%(start_date)s"
                }
            ]
        }
    }
}"""

class CrimeScraper(Scraper):
    schema = 'crime'
    primary_key = ('case_number',)
    #title_format = u'{secondary_type.name}'

    def data(self):
        # Build an inline query for crimes occurring within the past 180 days:
        start_date = datetime.date.today() - datetime.timedelta(days=10)
        inline_query = INLINE_QUERY_TEMPLATE % {
            'start_date': start_date.strftime('%Y-%m-%dT00:00:00'),
        }
        json_string = self.get(DATA_URL, inline_query, headers={'Content-Type': 'application/json'})
        for record in self.parse_socrata(json_string):
            yield self.convert_record(record)

    def convert_record(self, record):
        "Convert Socrata record into a record that we can insert as a newsitem."
        if record['longitude'] is None or record['latitude'] is None:
            location = None
        else:
            location = self.point(float(record['longitude']), float(record['latitude']))

        if record['x_coordinate'] is None or record['y_coordinate'] is None:
            xy = None
        else:
            xy = '%s;%s' % (record['x_coordinate'], record['y_coordinate'])

        number, street_name = record['block'].split(' ', 1)
        number = int(number.replace('X', '0'))
        location_name = '%s block %s' % (number, self.clean_address(street_name))

        dt = self.datetime(record['date'], '%Y-%m-%dT%H:%M:%S')

        return dict(
            title=record['description'],
            item_date=dt.date(),
            location=location,
            location_name=location_name,

            is_outdated=False,
            primary_type=record['primary_type'],
            secondary_type=record['iucr'],
            secondary_type__name=record['description'],
            case_number=record['case_number'],
            place=record['location_description'] or '',
            crime_time=dt.time(),
            beat=record['beat'],
            domestic=record['domestic'],
            xy=xy,
            real_address=location_name,
        )

if __name__ == "__main__":
    CrimeScraper().run()
Scraper for City of Chicago crime reports. Main page: http://data.cityofchicago.org/Public-Safety/Crimes-2001-to-present/ijzp-q8t2 Metadata: http://data.cityofchicago.org/views/ijzp-q8t2/columns.json {     "originalViewId": "ijzp-q8t2",     "name": "Inline Filter",     "query" : {         "filterCondition" : {             "type" : "operator",             "value" : "GREATER_THAN",             "children": [                 {                     "columnFieldName" : "date",                     "type" : "column"                 },                 {                     "type" : "literal",                     "value" : "%(start_date)s"                 }             ]         }     } } #title_format = u'{secondary_type.name}' # Build an inline query for crimes occurring within the past 180 days: