<s> import re <EOL> from django . conf import settings <EOL> from django . core . urlresolvers import get_script_prefix , resolve , reverse <EOL> from django . utils import simplejson as json <EOL> from django . utils . translation import ugettext_lazy as _ <EOL> from tastypie import fields , http <EOL> from tastypie . authorization import DjangoAuthorization <EOL> from tastypie . bundle import Bundle <EOL> from tastypie . exceptions import BadRequest , NotFound , ImmediateHttpResponse <EOL> from tastypie . utils import dict_strip_unicode_keys <EOL> from tastypie . utils . mime import build_content_type <EOL> from tastypie . validation import Validation <EOL> from panda import solr <EOL> from panda . api . datasets import DatasetResource <EOL> from panda . exceptions import DatasetLockedError <EOL> from panda . api . utils import PandaAuthentication , PandaPaginator , PandaResource , PandaSerializer <EOL> from panda . models import Category , Dataset , SearchLog , TaskStatus , UserProxy <EOL> from panda . tasks import ExportSearchTask , PurgeDataTask <EOL> class SolrObject ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , initial = None , ** kwargs ) : <EOL> self . __dict__ [ '<STR_LIT>' ] = { } <EOL> if hasattr ( initial , '<STR_LIT>' ) : <EOL> self . __dict__ [ '<STR_LIT>' ] = initial <EOL> self . __dict__ [ '<STR_LIT>' ] . update ( kwargs ) <EOL> def __getattr__ ( self , name ) : <EOL> return self . _data . get ( name , None ) <EOL> def __setattr__ ( self , name , value ) : <EOL> self . __dict__ [ '<STR_LIT>' ] [ name ] = value <EOL> def __str__ ( self ) : <EOL> return str ( self . __dict__ [ '<STR_LIT>' ] ) <EOL> def __unicode__ ( self ) : <EOL> return unicode ( self . __dict__ [ '<STR_LIT>' ] ) <EOL> def to_dict ( self ) : <EOL> return self . _data <EOL> class DataValidation ( Validation ) : <EOL> """<STR_LIT>""" <EOL> def is_valid ( self , bundle , request = None ) : <EOL> errors = { } <EOL> if '<STR_LIT:data>' not in bundle . data or not bundle . data [ '<STR_LIT:data>' ] : <EOL> errors [ '<STR_LIT:data>' ] = [ _ ( '<STR_LIT>' ) ] <EOL> if '<STR_LIT>' in bundle . data : <EOL> if not isinstance ( bundle . data [ '<STR_LIT>' ] , basestring ) : <EOL> errors [ '<STR_LIT>' ] = [ _ ( '<STR_LIT>' ) ] <EOL> elif not re . match ( '<STR_LIT>' , bundle . data [ '<STR_LIT>' ] ) : <EOL> errors [ '<STR_LIT>' ] = [ _ ( '<STR_LIT>' ) ] <EOL> return errors <EOL> class DataResource ( PandaResource ) : <EOL> """<STR_LIT>""" <EOL> dataset_slug = fields . CharField ( attribute = '<STR_LIT>' , <EOL> help_text = _ ( '<STR_LIT>' ) ) <EOL> external_id = fields . CharField ( attribute = '<STR_LIT>' , null = True , blank = True , <EOL> help_text = _ ( '<STR_LIT>' ) ) <EOL> data = fields . CharField ( attribute = '<STR_LIT:data>' , <EOL> help_text = _ ( '<STR_LIT>' ) ) <EOL> class Meta : <EOL> resource_name = '<STR_LIT:data>' <EOL> allowed_methods = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> always_return_data = True <EOL> authentication = PandaAuthentication ( ) <EOL> authorization = DjangoAuthorization ( ) <EOL> serializer = PandaSerializer ( ) <EOL> validation = DataValidation ( ) <EOL> object_class = SolrObject <EOL> def dehydrate_data ( self , bundle ) : <EOL> """<STR_LIT>""" <EOL> return json . loads ( bundle . data [ '<STR_LIT:data>' ] ) <EOL> def dehydrate ( self , bundle ) : <EOL> """<STR_LIT>""" <EOL> dataset = Dataset . objects . get ( slug = bundle . data [ '<STR_LIT>' ] ) <EOL> del bundle . data [ '<STR_LIT>' ] <EOL> bundle . data [ '<STR_LIT>' ] = DatasetResource ( ) . get_resource_uri ( dataset ) <EOL> return bundle <EOL> def get_resource_uri ( self , bundle_or_obj ) : <EOL> """<STR_LIT>""" <EOL> dr = DatasetResource ( ) <EOL> kwargs = { <EOL> '<STR_LIT>' : self . _meta . api_name , <EOL> '<STR_LIT>' : dr . _meta . resource_name , <EOL> '<STR_LIT>' : self . _meta . resource_name , <EOL> } <EOL> if isinstance ( bundle_or_obj , Bundle ) : <EOL> kwargs [ '<STR_LIT>' ] = bundle_or_obj . obj . dataset_slug <EOL> kwargs [ '<STR_LIT>' ] = bundle_or_obj . obj . external_id <EOL> else : <EOL> kwargs [ '<STR_LIT>' ] = bundle_or_obj . dataset_slug <EOL> kwargs [ '<STR_LIT>' ] = bundle_or_obj . external_id <EOL> if not kwargs [ '<STR_LIT>' ] : <EOL> return None <EOL> return dr . _build_reverse_url ( '<STR_LIT>' , kwargs = kwargs ) <EOL> def get_dataset_from_kwargs ( self , bundle , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> kwargs_slug = kwargs [ '<STR_LIT>' ] <EOL> bundle_uri = None <EOL> bundle_slug = None <EOL> if bundle : <EOL> bundle_uri = bundle . data . pop ( '<STR_LIT>' , None ) <EOL> if bundle_uri : <EOL> prefix = get_script_prefix ( ) <EOL> if prefix and bundle_uri . startswith ( prefix ) : <EOL> bundle_uri = bundle_uri [ len ( prefix ) - <NUM_LIT:1> : ] <EOL> view , args , kwargs = resolve ( bundle_uri ) <EOL> bundle_slug = kwargs [ '<STR_LIT>' ] <EOL> if bundle_slug and bundle_slug != kwargs_slug : <EOL> raise BadRequest ( _ ( '<STR_LIT>' ) ) <EOL> return Dataset . objects . get ( slug = kwargs_slug ) <EOL> def validate_bundle_data ( self , bundle , request , dataset ) : <EOL> """<STR_LIT>""" <EOL> errors = { } <EOL> field_count = len ( bundle . data [ '<STR_LIT:data>' ] ) <EOL> if dataset . initial_upload and not dataset . row_count : <EOL> errors [ '<STR_LIT>' ] = [ _ ( '<STR_LIT>' ) ] <EOL> if dataset . column_schema is None : <EOL> errors [ '<STR_LIT>' ] = [ _ ( '<STR_LIT>' ) ] <EOL> else : <EOL> expected_field_count = len ( dataset . column_schema ) <EOL> if field_count != expected_field_count : <EOL> errors [ '<STR_LIT:data>' ] = [ _ ( '<STR_LIT>' ) % { '<STR_LIT>' : field_count , '<STR_LIT>' : expected_field_count } ] <EOL> if errors : <EOL> if request : <EOL> desired_format = self . determine_format ( request ) <EOL> else : <EOL> desired_format = self . _meta . default_format <EOL> serialized = self . serialize ( request , errors , desired_format ) <EOL> response = http . HttpBadRequest ( content = serialized , content_type = build_content_type ( desired_format ) ) <EOL> raise ImmediateHttpResponse ( response = response ) <EOL> def get_object_list ( ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> def obj_get_list ( self , request = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> def obj_get ( self , request = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> dataset = Dataset . objects . get ( slug = kwargs [ '<STR_LIT>' ] ) <EOL> row = dataset . get_row ( kwargs [ '<STR_LIT>' ] ) <EOL> if not row : <EOL> raise NotFound ( ) <EOL> return SolrObject ( row ) <EOL> def obj_create ( self , bundle , request = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> dataset = self . get_dataset_from_kwargs ( bundle , ** kwargs ) <EOL> self . validate_bundle_data ( bundle , request , dataset ) <EOL> if '<STR_LIT>' in bundle . data : <EOL> external_id = bundle . data [ '<STR_LIT>' ] <EOL> elif '<STR_LIT>' in kwargs : <EOL> external_id = kwargs [ '<STR_LIT>' ] <EOL> else : <EOL> external_id = None <EOL> user = UserProxy . objects . get ( id = request . user . id ) <EOL> try : <EOL> row = dataset . add_row ( user , bundle . data [ '<STR_LIT:data>' ] , external_id = external_id ) <EOL> except DatasetLockedError : <EOL> raise ImmediateHttpResponse ( response = http . HttpForbidden ( _ ( '<STR_LIT>' ) ) ) <EOL> bundle . obj = SolrObject ( row ) <EOL> return bundle <EOL> def obj_update ( self , bundle , request = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return self . obj_create ( bundle , request , ** kwargs ) <EOL> def obj_delete_list ( self , request = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> def obj_delete ( self , request = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> dataset = Dataset . objects . get ( slug = kwargs [ '<STR_LIT>' ] ) <EOL> user = UserProxy . objects . get ( id = request . user . id ) <EOL> try : <EOL> dataset . delete_row ( user , kwargs [ '<STR_LIT>' ] ) <EOL> except DatasetLockedError : <EOL> raise ImmediateHttpResponse ( response = http . HttpForbidden ( _ ( '<STR_LIT>' ) ) ) <EOL> def rollback ( self , bundles ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> def get_list ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> results = self . search_dataset_data ( request , ** kwargs ) <EOL> return self . create_response ( request , results ) <EOL> def get_detail ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return super ( DataResource , self ) . get_detail ( request , ** kwargs ) <EOL> def put_list ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> deserialized = self . deserialize ( request , request . raw_post_data , format = request . META . get ( '<STR_LIT>' , '<STR_LIT:application/json>' ) ) <EOL> deserialized = self . alter_deserialized_list_data ( request , deserialized ) <EOL> if not '<STR_LIT>' in deserialized : <EOL> raise BadRequest ( _ ( "<STR_LIT>" ) ) <EOL> bundles = [ ] <EOL> data = [ ] <EOL> dataset = self . get_dataset_from_kwargs ( None , ** kwargs ) <EOL> for object_data in deserialized [ '<STR_LIT>' ] : <EOL> bundle = self . build_bundle ( data = dict_strip_unicode_keys ( object_data ) , request = request ) <EOL> self . is_valid ( bundle , request ) <EOL> if bundle . errors : <EOL> self . error_response ( bundle . errors , request ) <EOL> bundles . append ( bundle ) <EOL> data . append ( ( <EOL> bundle . data [ '<STR_LIT:data>' ] , <EOL> bundle . data . get ( '<STR_LIT>' , None ) <EOL> ) ) <EOL> self . validate_bundle_data ( bundle , request , dataset ) <EOL> user = UserProxy . objects . get ( id = request . user . id ) <EOL> try : <EOL> solr_rows = dataset . add_many_rows ( user , data ) <EOL> except DatasetLockedError : <EOL> raise ImmediateHttpResponse ( response = http . HttpForbidden ( _ ( '<STR_LIT>' ) ) ) <EOL> for bundle , solr_row in zip ( bundles , solr_rows ) : <EOL> bundle . obj = SolrObject ( solr_row ) <EOL> if not self . _meta . always_return_data : <EOL> return http . HttpNoContent ( ) <EOL> else : <EOL> to_be_serialized = { } <EOL> to_be_serialized [ '<STR_LIT>' ] = [ self . full_dehydrate ( bundle ) for bundle in bundles ] <EOL> to_be_serialized = self . alter_list_data_to_serialize ( request , to_be_serialized ) <EOL> return self . create_response ( request , to_be_serialized , response_class = http . HttpAccepted ) <EOL> def put_detail ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return super ( DataResource , self ) . put_detail ( request , ** kwargs ) <EOL> def post_list ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return super ( DataResource , self ) . post_list ( request , ** kwargs ) <EOL> def post_detail ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return super ( DataResource , self ) . post_detail ( request , ** kwargs ) <EOL> def delete_list ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> dataset = Dataset . objects . get ( slug = kwargs [ '<STR_LIT>' ] ) <EOL> user = UserProxy . objects . get ( id = request . user . id ) <EOL> try : <EOL> dataset . delete_all_rows ( user ) <EOL> except DatasetLockedError : <EOL> raise ImmediateHttpResponse ( response = http . HttpForbidden ( _ ( '<STR_LIT>' ) ) ) <EOL> return http . HttpNoContent ( ) <EOL> def delete_detail ( self , request , ** kwargs ) : <EOL> return super ( DataResource , self ) . delete_detail ( request , ** kwargs ) <EOL> def search_all_data ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . method_check ( request , allowed = [ '<STR_LIT>' ] ) <EOL> self . is_authenticated ( request ) <EOL> self . throttle_check ( request ) <EOL> try : <EOL> query = '<STR_LIT>' % request . GET [ '<STR_LIT:q>' ] <EOL> except KeyError : <EOL> query = '<STR_LIT>' <EOL> category = request . GET . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> since = request . GET . get ( '<STR_LIT>' , None ) <EOL> limit = int ( request . GET . get ( '<STR_LIT>' , settings . PANDA_DEFAULT_SEARCH_GROUPS ) ) <EOL> offset = int ( request . GET . get ( '<STR_LIT>' , <NUM_LIT:0> ) ) <EOL> group_limit = int ( request . GET . get ( '<STR_LIT>' , settings . PANDA_DEFAULT_SEARCH_ROWS_PER_GROUP ) ) <EOL> group_offset = int ( request . GET . get ( '<STR_LIT>' , <NUM_LIT:0> ) ) <EOL> export = bool ( request . GET . get ( '<STR_LIT>' , False ) ) <EOL> solr_query_bits = [ query ] <EOL> if category : <EOL> if category != '<STR_LIT>' : <EOL> category = Category . objects . get ( slug = category ) <EOL> dataset_slugs = category . datasets . values_list ( '<STR_LIT>' , flat = True ) <EOL> else : <EOL> dataset_slugs = Dataset . objects . filter ( categories = None ) . values_list ( '<STR_LIT>' , flat = True ) <EOL> solr_query_bits . append ( '<STR_LIT>' % '<STR_LIT:U+0020>' . join ( dataset_slugs ) ) <EOL> if since : <EOL> solr_query_bits . append ( '<STR_LIT>' + since + '<STR_LIT>' ) <EOL> user = UserProxy . objects . get ( id = request . user . id ) <EOL> if export : <EOL> task_type = ExportSearchTask <EOL> task = TaskStatus . objects . create ( <EOL> task_name = task_type . name , <EOL> task_description = _ ( '<STR_LIT>' ) % query , <EOL> creator = user <EOL> ) <EOL> task_type . apply_async ( <EOL> args = [ query , task . id ] , <EOL> kwargs = { } , <EOL> task_id = task . id <EOL> ) <EOL> else : <EOL> response = solr . query_grouped ( <EOL> settings . SOLR_DATA_CORE , <EOL> '<STR_LIT>' . join ( solr_query_bits ) , <EOL> '<STR_LIT>' , <EOL> offset = offset , <EOL> limit = limit , <EOL> group_limit = group_limit , <EOL> group_offset = group_offset <EOL> ) <EOL> groups = response [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> page = PandaPaginator ( <EOL> request . GET , <EOL> groups , <EOL> resource_uri = request . path_info , <EOL> count = response [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> ) . page ( ) <EOL> datasets = [ ] <EOL> for group in groups : <EOL> dataset_slug = group [ '<STR_LIT>' ] <EOL> results = group [ '<STR_LIT>' ] <EOL> try : <EOL> dataset = Dataset . objects . get ( slug = dataset_slug ) <EOL> except Dataset . DoesNotExist : <EOL> PurgeDataTask . apply_async ( args = [ dataset_slug ] ) <EOL> solr . delete ( settings . SOLR_DATASETS_CORE , '<STR_LIT>' % dataset_slug ) <EOL> page [ '<STR_LIT>' ] [ '<STR_LIT>' ] -= <NUM_LIT:1> <EOL> continue <EOL> dataset_resource = DatasetResource ( ) <EOL> dataset_bundle = dataset_resource . build_bundle ( obj = dataset , request = request ) <EOL> dataset_bundle = dataset_resource . full_dehydrate ( dataset_bundle ) <EOL> dataset_bundle = dataset_resource . simplify_bundle ( dataset_bundle ) <EOL> objects = [ SolrObject ( obj ) for obj in results [ '<STR_LIT>' ] ] <EOL> dataset_search_url = reverse ( '<STR_LIT>' , kwargs = { '<STR_LIT>' : self . _meta . api_name , '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT:data>' , '<STR_LIT>' : dataset . slug } ) <EOL> data_page = PandaPaginator ( <EOL> { '<STR_LIT>' : str ( group_limit ) , '<STR_LIT>' : str ( group_offset ) , '<STR_LIT:q>' : query } , <EOL> objects , <EOL> resource_uri = dataset_search_url , <EOL> count = results [ '<STR_LIT>' ] <EOL> ) . page ( ) <EOL> dataset_bundle . data . update ( data_page ) <EOL> dataset_bundle . data [ '<STR_LIT>' ] = [ ] <EOL> for obj in objects : <EOL> data_bundle = self . build_bundle ( obj = obj , request = request ) <EOL> data_bundle = self . full_dehydrate ( data_bundle ) <EOL> dataset_bundle . data [ '<STR_LIT>' ] . append ( data_bundle ) <EOL> datasets . append ( dataset_bundle . data ) <EOL> page [ '<STR_LIT>' ] = datasets <EOL> SearchLog . objects . create ( user = user , dataset = None , query = query ) <EOL> self . log_throttled_access ( request ) <EOL> if export : <EOL> return self . create_response ( request , _ ( '<STR_LIT>' ) ) <EOL> else : <EOL> return self . create_response ( request , page ) <EOL> def search_dataset_data ( self , request , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> dataset = Dataset . objects . get ( slug = kwargs [ '<STR_LIT>' ] ) <EOL> try : <EOL> query = '<STR_LIT>' % request . GET [ '<STR_LIT:q>' ] <EOL> except KeyError : <EOL> query = '<STR_LIT>' <EOL> since = request . GET . get ( '<STR_LIT>' , None ) <EOL> limit = int ( request . GET . get ( '<STR_LIT>' , settings . PANDA_DEFAULT_SEARCH_ROWS ) ) <EOL> offset = int ( request . GET . get ( '<STR_LIT>' , <NUM_LIT:0> ) ) <EOL> sort = request . GET . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> solr_query_bits = [ query ] <EOL> solr_query_bits . append ( '<STR_LIT>' % dataset . slug ) <EOL> if since : <EOL> solr_query_bits . append ( '<STR_LIT>' + since + '<STR_LIT>' ) <EOL> response = solr . query ( <EOL> settings . SOLR_DATA_CORE , <EOL> '<STR_LIT>' . join ( solr_query_bits ) , <EOL> offset = offset , <EOL> sort = sort , <EOL> limit = limit <EOL> ) <EOL> dataset_resource = DatasetResource ( ) <EOL> dataset_bundle = dataset_resource . build_bundle ( obj = dataset , request = request ) <EOL> dataset_bundle = dataset_resource . full_dehydrate ( dataset_bundle ) <EOL> dataset_bundle = dataset_resource . simplify_bundle ( dataset_bundle ) <EOL> results = [ SolrObject ( d ) for d in response [ '<STR_LIT>' ] [ '<STR_LIT>' ] ] <EOL> page = PandaPaginator ( <EOL> request . GET , <EOL> results , <EOL> resource_uri = request . path_info , <EOL> count = response [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> ) . page ( ) <EOL> dataset_bundle . data . update ( page ) <EOL> dataset_bundle . data [ '<STR_LIT>' ] = [ ] <EOL> for obj in results : <EOL> bundle = self . build_bundle ( obj = obj , request = request ) <EOL> bundle = self . full_dehydrate ( bundle ) <EOL> dataset_bundle . data [ '<STR_LIT>' ] . append ( bundle . data ) <EOL> user = UserProxy . objects . get ( id = request . user . id ) <EOL> SearchLog . objects . create ( user = user , dataset = dataset , query = query ) <EOL> return dataset_bundle </s>
