<s> from __future__ import unicode_literals <EOL> import itertools <EOL> import json <EOL> import requests <EOL> import scorched . dates <EOL> import scorched . response <EOL> import scorched . search <EOL> import scorched . exc <EOL> import scorched . compat <EOL> import time <EOL> import warnings <EOL> from scorched . compat import str <EOL> MAX_LENGTH_GET_URL = <NUM_LIT> <EOL> def is_iter ( val ) : <EOL> return isinstance ( val , ( tuple , list ) ) <EOL> class SolrConnection ( object ) : <EOL> readable = True <EOL> writeable = True <EOL> def __init__ ( self , url , http_connection , mode , retry_timeout , <EOL> max_length_get_url , search_timeout = ( ) ) : <EOL> """<STR_LIT>""" <EOL> self . http_connection = requests . Session ( ) <EOL> if mode == '<STR_LIT:r>' : <EOL> self . writeable = False <EOL> elif mode == '<STR_LIT:w>' : <EOL> self . readable = False <EOL> self . url = url . rstrip ( "<STR_LIT:/>" ) + "<STR_LIT:/>" <EOL> self . update_url = self . url + "<STR_LIT>" <EOL> self . select_url = self . url + "<STR_LIT>" <EOL> self . mlt_url = self . url + "<STR_LIT>" <EOL> self . get_url = self . url + "<STR_LIT>" <EOL> self . retry_timeout = retry_timeout <EOL> self . max_length_get_url = max_length_get_url <EOL> self . search_timeout = search_timeout <EOL> def request ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return self . http_connection . request ( * args , ** kwargs ) <EOL> except ( requests . exceptions . ConnectionError , <EOL> requests . exceptions . Timeout ) : <EOL> if self . retry_timeout < <NUM_LIT:0> : <EOL> raise <EOL> time . sleep ( self . retry_timeout ) <EOL> return self . http_connection . request ( * args , ** kwargs ) <EOL> def get ( self , ids , fl = None ) : <EOL> """<STR_LIT>""" <EOL> params = [ <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ] <EOL> if is_iter ( ids ) : <EOL> for id in ids : <EOL> params . append ( ( "<STR_LIT:id>" , id ) ) <EOL> else : <EOL> params . append ( ( "<STR_LIT:id>" , ids ) ) <EOL> if fl : <EOL> params . append ( ( "<STR_LIT>" , "<STR_LIT:U+002C>" . join ( fl ) ) ) <EOL> qs = scorched . compat . urlencode ( params ) <EOL> url = "<STR_LIT>" % ( self . get_url , qs ) <EOL> response = self . request ( "<STR_LIT:GET>" , url ) <EOL> if response . status_code != <NUM_LIT:200> : <EOL> raise scorched . exc . SolrError ( response ) <EOL> return response . text <EOL> def update ( self , update_doc , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if not self . writeable : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> body = update_doc <EOL> if body : <EOL> headers = { "<STR_LIT:Content-Type>" : "<STR_LIT>" } <EOL> else : <EOL> headers = { } <EOL> url = self . url_for_update ( ** kwargs ) <EOL> response = self . request ( '<STR_LIT:POST>' , url , data = body , headers = headers ) <EOL> if response . status_code != <NUM_LIT:200> : <EOL> raise scorched . exc . SolrError ( response ) <EOL> return response . text <EOL> def url_for_update ( self , commit = None , commitWithin = None , softCommit = None , <EOL> optimize = None , waitSearcher = None , expungeDeletes = None , <EOL> maxSegments = None ) : <EOL> """<STR_LIT>""" <EOL> extra_params = { } <EOL> if commit is not None : <EOL> extra_params [ '<STR_LIT>' ] = "<STR_LIT:true>" if commit else "<STR_LIT:false>" <EOL> if commitWithin is not None : <EOL> try : <EOL> extra_params [ '<STR_LIT>' ] = int ( commitWithin ) <EOL> except ( TypeError , ValueError ) : <EOL> raise ValueError ( <EOL> "<STR_LIT>" ) <EOL> if extra_params [ '<STR_LIT>' ] < <NUM_LIT:0> : <EOL> raise ValueError ( <EOL> "<STR_LIT>" ) <EOL> extra_params [ '<STR_LIT>' ] = str ( extra_params [ '<STR_LIT>' ] ) <EOL> if softCommit is not None : <EOL> extra_params [ '<STR_LIT>' ] = "<STR_LIT:true>" if softCommit else "<STR_LIT:false>" <EOL> if optimize is not None : <EOL> extra_params [ '<STR_LIT>' ] = "<STR_LIT:true>" if optimize else "<STR_LIT:false>" <EOL> if waitSearcher is not None : <EOL> extra_params [ '<STR_LIT>' ] = "<STR_LIT:true>" if waitSearcher else "<STR_LIT:false>" <EOL> if expungeDeletes is not None : <EOL> extra_params [ <EOL> '<STR_LIT>' ] = "<STR_LIT:true>" if expungeDeletes else "<STR_LIT:false>" <EOL> if maxSegments is not None : <EOL> try : <EOL> extra_params [ '<STR_LIT>' ] = int ( maxSegments ) <EOL> except ( TypeError , ValueError ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if extra_params [ '<STR_LIT>' ] <= <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> extra_params [ '<STR_LIT>' ] = str ( extra_params [ '<STR_LIT>' ] ) <EOL> if '<STR_LIT>' in extra_params and '<STR_LIT>' not in extra_params : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if '<STR_LIT>' in extra_params and '<STR_LIT>' not in extra_params : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if extra_params : <EOL> return "<STR_LIT>" % ( self . update_url , scorched . compat . urlencode ( <EOL> sorted ( extra_params . items ( ) ) ) ) <EOL> else : <EOL> return self . update_url <EOL> def select ( self , params ) : <EOL> """<STR_LIT>""" <EOL> if not self . readable : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> params . append ( ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> qs = scorched . compat . urlencode ( params ) <EOL> url = "<STR_LIT>" % ( self . select_url , qs ) <EOL> if len ( url ) > self . max_length_get_url : <EOL> warnings . warn ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> url = self . select_url <EOL> method = '<STR_LIT:POST>' <EOL> kwargs = { <EOL> '<STR_LIT:data>' : qs , <EOL> '<STR_LIT>' : { <EOL> "<STR_LIT:Content-Type>" : "<STR_LIT>" } } <EOL> else : <EOL> method = '<STR_LIT:GET>' <EOL> kwargs = { } <EOL> if self . search_timeout != ( ) : <EOL> kwargs [ '<STR_LIT>' ] = self . search_timeout <EOL> response = self . request ( method , url , ** kwargs ) <EOL> if response . status_code != <NUM_LIT:200> : <EOL> raise scorched . exc . SolrError ( response ) <EOL> return response . text <EOL> def mlt ( self , params , content = None ) : <EOL> """<STR_LIT>""" <EOL> if not self . readable : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> params . append ( ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> qs = scorched . compat . urlencode ( params ) <EOL> base_url = "<STR_LIT>" % ( self . mlt_url , qs ) <EOL> method = '<STR_LIT:GET>' <EOL> kwargs = { } <EOL> if content is None : <EOL> url = base_url <EOL> else : <EOL> get_url = "<STR_LIT>" % ( <EOL> base_url , scorched . compat . quote_plus ( content ) ) <EOL> if len ( get_url ) <= self . max_length_get_url : <EOL> url = get_url <EOL> else : <EOL> url = base_url <EOL> method = '<STR_LIT:POST>' <EOL> kwargs = { <EOL> '<STR_LIT:data>' : content , <EOL> '<STR_LIT>' : { "<STR_LIT:Content-Type>" : "<STR_LIT>" } } <EOL> response = self . request ( method , url , ** kwargs ) <EOL> if response . status_code != <NUM_LIT:200> : <EOL> raise scorched . exc . SolrError ( response . content ) <EOL> return response . text <EOL> class SolrInterface ( object ) : <EOL> remote_schema_file = "<STR_LIT>" <EOL> def __init__ ( self , url , http_connection = None , mode = '<STR_LIT>' , <EOL> retry_timeout = - <NUM_LIT:1> , max_length_get_url = MAX_LENGTH_GET_URL , <EOL> search_timeout = ( ) ) : <EOL> """<STR_LIT>""" <EOL> self . conn = SolrConnection ( <EOL> url , http_connection , mode , retry_timeout , max_length_get_url ) <EOL> self . schema = self . init_schema ( ) <EOL> self . _datefields = self . _extract_datefields ( self . schema ) <EOL> def init_schema ( self ) : <EOL> response = self . conn . request ( <EOL> '<STR_LIT:GET>' , scorched . compat . urljoin ( self . conn . url , <EOL> self . remote_schema_file ) ) <EOL> if response . status_code != <NUM_LIT:200> : <EOL> raise EnvironmentError ( <EOL> "<STR_LIT>" % ( <EOL> response . status_code , response . content ) <EOL> ) <EOL> return response . json ( ) [ '<STR_LIT>' ] <EOL> def _extract_datefields ( self , schema ) : <EOL> ret = [ x [ '<STR_LIT:name>' ] for x in <EOL> schema [ '<STR_LIT>' ] if x [ '<STR_LIT:type>' ] == '<STR_LIT:date>' ] <EOL> ret . extend ( [ x [ '<STR_LIT:name>' ] for x in schema [ '<STR_LIT>' ] <EOL> if x [ '<STR_LIT:type>' ] == '<STR_LIT:date>' ] ) <EOL> return ret <EOL> def _prepare_docs ( self , docs ) : <EOL> prepared_docs = [ ] <EOL> for doc in docs : <EOL> new_doc = { } <EOL> for name , value in list ( doc . items ( ) ) : <EOL> if value is None : <EOL> continue <EOL> if scorched . dates . is_datetime_field ( name , self . _datefields ) : <EOL> value = str ( scorched . dates . solr_date ( value ) ) <EOL> new_doc [ name ] = value <EOL> prepared_docs . append ( new_doc ) <EOL> return prepared_docs <EOL> def add ( self , docs , chunk = <NUM_LIT:100> , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if hasattr ( docs , "<STR_LIT>" ) or not is_iter ( docs ) : <EOL> docs = [ docs ] <EOL> ret = [ ] <EOL> for doc_chunk in grouper ( docs , chunk ) : <EOL> update_message = json . dumps ( self . _prepare_docs ( doc_chunk ) ) <EOL> ret . append ( scorched . response . SolrUpdateResponse . from_json ( <EOL> self . conn . update ( update_message , ** kwargs ) ) ) <EOL> return ret <EOL> def delete_by_query ( self , query , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> delete_message = json . dumps ( { "<STR_LIT>" : { "<STR_LIT>" : str ( query ) } } ) <EOL> ret = scorched . response . SolrUpdateResponse . from_json ( <EOL> self . conn . update ( delete_message , ** kwargs ) ) <EOL> return ret <EOL> def delete_by_ids ( self , ids , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> delete_message = json . dumps ( { "<STR_LIT>" : ids } ) <EOL> ret = scorched . response . SolrUpdateResponse . from_json ( <EOL> self . conn . update ( delete_message , ** kwargs ) ) <EOL> return ret <EOL> def commit ( self , waitSearcher = None , expungeDeletes = None , softCommit = None ) : <EOL> """<STR_LIT>""" <EOL> ret = scorched . response . SolrUpdateResponse . from_json ( <EOL> self . conn . update ( '<STR_LIT>' , commit = True , <EOL> waitSearcher = waitSearcher , <EOL> expungeDeletes = expungeDeletes , <EOL> softCommit = softCommit ) ) <EOL> return ret <EOL> def optimize ( self , waitSearcher = None , maxSegments = None ) : <EOL> """<STR_LIT>""" <EOL> ret = scorched . response . SolrUpdateResponse . from_json ( <EOL> self . conn . update ( '<STR_LIT>' , optimize = True , <EOL> waitSearcher = waitSearcher , <EOL> maxSegments = maxSegments ) ) <EOL> return ret <EOL> def rollback ( self ) : <EOL> """<STR_LIT>""" <EOL> ret = scorched . response . SolrUpdateResponse . from_json ( <EOL> self . conn . update ( '<STR_LIT>' ) ) <EOL> return ret <EOL> def delete_all ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . delete_by_query ( self . Q ( ** { "<STR_LIT:*>" : "<STR_LIT:*>" } ) ) <EOL> def get ( self , ids , fields = None ) : <EOL> """<STR_LIT>""" <EOL> ret = scorched . response . SolrResponse . from_get_json ( <EOL> self . conn . get ( ids , fields ) , self . _datefields ) <EOL> return ret <EOL> def search ( self , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> params = scorched . search . params_from_dict ( ** kwargs ) <EOL> ret = scorched . response . SolrResponse . from_json ( <EOL> self . conn . select ( params ) , self . _datefields ) <EOL> return ret <EOL> def query ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> q = scorched . search . SolrSearch ( self ) <EOL> if len ( args ) + len ( kwargs ) > <NUM_LIT:0> : <EOL> return q . query ( * args , ** kwargs ) <EOL> else : <EOL> return q <EOL> def mlt_search ( self , content = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> params = scorched . search . params_from_dict ( ** kwargs ) <EOL> ret = scorched . response . SolrResponse . from_json ( <EOL> self . conn . mlt ( params , content = content ) , self . _datefields ) <EOL> return ret <EOL> def mlt_query ( self , fields , content = None , content_charset = None , <EOL> url = None , query_fields = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> q = scorched . search . MltSolrSearch ( <EOL> self , content = content , content_charset = content_charset , url = url ) <EOL> return q . mlt ( fields = fields , query_fields = query_fields , ** kwargs ) <EOL> def Q ( self , * args , ** kwargs ) : <EOL> q = scorched . search . LuceneQuery ( ) <EOL> q . add ( args , kwargs ) <EOL> return q <EOL> def grouper ( iterable , n ) : <EOL> """<STR_LIT>""" <EOL> i = iter ( iterable ) <EOL> g = list ( itertools . islice ( i , <NUM_LIT:0> , n ) ) <EOL> while g : <EOL> yield g <EOL> g = list ( itertools . islice ( i , <NUM_LIT:0> , n ) ) </s>
