<s> from __future__ import division <EOL> import copy <EOL> import functools <EOL> import logging <EOL> import math <EOL> import re <EOL> import unicodedata <EOL> from elasticsearch import ( <EOL> ConnectionError , <EOL> Elasticsearch , <EOL> NotFoundError , <EOL> RequestError , <EOL> TransportError , <EOL> helpers , <EOL> ) <EOL> from modularodm import Q <EOL> import six <EOL> from framework import sentry <EOL> from framework . celery_tasks import app as celery_app <EOL> from framework . mongo . utils import paginated <EOL> from website import settings <EOL> from website . filters import gravatar <EOL> from website . models import User , Node <EOL> from website . project . licenses import serialize_node_license_record <EOL> from website . search import exceptions <EOL> from website . search . util import build_query <EOL> from website . util import sanitize <EOL> from website . views import validate_page_num <EOL> logger = logging . getLogger ( __name__ ) <EOL> ALIASES = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:user>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:file>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> NOT_ANALYZED_PROPERTY = { '<STR_LIT:type>' : '<STR_LIT:string>' , '<STR_LIT:index>' : '<STR_LIT>' } <EOL> ENGLISH_ANALYZER_PROPERTY = { '<STR_LIT:type>' : '<STR_LIT:string>' , '<STR_LIT>' : '<STR_LIT>' } <EOL> INDEX = settings . ELASTIC_INDEX <EOL> try : <EOL> es = Elasticsearch ( <EOL> settings . ELASTIC_URI , <EOL> request_timeout = settings . ELASTIC_TIMEOUT <EOL> ) <EOL> logging . getLogger ( '<STR_LIT>' ) . setLevel ( logging . WARN ) <EOL> logging . getLogger ( '<STR_LIT>' ) . setLevel ( logging . WARN ) <EOL> logging . getLogger ( '<STR_LIT>' ) . setLevel ( logging . WARN ) <EOL> logging . getLogger ( '<STR_LIT>' ) . setLevel ( logging . WARN ) <EOL> es . cluster . health ( wait_for_status = '<STR_LIT>' ) <EOL> except ConnectionError as e : <EOL> sentry . log_exception ( ) <EOL> sentry . log_message ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> es = None <EOL> def requires_search ( func ) : <EOL> def wrapped ( * args , ** kwargs ) : <EOL> if es is not None : <EOL> try : <EOL> return func ( * args , ** kwargs ) <EOL> except ConnectionError : <EOL> raise exceptions . SearchUnavailableError ( '<STR_LIT>' ) <EOL> except NotFoundError as e : <EOL> raise exceptions . IndexNotFoundError ( e . error ) <EOL> except RequestError as e : <EOL> if '<STR_LIT>' in e . error : <EOL> raise exceptions . MalformedQueryError ( e . error ) <EOL> raise exceptions . SearchException ( e . error ) <EOL> except TransportError as e : <EOL> raise exceptions . SearchException ( e . error ) <EOL> sentry . log_message ( '<STR_LIT>' ) <EOL> raise exceptions . SearchUnavailableError ( "<STR_LIT>" ) <EOL> return wrapped <EOL> @ requires_search <EOL> def get_aggregations ( query , doc_type ) : <EOL> query [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } <EOL> } <EOL> } <EOL> res = es . search ( index = INDEX , doc_type = doc_type , search_type = '<STR_LIT:count>' , body = query ) <EOL> ret = { <EOL> doc_type : { <EOL> item [ '<STR_LIT:key>' ] : item [ '<STR_LIT>' ] <EOL> for item in agg [ '<STR_LIT>' ] <EOL> } <EOL> for doc_type , agg in res [ '<STR_LIT>' ] . iteritems ( ) <EOL> } <EOL> ret [ '<STR_LIT>' ] = res [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> return ret <EOL> @ requires_search <EOL> def get_counts ( count_query , clean = True ) : <EOL> count_query [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> } <EOL> } <EOL> res = es . search ( index = INDEX , doc_type = None , search_type = '<STR_LIT:count>' , body = count_query ) <EOL> counts = { x [ '<STR_LIT:key>' ] : x [ '<STR_LIT>' ] for x in res [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] if x [ '<STR_LIT:key>' ] in ALIASES . keys ( ) } <EOL> counts [ '<STR_LIT>' ] = sum ( [ val for val in counts . values ( ) ] ) <EOL> return counts <EOL> @ requires_search <EOL> def get_tags ( query , index ) : <EOL> query [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : { '<STR_LIT>' : '<STR_LIT>' } <EOL> } <EOL> } <EOL> results = es . search ( index = index , doc_type = None , body = query ) <EOL> tags = results [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> return tags <EOL> @ requires_search <EOL> def search ( query , index = None , doc_type = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> index = index or INDEX <EOL> tag_query = copy . deepcopy ( query ) <EOL> aggs_query = copy . deepcopy ( query ) <EOL> count_query = copy . deepcopy ( query ) <EOL> for key in [ '<STR_LIT>' , '<STR_LIT:size>' , '<STR_LIT>' ] : <EOL> try : <EOL> del tag_query [ key ] <EOL> del aggs_query [ key ] <EOL> del count_query [ key ] <EOL> except KeyError : <EOL> pass <EOL> tags = get_tags ( tag_query , index ) <EOL> try : <EOL> del aggs_query [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> del count_query [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> except KeyError : <EOL> pass <EOL> aggregations = get_aggregations ( aggs_query , doc_type = doc_type ) <EOL> counts = get_counts ( count_query , index ) <EOL> raw_results = es . search ( index = index , doc_type = doc_type , body = query ) <EOL> results = [ hit [ '<STR_LIT>' ] for hit in raw_results [ '<STR_LIT>' ] [ '<STR_LIT>' ] ] <EOL> return_value = { <EOL> '<STR_LIT>' : format_results ( results ) , <EOL> '<STR_LIT>' : counts , <EOL> '<STR_LIT>' : aggregations , <EOL> '<STR_LIT>' : tags , <EOL> '<STR_LIT>' : ALIASES <EOL> } <EOL> return return_value <EOL> def format_results ( results ) : <EOL> ret = [ ] <EOL> for result in results : <EOL> if result . get ( '<STR_LIT>' ) == '<STR_LIT:user>' : <EOL> result [ '<STR_LIT:url>' ] = '<STR_LIT>' + result [ '<STR_LIT:id>' ] <EOL> elif result . get ( '<STR_LIT>' ) == '<STR_LIT:file>' : <EOL> parent_info = load_parent ( result . get ( '<STR_LIT>' ) ) <EOL> result [ '<STR_LIT>' ] = parent_info . get ( '<STR_LIT:url>' ) if parent_info else None <EOL> result [ '<STR_LIT>' ] = parent_info . get ( '<STR_LIT:title>' ) if parent_info else None <EOL> elif result . get ( '<STR_LIT>' ) in { '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' } : <EOL> result = format_result ( result , result . get ( '<STR_LIT>' ) ) <EOL> ret . append ( result ) <EOL> return ret <EOL> def format_result ( result , parent_id = None ) : <EOL> parent_info = load_parent ( parent_id ) <EOL> formatted_result = { <EOL> '<STR_LIT>' : result [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : result [ '<STR_LIT:url>' ] + '<STR_LIT>' , <EOL> '<STR_LIT:title>' : sanitize . unescape_entities ( result [ '<STR_LIT:title>' ] ) , <EOL> '<STR_LIT:url>' : result [ '<STR_LIT:url>' ] , <EOL> '<STR_LIT>' : False if parent_info is None else True , <EOL> '<STR_LIT>' : sanitize . unescape_entities ( parent_info . get ( '<STR_LIT:title>' ) ) if parent_info else None , <EOL> '<STR_LIT>' : parent_info . get ( '<STR_LIT:url>' ) if parent_info is not None else None , <EOL> '<STR_LIT>' : result [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : ( result [ '<STR_LIT>' ] if parent_info is None <EOL> else parent_info . get ( '<STR_LIT>' ) ) , <EOL> '<STR_LIT>' : result [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : result [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : result [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : result [ '<STR_LIT>' ] , <EOL> '<STR_LIT:description>' : result [ '<STR_LIT:description>' ] if parent_info is None else None , <EOL> '<STR_LIT>' : result . get ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' : result . get ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' : result . get ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' : len ( result [ '<STR_LIT>' ] ) , <EOL> '<STR_LIT>' : result . get ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' : result . get ( '<STR_LIT>' ) , <EOL> } <EOL> return formatted_result <EOL> def load_parent ( parent_id ) : <EOL> parent = Node . load ( parent_id ) <EOL> if parent is None : <EOL> return None <EOL> parent_info = { } <EOL> if parent is not None and parent . is_public : <EOL> parent_info [ '<STR_LIT:title>' ] = parent . title <EOL> parent_info [ '<STR_LIT:url>' ] = parent . url <EOL> parent_info [ '<STR_LIT>' ] = parent . is_registration <EOL> parent_info [ '<STR_LIT:id>' ] = parent . _id <EOL> else : <EOL> parent_info [ '<STR_LIT:title>' ] = '<STR_LIT>' <EOL> parent_info [ '<STR_LIT:url>' ] = '<STR_LIT>' <EOL> parent_info [ '<STR_LIT>' ] = None <EOL> parent_info [ '<STR_LIT:id>' ] = None <EOL> return parent_info <EOL> COMPONENT_CATEGORIES = set ( [ k for k in Node . CATEGORY_MAP . keys ( ) if not k == '<STR_LIT>' ] ) <EOL> def get_doctype_from_node ( node ) : <EOL> if node . is_registration : <EOL> return '<STR_LIT>' <EOL> elif node . category in COMPONENT_CATEGORIES : <EOL> return '<STR_LIT>' <EOL> else : <EOL> return node . category <EOL> @ celery_app . task ( bind = True , max_retries = <NUM_LIT:5> , default_retry_delay = <NUM_LIT> ) <EOL> def update_node_async ( self , node_id , index = None , bulk = False ) : <EOL> node = Node . load ( node_id ) <EOL> try : <EOL> update_node ( node = node , index = index , bulk = bulk ) <EOL> except Exception as exc : <EOL> self . retry ( exc = exc ) <EOL> @ requires_search <EOL> def update_node ( node , index = None , bulk = False ) : <EOL> index = index or INDEX <EOL> from website . addons . wiki . model import NodeWikiPage <EOL> category = get_doctype_from_node ( node ) <EOL> if category == '<STR_LIT>' : <EOL> elastic_document_id = node . _id <EOL> parent_id = None <EOL> else : <EOL> try : <EOL> elastic_document_id = node . _id <EOL> parent_id = node . parent_id <EOL> except IndexError : <EOL> return <EOL> from website . files . models . osfstorage import OsfStorageFile <EOL> for file_ in paginated ( OsfStorageFile , Q ( '<STR_LIT>' , '<STR_LIT>' , node ) ) : <EOL> update_file ( file_ , index = index ) <EOL> if node . is_deleted or not node . is_public or node . archiving : <EOL> delete_doc ( elastic_document_id , node ) <EOL> else : <EOL> try : <EOL> normalized_title = six . u ( node . title ) <EOL> except TypeError : <EOL> normalized_title = node . title <EOL> normalized_title = unicodedata . normalize ( '<STR_LIT>' , normalized_title ) . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) <EOL> elastic_document = { <EOL> '<STR_LIT:id>' : elastic_document_id , <EOL> '<STR_LIT>' : [ <EOL> { <EOL> '<STR_LIT>' : x . fullname , <EOL> '<STR_LIT:url>' : x . profile_url if x . is_active else None <EOL> } <EOL> for x in node . visible_contributors <EOL> if x is not None <EOL> ] , <EOL> '<STR_LIT:title>' : node . title , <EOL> '<STR_LIT>' : normalized_title , <EOL> '<STR_LIT>' : category , <EOL> '<STR_LIT>' : node . is_public , <EOL> '<STR_LIT>' : [ tag . _id for tag in node . tags if tag ] , <EOL> '<STR_LIT:description>' : node . description , <EOL> '<STR_LIT:url>' : node . url , <EOL> '<STR_LIT>' : node . is_registration , <EOL> '<STR_LIT>' : node . is_pending_registration , <EOL> '<STR_LIT>' : node . is_retracted , <EOL> '<STR_LIT>' : node . is_pending_retraction , <EOL> '<STR_LIT>' : node . embargo_end_date . strftime ( "<STR_LIT>" ) if node . embargo_end_date else False , <EOL> '<STR_LIT>' : node . is_pending_embargo , <EOL> '<STR_LIT>' : node . registered_date , <EOL> '<STR_LIT>' : { } , <EOL> '<STR_LIT>' : parent_id , <EOL> '<STR_LIT>' : node . date_created , <EOL> '<STR_LIT>' : serialize_node_license_record ( node . license ) , <EOL> '<STR_LIT>' : node . primary_institution . name if node . primary_institution else None , <EOL> '<STR_LIT>' : int ( not node . is_registration ) + <NUM_LIT:1> , <EOL> } <EOL> if not node . is_retracted : <EOL> for wiki in [ <EOL> NodeWikiPage . load ( x ) <EOL> for x in node . wiki_pages_current . values ( ) <EOL> ] : <EOL> elastic_document [ '<STR_LIT>' ] [ wiki . page_name ] = wiki . raw_text ( node ) <EOL> if bulk : <EOL> return elastic_document <EOL> else : <EOL> es . index ( index = index , doc_type = category , id = elastic_document_id , body = elastic_document , refresh = True ) <EOL> def bulk_update_nodes ( serialize , nodes , index = None ) : <EOL> """<STR_LIT>""" <EOL> index = index or INDEX <EOL> actions = [ ] <EOL> for node in nodes : <EOL> logger . info ( '<STR_LIT>' . format ( node . _id ) ) <EOL> serialized = serialize ( node ) <EOL> if serialized : <EOL> actions . append ( { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : index , <EOL> '<STR_LIT>' : node . _id , <EOL> '<STR_LIT>' : get_doctype_from_node ( node ) , <EOL> '<STR_LIT>' : serialized <EOL> } ) <EOL> if actions : <EOL> return helpers . bulk ( es , actions ) <EOL> def serialize_contributors ( node ) : <EOL> return { <EOL> '<STR_LIT>' : [ <EOL> { <EOL> '<STR_LIT>' : user . fullname , <EOL> '<STR_LIT:url>' : user . profile_url if user . is_active else None <EOL> } for user in node . visible_contributors <EOL> if user is not None <EOL> and user . is_active <EOL> ] <EOL> } <EOL> bulk_update_contributors = functools . partial ( bulk_update_nodes , serialize_contributors ) <EOL> @ requires_search <EOL> def update_user ( user , index = None ) : <EOL> index = index or INDEX <EOL> if not user . is_active : <EOL> try : <EOL> es . delete ( index = index , doc_type = '<STR_LIT:user>' , id = user . _id , refresh = True , ignore = [ <NUM_LIT> ] ) <EOL> except NotFoundError : <EOL> pass <EOL> return <EOL> names = dict ( <EOL> fullname = user . fullname , <EOL> given_name = user . given_name , <EOL> family_name = user . family_name , <EOL> middle_names = user . middle_names , <EOL> suffix = user . suffix <EOL> ) <EOL> normalized_names = { } <EOL> for key , val in names . items ( ) : <EOL> if val is not None : <EOL> try : <EOL> val = six . u ( val ) <EOL> except TypeError : <EOL> pass <EOL> normalized_names [ key ] = unicodedata . normalize ( '<STR_LIT>' , val ) . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) <EOL> user_doc = { <EOL> '<STR_LIT:id>' : user . _id , <EOL> '<STR_LIT:user>' : user . fullname , <EOL> '<STR_LIT>' : normalized_names [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : normalized_names , <EOL> '<STR_LIT>' : names , <EOL> '<STR_LIT>' : user . jobs [ <NUM_LIT:0> ] [ '<STR_LIT>' ] if user . jobs else '<STR_LIT>' , <EOL> '<STR_LIT>' : user . jobs [ <NUM_LIT:0> ] [ '<STR_LIT:title>' ] if user . jobs else '<STR_LIT>' , <EOL> '<STR_LIT>' : [ job [ '<STR_LIT>' ] for job in user . jobs [ <NUM_LIT:1> : ] ] , <EOL> '<STR_LIT>' : user . schools [ <NUM_LIT:0> ] [ '<STR_LIT>' ] if user . schools else '<STR_LIT>' , <EOL> '<STR_LIT>' : [ school [ '<STR_LIT>' ] for school in user . schools ] , <EOL> '<STR_LIT>' : '<STR_LIT:user>' , <EOL> '<STR_LIT>' : user . schools [ <NUM_LIT:0> ] [ '<STR_LIT>' ] if user . schools else '<STR_LIT>' , <EOL> '<STR_LIT>' : user . social_links , <EOL> '<STR_LIT>' : <NUM_LIT:2> , <EOL> } <EOL> es . index ( index = index , doc_type = '<STR_LIT:user>' , body = user_doc , id = user . _id , refresh = True ) <EOL> @ requires_search <EOL> def update_file ( file_ , index = None , delete = False ) : <EOL> index = index or INDEX <EOL> if not file_ . node . is_public or delete or file_ . node . is_deleted or file_ . node . archiving : <EOL> es . delete ( <EOL> index = index , <EOL> doc_type = '<STR_LIT:file>' , <EOL> id = file_ . _id , <EOL> refresh = True , <EOL> ignore = [ <NUM_LIT> ] <EOL> ) <EOL> return <EOL> file_deep_url = '<STR_LIT>' . format ( <EOL> node_id = file_ . node . _id , <EOL> provider = file_ . provider , <EOL> path = file_ . path , <EOL> ) <EOL> node_url = '<STR_LIT>' . format ( node_id = file_ . node . _id ) <EOL> file_doc = { <EOL> '<STR_LIT:id>' : file_ . _id , <EOL> '<STR_LIT>' : file_deep_url , <EOL> '<STR_LIT>' : [ tag . _id for tag in file_ . tags ] , <EOL> '<STR_LIT:name>' : file_ . name , <EOL> '<STR_LIT>' : '<STR_LIT:file>' , <EOL> '<STR_LIT>' : node_url , <EOL> '<STR_LIT>' : file_ . node . title , <EOL> '<STR_LIT>' : file_ . node . parent_node . _id if file_ . node . parent_node else None , <EOL> '<STR_LIT>' : file_ . node . is_registration , <EOL> } <EOL> es . index ( <EOL> index = index , <EOL> doc_type = '<STR_LIT:file>' , <EOL> body = file_doc , <EOL> id = file_ . _id , <EOL> refresh = True <EOL> ) <EOL> @ requires_search <EOL> def update_institution ( institution , index = None ) : <EOL> index = index or INDEX <EOL> institution_doc = { <EOL> '<STR_LIT:id>' : institution . _id , <EOL> '<STR_LIT:url>' : '<STR_LIT>' . format ( institution . _id ) , <EOL> '<STR_LIT>' : institution . logo_path , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:name>' : institution . name , <EOL> } <EOL> es . index ( index = index , doc_type = '<STR_LIT>' , body = institution_doc , id = institution . _id , refresh = True ) <EOL> @ requires_search <EOL> def delete_all ( ) : <EOL> delete_index ( INDEX ) <EOL> @ requires_search <EOL> def delete_index ( index ) : <EOL> es . indices . delete ( index , ignore = [ <NUM_LIT> ] ) <EOL> @ requires_search <EOL> def create_index ( index = None ) : <EOL> '''<STR_LIT>''' <EOL> index = index or INDEX <EOL> document_types = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:user>' , '<STR_LIT:file>' , '<STR_LIT>' ] <EOL> project_like_types = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> analyzed_fields = [ '<STR_LIT:title>' , '<STR_LIT:description>' ] <EOL> es . indices . create ( index , ignore = [ <NUM_LIT> ] ) <EOL> for type_ in document_types : <EOL> mapping = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : NOT_ANALYZED_PROPERTY , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:id>' : NOT_ANALYZED_PROPERTY , <EOL> '<STR_LIT:name>' : NOT_ANALYZED_PROPERTY , <EOL> } <EOL> } <EOL> } <EOL> } <EOL> if type_ in project_like_types : <EOL> analyzers = { field : ENGLISH_ANALYZER_PROPERTY <EOL> for field in analyzed_fields } <EOL> mapping [ '<STR_LIT>' ] . update ( analyzers ) <EOL> if type_ == '<STR_LIT:user>' : <EOL> fields = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:type>' : '<STR_LIT:string>' , <EOL> '<STR_LIT>' : '<STR_LIT:1>' , <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:type>' : '<STR_LIT:string>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:type>' : '<STR_LIT:string>' , <EOL> '<STR_LIT>' : '<STR_LIT:1>' , <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:type>' : '<STR_LIT:string>' , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } , <EOL> } <EOL> mapping [ '<STR_LIT>' ] . update ( fields ) <EOL> es . indices . put_mapping ( index = index , doc_type = type_ , body = mapping , ignore = [ <NUM_LIT> , <NUM_LIT> ] ) <EOL> @ requires_search <EOL> def delete_doc ( elastic_document_id , node , index = None , category = None ) : <EOL> index = index or INDEX <EOL> category = category or '<STR_LIT>' if node . is_registration else node . project_or_component <EOL> es . delete ( index = index , doc_type = category , id = elastic_document_id , refresh = True , ignore = [ <NUM_LIT> ] ) <EOL> @ requires_search <EOL> def search_contributor ( query , page = <NUM_LIT:0> , size = <NUM_LIT:10> , exclude = None , current_user = None ) : <EOL> """<STR_LIT>""" <EOL> start = ( page * size ) <EOL> items = re . split ( r'<STR_LIT>' , query ) <EOL> exclude = exclude or [ ] <EOL> normalized_items = [ ] <EOL> for item in items : <EOL> try : <EOL> normalized_item = six . u ( item ) <EOL> except TypeError : <EOL> normalized_item = item <EOL> normalized_item = unicodedata . normalize ( '<STR_LIT>' , normalized_item ) . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) <EOL> normalized_items . append ( normalized_item ) <EOL> items = normalized_items <EOL> query = "<STR_LIT>" . join ( '<STR_LIT>' . format ( re . escape ( item ) ) for item in items ) + "<STR_LIT>" . join ( '<STR_LIT>' . format ( excluded . _id ) for excluded in exclude ) <EOL> results = search ( build_query ( query , start = start , size = size ) , index = INDEX , doc_type = '<STR_LIT:user>' ) <EOL> docs = results [ '<STR_LIT>' ] <EOL> pages = math . ceil ( results [ '<STR_LIT>' ] . get ( '<STR_LIT:user>' , <NUM_LIT:0> ) / size ) <EOL> validate_page_num ( page , pages ) <EOL> users = [ ] <EOL> for doc in docs : <EOL> user = User . load ( doc [ '<STR_LIT:id>' ] ) <EOL> if current_user and current_user . _id == user . _id : <EOL> n_projects_in_common = - <NUM_LIT:1> <EOL> elif current_user : <EOL> n_projects_in_common = current_user . n_projects_in_common ( user ) <EOL> else : <EOL> n_projects_in_common = <NUM_LIT:0> <EOL> if user is None : <EOL> logger . error ( '<STR_LIT>' . format ( doc [ '<STR_LIT:id>' ] ) ) <EOL> continue <EOL> if user . is_active : <EOL> current_employment = None <EOL> education = None <EOL> if user . jobs : <EOL> current_employment = user . jobs [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> if user . schools : <EOL> education = user . schools [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> users . append ( { <EOL> '<STR_LIT>' : doc [ '<STR_LIT:user>' ] , <EOL> '<STR_LIT:id>' : doc [ '<STR_LIT:id>' ] , <EOL> '<STR_LIT>' : current_employment , <EOL> '<STR_LIT>' : education , <EOL> '<STR_LIT>' : n_projects_in_common , <EOL> '<STR_LIT>' : gravatar ( <EOL> user , <EOL> use_ssl = True , <EOL> size = settings . PROFILE_IMAGE_MEDIUM <EOL> ) , <EOL> '<STR_LIT>' : user . profile_url , <EOL> '<STR_LIT>' : user . is_registered , <EOL> '<STR_LIT>' : user . is_active <EOL> } ) <EOL> return { <EOL> '<STR_LIT>' : users , <EOL> '<STR_LIT>' : results [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : pages , <EOL> '<STR_LIT>' : page , <EOL> } </s>
