<s> import itertools <EOL> import json <EOL> import logging <EOL> import re <EOL> from django . utils . translation import ugettext as _ <EOL> from desktop . lib . django_util import JsonResponse <EOL> from desktop . lib . exceptions_renderable import PopupException <EOL> from search . models import Collection <EOL> from indexer . controller import CollectionManagerController <EOL> from indexer . utils import fields_from_log , field_values_from_separated_file , get_type_from_morphline_type , get_field_types <EOL> LOG = logging . getLogger ( __name__ ) <EOL> def parse_fields ( request ) : <EOL> if request . method != '<STR_LIT:POST>' : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) ) <EOL> result = { '<STR_LIT:status>' : - <NUM_LIT:1> } <EOL> source_type = request . POST . get ( '<STR_LIT:source>' ) <EOL> if source_type == '<STR_LIT:file>' : <EOL> content_type = request . POST . get ( '<STR_LIT:type>' ) <EOL> try : <EOL> if content_type == '<STR_LIT>' : <EOL> delimiter = request . POST . get ( '<STR_LIT>' , '<STR_LIT:U+002C>' ) <EOL> quote = request . POST . get ( '<STR_LIT>' , '<STR_LIT:">' ) <EOL> file_obj = request . fs . open ( request . POST . get ( '<STR_LIT:path>' ) ) <EOL> field_list = field_values_from_separated_file ( file_obj , delimiter , quote ) <EOL> row = next ( field_list ) <EOL> field_names = row . keys ( ) <EOL> field_types = get_field_types ( ( row . values ( ) for row in itertools . chain ( [ row ] , field_list ) ) , iterations = <NUM_LIT> ) <EOL> file_obj . close ( ) <EOL> result [ '<STR_LIT:data>' ] = zip ( field_names , field_types ) <EOL> result [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> elif content_type == '<STR_LIT>' : <EOL> morphlines = json . loads ( request . POST . get ( '<STR_LIT>' ) ) <EOL> field_results = re . findall ( r'<STR_LIT>' , morphlines [ '<STR_LIT>' ] ) <EOL> if field_results : <EOL> result [ '<STR_LIT:data>' ] = [ ] <EOL> for field_result in field_results : <EOL> result [ '<STR_LIT:data>' ] . append ( ( field_result [ <NUM_LIT:1> ] , get_type_from_morphline_type ( field_result [ <NUM_LIT:0> ] ) ) ) <EOL> result [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> else : <EOL> result [ '<STR_LIT:status>' ] = <NUM_LIT:1> <EOL> result [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> elif content_type == '<STR_LIT>' : <EOL> file_obj = request . fs . open ( request . POST . get ( '<STR_LIT:path>' ) ) <EOL> result [ '<STR_LIT:data>' ] = fields_from_log ( file_obj ) <EOL> file_obj . close ( ) <EOL> result [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> else : <EOL> result [ '<STR_LIT:status>' ] = <NUM_LIT:1> <EOL> result [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) % content_type <EOL> except Exception , e : <EOL> LOG . exception ( e . message ) <EOL> result [ '<STR_LIT:message>' ] = e . message <EOL> else : <EOL> result [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) % source_type <EOL> return JsonResponse ( result ) <EOL> def collections ( request ) : <EOL> searcher = CollectionManagerController ( request . user ) <EOL> solr_collections = searcher . get_collections ( ) <EOL> massaged_collections = [ ] <EOL> for collection in solr_collections : <EOL> massaged_collections . append ( { <EOL> '<STR_LIT:name>' : collection , <EOL> '<STR_LIT>' : solr_collections [ collection ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : solr_collections [ collection ] . get ( '<STR_LIT>' , False ) , <EOL> '<STR_LIT>' : solr_collections [ collection ] . get ( '<STR_LIT>' , [ ] ) , <EOL> } ) <EOL> response = { <EOL> '<STR_LIT:status>' : <NUM_LIT:0> , <EOL> '<STR_LIT>' : massaged_collections <EOL> } <EOL> return JsonResponse ( response ) <EOL> def collections_create ( request ) : <EOL> if request . method != '<STR_LIT:POST>' : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) ) <EOL> response = { '<STR_LIT:status>' : - <NUM_LIT:1> } <EOL> collection = json . loads ( request . POST . get ( '<STR_LIT>' , '<STR_LIT:{}>' ) ) <EOL> if collection : <EOL> searcher = CollectionManagerController ( request . user ) <EOL> searcher . create_collection ( collection . get ( '<STR_LIT:name>' ) , collection . get ( '<STR_LIT>' , [ ] ) , collection . get ( '<STR_LIT>' ) , collection . get ( '<STR_LIT>' ) ) <EOL> try : <EOL> if request . POST . get ( '<STR_LIT:source>' ) == '<STR_LIT:file>' : <EOL> searcher . update_data_from_hdfs ( request . fs , <EOL> collection . get ( '<STR_LIT:name>' ) , <EOL> collection . get ( '<STR_LIT>' , [ ] ) , <EOL> request . POST . get ( '<STR_LIT:path>' ) , <EOL> request . POST . get ( '<STR_LIT:type>' ) , <EOL> separator = request . POST . get ( '<STR_LIT>' ) , <EOL> quote_character = request . POST . get ( '<STR_LIT>' ) ) <EOL> elif request . POST . get ( '<STR_LIT:source>' ) == '<STR_LIT>' : <EOL> from beeswax . server import dbms <EOL> db = dbms . get ( request . user ) <EOL> database = request . POST . get ( '<STR_LIT>' ) <EOL> table = request . POST . get ( '<STR_LIT>' ) <EOL> columns = [ field [ '<STR_LIT:name>' ] for field in collection . get ( '<STR_LIT>' , [ ] ) ] <EOL> searcher . update_data_from_hive ( db , collection . get ( '<STR_LIT:name>' ) , database , table , columns ) <EOL> response [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> except Exception , e : <EOL> LOG . error ( e ) <EOL> raise <EOL> else : <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> return JsonResponse ( response ) <EOL> def collections_import ( request ) : <EOL> if request . method != '<STR_LIT:POST>' : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) ) <EOL> response = { '<STR_LIT:status>' : - <NUM_LIT:1> } <EOL> collection = json . loads ( request . POST . get ( '<STR_LIT>' , '<STR_LIT:{}>' ) ) <EOL> if collection : <EOL> searcher = CollectionManagerController ( request . user ) <EOL> unique_key , fields = searcher . get_fields ( collection . get ( '<STR_LIT:name>' ) ) <EOL> hue_collection , created = Collection . objects . get_or_create ( name = collection . get ( '<STR_LIT:name>' ) , solr_properties = '<STR_LIT:{}>' , is_enabled = True , user = request . user ) <EOL> properties_dict = hue_collection . properties_dict <EOL> properties_dict [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> properties_dict [ '<STR_LIT>' ] = [ field_name for field_name in fields ] <EOL> hue_collection . properties = json . dumps ( properties_dict ) <EOL> hue_collection . save ( ) <EOL> response [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> else : <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> return JsonResponse ( response ) <EOL> def collections_remove ( request ) : <EOL> if request . method != '<STR_LIT:POST>' : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) ) <EOL> response = { '<STR_LIT:status>' : - <NUM_LIT:1> } <EOL> collections = json . loads ( request . POST . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> if not collections : <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> if response . get ( '<STR_LIT:message>' , None ) is None : <EOL> searcher = CollectionManagerController ( request . user ) <EOL> solr_collections = searcher . get_collections ( ) <EOL> for collection in collections : <EOL> if collection . get ( '<STR_LIT:name>' ) in solr_collections : <EOL> searcher . delete_collection ( collection . get ( '<STR_LIT:name>' ) , collection . get ( '<STR_LIT>' ) ) <EOL> response [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> return JsonResponse ( response ) <EOL> def collections_fields ( request , collection ) : <EOL> if request . method != '<STR_LIT:GET>' : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) ) <EOL> response = { } <EOL> searcher = CollectionManagerController ( request . user ) <EOL> unique_key , fields = searcher . get_fields ( collection ) <EOL> response [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> response [ '<STR_LIT>' ] = [ ( field , fields [ field ] [ '<STR_LIT:type>' ] , fields [ field ] . get ( '<STR_LIT>' , None ) , fields [ field ] . get ( '<STR_LIT>' , None ) ) for field in fields ] <EOL> response [ '<STR_LIT>' ] = unique_key <EOL> return JsonResponse ( response ) <EOL> def collections_update ( request , collection ) : <EOL> if request . method != '<STR_LIT:POST>' : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) ) <EOL> response = { '<STR_LIT:status>' : - <NUM_LIT:1> } <EOL> collection = json . loads ( request . POST . get ( '<STR_LIT>' , '<STR_LIT:{}>' ) ) <EOL> if not collection : <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> if response . get ( '<STR_LIT:message>' , None ) is None : <EOL> searcher = CollectionManagerController ( request . user ) <EOL> searcher . update_collection ( collection . get ( '<STR_LIT:name>' ) , collection . get ( '<STR_LIT>' , [ ] ) ) <EOL> response [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> return JsonResponse ( response ) <EOL> def collections_data ( request , collection ) : <EOL> if request . method != '<STR_LIT:POST>' : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) ) <EOL> response = { '<STR_LIT:status>' : - <NUM_LIT:1> } <EOL> source = request . POST . get ( '<STR_LIT:source>' ) <EOL> if source == '<STR_LIT:file>' : <EOL> searcher = CollectionManagerController ( request . user ) <EOL> searcher . update_data_from_hdfs ( request . fs , <EOL> collection , <EOL> None , <EOL> request . POST . get ( '<STR_LIT:path>' ) , <EOL> request . POST . get ( '<STR_LIT:type>' ) , <EOL> separator = request . POST . get ( '<STR_LIT>' ) , <EOL> quote_character = request . POST . get ( '<STR_LIT>' ) ) <EOL> response [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> else : <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) % source <EOL> return JsonResponse ( response ) </s>
