<s> import json <EOL> import logging <EOL> import re <EOL> import sys <EOL> import time <EOL> from django import forms <EOL> from django . contrib import messages <EOL> from django . contrib . auth . models import User <EOL> from django . db . models import Q <EOL> from django . http import HttpResponse , QueryDict <EOL> from django . shortcuts import redirect <EOL> from django . utils . html import escape <EOL> from django . utils . translation import ugettext as _ <EOL> from django . core . urlresolvers import reverse <EOL> from desktop . appmanager import get_apps_dict <EOL> from desktop . context_processors import get_app_name <EOL> from desktop . lib . paginator import Paginator <EOL> from desktop . lib . django_util import JsonResponse <EOL> from desktop . lib . django_util import copy_query_dict , format_preserving_redirect , render <EOL> from desktop . lib . django_util import login_notrequired , get_desktop_uri_prefix <EOL> from desktop . lib . exceptions_renderable import PopupException <EOL> from desktop . models import Document <EOL> from desktop . lib . parameterization import find_variables <EOL> from notebook . models import escape_rows <EOL> import beeswax . forms <EOL> import beeswax . design <EOL> import beeswax . management . commands . beeswax_install_examples <EOL> from beeswax import common , data_export , models <EOL> from beeswax . models import QueryHistory , SavedQuery , Session <EOL> from beeswax . server import dbms <EOL> from beeswax . server . dbms import expand_exception , get_query_server_config , QueryServerException <EOL> LOG = logging . getLogger ( __name__ ) <EOL> HADOOP_JOBS_RE = re . compile ( "<STR_LIT>" ) <EOL> SPARK_APPLICATION_RE = re . compile ( "<STR_LIT>" ) <EOL> def index ( request ) : <EOL> return execute_query ( request ) <EOL> """<STR_LIT>""" <EOL> def save_design ( request , form , type_ , design , explicit_save ) : <EOL> """<STR_LIT>""" <EOL> authorized_get_design ( request , design . id ) <EOL> assert form . saveform . is_valid ( ) <EOL> sub_design_form = form <EOL> if type_ == models . HQL : <EOL> design_cls = beeswax . design . HQLdesign <EOL> elif type_ == models . IMPALA : <EOL> design_cls = beeswax . design . HQLdesign <EOL> elif type_ == models . SPARK : <EOL> from spark . design import SparkDesign <EOL> design_cls = SparkDesign <EOL> sub_design_form = form . query <EOL> else : <EOL> raise ValueError ( _ ( '<STR_LIT>' ) % { '<STR_LIT:type>' : type_ } ) <EOL> design_obj = design_cls ( sub_design_form , query_type = type_ ) <EOL> name = form . saveform . cleaned_data [ '<STR_LIT:name>' ] <EOL> desc = form . saveform . cleaned_data [ '<STR_LIT>' ] <EOL> return _save_design ( request . user , design , type_ , design_obj , explicit_save , name , desc ) <EOL> def _save_design ( user , design , type_ , design_obj , explicit_save , name = None , desc = None ) : <EOL> old_design = design <EOL> new_data = design_obj . dumps ( ) <EOL> if explicit_save and ( not design . doc . exists ( ) or design . doc . get ( ) . can_write_or_exception ( user ) ) : <EOL> design . name = name <EOL> design . desc = desc <EOL> design . is_auto = False <EOL> elif design_obj != old_design . get_design ( ) : <EOL> if old_design . id is not None : <EOL> design = old_design . clone ( new_owner = user ) <EOL> if not old_design . is_auto : <EOL> design . name = old_design . name + models . SavedQuery . AUTO_DESIGN_SUFFIX <EOL> else : <EOL> design . name = models . SavedQuery . DEFAULT_NEW_DESIGN_NAME <EOL> design . is_auto = True <EOL> design . name = design . name [ : <NUM_LIT:64> ] <EOL> design . type = type_ <EOL> design . data = new_data <EOL> design . save ( ) <EOL> LOG . info ( '<STR_LIT>' % ( explicit_save and '<STR_LIT>' or '<STR_LIT>' , design . name , design . id , design . owner ) ) <EOL> if design . doc . exists ( ) : <EOL> design . doc . update ( name = design . name , description = design . desc ) <EOL> else : <EOL> Document . objects . link ( design , owner = design . owner , extra = design . type , name = design . name , description = design . desc ) <EOL> if design . is_auto : <EOL> design . doc . get ( ) . add_to_history ( ) <EOL> return design <EOL> def delete_design ( request ) : <EOL> if request . method == '<STR_LIT:POST>' : <EOL> ids = request . POST . getlist ( '<STR_LIT>' ) <EOL> designs = dict ( [ ( design_id , authorized_get_design ( request , design_id , owner_only = True ) ) for design_id in ids ] ) <EOL> if None in designs . values ( ) : <EOL> LOG . error ( '<STR_LIT>' % '<STR_LIT:U+002C>' . join ( [ key for key , name in designs . items ( ) if name is None ] ) ) <EOL> return list_designs ( request ) <EOL> for design in designs . values ( ) : <EOL> if request . POST . get ( '<STR_LIT>' , '<STR_LIT:false>' ) == '<STR_LIT:false>' : <EOL> design . doc . get ( ) . send_to_trash ( ) <EOL> else : <EOL> design . doc . all ( ) . delete ( ) <EOL> design . delete ( ) <EOL> return redirect ( reverse ( get_app_name ( request ) + '<STR_LIT>' ) ) <EOL> else : <EOL> return render ( '<STR_LIT>' , request , { '<STR_LIT:url>' : request . path , '<STR_LIT:title>' : _ ( '<STR_LIT>' ) } ) <EOL> def restore_design ( request ) : <EOL> if request . method == '<STR_LIT:POST>' : <EOL> ids = request . POST . getlist ( '<STR_LIT>' ) <EOL> designs = dict ( [ ( design_id , authorized_get_design ( request , design_id ) ) for design_id in ids ] ) <EOL> if None in designs . values ( ) : <EOL> LOG . error ( '<STR_LIT>' % '<STR_LIT:U+002C>' . join ( [ key for key , name in designs . items ( ) if name is None ] ) ) <EOL> return list_designs ( request ) <EOL> for design in designs . values ( ) : <EOL> design . doc . get ( ) . restore_from_trash ( ) <EOL> return redirect ( reverse ( get_app_name ( request ) + '<STR_LIT>' ) ) <EOL> else : <EOL> return render ( '<STR_LIT>' , request , { '<STR_LIT:url>' : request . path , '<STR_LIT:title>' : _ ( '<STR_LIT>' ) } ) <EOL> def clone_design ( request , design_id ) : <EOL> """<STR_LIT>""" <EOL> design = authorized_get_design ( request , design_id ) <EOL> if design is None : <EOL> LOG . error ( '<STR_LIT>' % ( design_id , ) ) <EOL> return list_designs ( request ) <EOL> copy = design . clone ( request . user ) <EOL> copy . save ( ) <EOL> name = copy . name + '<STR_LIT>' <EOL> design . doc . get ( ) . copy ( content_object = copy , name = name , owner = request . user ) <EOL> messages . info ( request , _ ( '<STR_LIT>' ) % { '<STR_LIT:name>' : design . name } ) <EOL> return format_preserving_redirect ( request , reverse ( get_app_name ( request ) + '<STR_LIT>' , kwargs = { '<STR_LIT>' : copy . id } ) ) <EOL> def list_designs ( request ) : <EOL> """<STR_LIT>""" <EOL> DEFAULT_PAGE_SIZE = <NUM_LIT:20> <EOL> app_name = get_app_name ( request ) <EOL> prefix = '<STR_LIT>' <EOL> querydict_query = _copy_prefix ( prefix , request . GET ) <EOL> querydict_query [ prefix + '<STR_LIT:type>' ] = app_name <EOL> search_filter = request . GET . get ( '<STR_LIT:text>' , None ) <EOL> if search_filter is not None : <EOL> querydict_query [ prefix + '<STR_LIT:text>' ] = search_filter <EOL> page , filter_params = _list_designs ( request . user , querydict_query , DEFAULT_PAGE_SIZE , prefix ) <EOL> return render ( '<STR_LIT>' , request , { <EOL> '<STR_LIT>' : page , <EOL> '<STR_LIT>' : filter_params , <EOL> '<STR_LIT>' : prefix , <EOL> '<STR_LIT:user>' : request . user , <EOL> '<STR_LIT>' : json . dumps ( [ query . id for query in page . object_list ] ) <EOL> } ) <EOL> def list_trashed_designs ( request ) : <EOL> DEFAULT_PAGE_SIZE = <NUM_LIT:20> <EOL> app_name = get_app_name ( request ) <EOL> user = request . user <EOL> prefix = '<STR_LIT>' <EOL> querydict_query = _copy_prefix ( prefix , request . GET ) <EOL> querydict_query [ prefix + '<STR_LIT:type>' ] = app_name <EOL> search_filter = request . GET . get ( '<STR_LIT:text>' , None ) <EOL> if search_filter is not None : <EOL> querydict_query [ prefix + '<STR_LIT:text>' ] = search_filter <EOL> page , filter_params = _list_designs ( user , querydict_query , DEFAULT_PAGE_SIZE , prefix , is_trashed = True ) <EOL> return render ( '<STR_LIT>' , request , { <EOL> '<STR_LIT>' : page , <EOL> '<STR_LIT>' : filter_params , <EOL> '<STR_LIT>' : prefix , <EOL> '<STR_LIT:user>' : request . user , <EOL> '<STR_LIT>' : json . dumps ( [ query . id for query in page . object_list ] ) <EOL> } ) <EOL> def my_queries ( request ) : <EOL> """<STR_LIT>""" <EOL> DEFAULT_PAGE_SIZE = <NUM_LIT:30> <EOL> app_name = get_app_name ( request ) <EOL> prefix = '<STR_LIT>' <EOL> querydict_history = _copy_prefix ( prefix , request . GET ) <EOL> querydict_history [ prefix + '<STR_LIT:user>' ] = request . user <EOL> querydict_history [ prefix + '<STR_LIT:type>' ] = app_name <EOL> hist_page , hist_filter = _list_query_history ( request . user , <EOL> querydict_history , <EOL> DEFAULT_PAGE_SIZE , <EOL> prefix ) <EOL> prefix = '<STR_LIT>' <EOL> querydict_query = _copy_prefix ( prefix , request . GET ) <EOL> querydict_query [ prefix + '<STR_LIT:user>' ] = request . user <EOL> querydict_query [ prefix + '<STR_LIT:type>' ] = app_name <EOL> query_page , query_filter = _list_designs ( request . user , querydict_query , DEFAULT_PAGE_SIZE , prefix ) <EOL> filter_params = hist_filter <EOL> filter_params . update ( query_filter ) <EOL> return render ( '<STR_LIT>' , request , { <EOL> '<STR_LIT>' : request , <EOL> '<STR_LIT>' : hist_page , <EOL> '<STR_LIT>' : query_page , <EOL> '<STR_LIT>' : filter_params , <EOL> '<STR_LIT>' : json . dumps ( [ query . id for query in query_page . object_list ] ) <EOL> } ) <EOL> def list_query_history ( request ) : <EOL> """<STR_LIT>""" <EOL> DEFAULT_PAGE_SIZE = <NUM_LIT:100> <EOL> prefix = '<STR_LIT>' <EOL> share_queries = request . user . is_superuser <EOL> querydict_query = request . GET . copy ( ) <EOL> if not share_queries : <EOL> querydict_query [ prefix + '<STR_LIT:user>' ] = request . user . username <EOL> app_name = get_app_name ( request ) <EOL> querydict_query [ prefix + '<STR_LIT:type>' ] = app_name <EOL> page , filter_params = _list_query_history ( request . user , querydict_query , DEFAULT_PAGE_SIZE , prefix ) <EOL> filter = request . GET . get ( prefix + '<STR_LIT>' ) and request . GET . get ( prefix + '<STR_LIT>' ) or '<STR_LIT>' <EOL> if request . GET . get ( '<STR_LIT>' ) == '<STR_LIT>' : <EOL> resp = { <EOL> '<STR_LIT>' : [ massage_query_history_for_json ( app_name , query_history ) for query_history in page . object_list ] <EOL> } <EOL> return JsonResponse ( resp ) <EOL> return render ( '<STR_LIT>' , request , { <EOL> '<STR_LIT>' : request , <EOL> '<STR_LIT>' : page , <EOL> '<STR_LIT>' : filter_params , <EOL> '<STR_LIT>' : share_queries , <EOL> '<STR_LIT>' : prefix , <EOL> '<STR_LIT>' : filter , <EOL> } ) <EOL> def massage_query_history_for_json ( app_name , query_history ) : <EOL> return { <EOL> '<STR_LIT:id>' : query_history . id , <EOL> '<STR_LIT>' : query_history . design . id , <EOL> '<STR_LIT>' : escape ( query_history . query ) , <EOL> '<STR_LIT>' : time . mktime ( query_history . submission_date . timetuple ( ) ) , <EOL> '<STR_LIT>' : query_history . submission_date . strftime ( "<STR_LIT>" ) , <EOL> '<STR_LIT>' : reverse ( app_name + '<STR_LIT>' , kwargs = { '<STR_LIT>' : query_history . design . id } ) , <EOL> '<STR_LIT>' : not query_history . is_failure ( ) and reverse ( app_name + '<STR_LIT>' , kwargs = { '<STR_LIT>' : query_history . id } ) or "<STR_LIT>" <EOL> } <EOL> def download ( request , id , format ) : <EOL> try : <EOL> query_history = authorized_get_query_history ( request , id , must_exist = True ) <EOL> db = dbms . get ( request . user , query_history . get_query_server_config ( ) ) <EOL> LOG . debug ( '<STR_LIT>' % ( query_history . server_id , query_history . query ) ) <EOL> return data_export . download ( query_history . get_handle ( ) , format , db ) <EOL> except Exception , e : <EOL> if not hasattr ( e , '<STR_LIT:message>' ) or not e . message : <EOL> message = e <EOL> else : <EOL> message = e . message <EOL> raise PopupException ( message , detail = '<STR_LIT>' ) <EOL> """<STR_LIT>""" <EOL> def execute_query ( request , design_id = None , query_history_id = None ) : <EOL> """<STR_LIT>""" <EOL> action = '<STR_LIT>' <EOL> if query_history_id : <EOL> query_history = authorized_get_query_history ( request , query_history_id , must_exist = True ) <EOL> design = query_history . design <EOL> try : <EOL> if query_history . server_id and query_history . server_guid : <EOL> handle , state = _get_query_handle_and_state ( query_history ) <EOL> if '<STR_LIT>' in request . GET : <EOL> if request . GET . get ( '<STR_LIT>' ) : <EOL> action = '<STR_LIT>' <EOL> else : <EOL> action = '<STR_LIT>' <EOL> else : <EOL> action = '<STR_LIT>' <EOL> except QueryServerException , e : <EOL> if '<STR_LIT>' in e . message or '<STR_LIT>' in e . message : <EOL> query_history . save_state ( QueryHistory . STATE . expired ) <EOL> LOG . warn ( "<STR_LIT>" , exc_info = sys . exc_info ( ) ) <EOL> action = '<STR_LIT>' <EOL> else : <EOL> raise e <EOL> else : <EOL> authorized_get_design ( request , design_id ) <EOL> app_name = get_app_name ( request ) <EOL> query_type = SavedQuery . TYPES_MAPPING [ app_name ] <EOL> design = safe_get_design ( request , query_type , design_id ) <EOL> query_history = None <EOL> doc = design and design . id and design . doc . get ( ) <EOL> context = { <EOL> '<STR_LIT>' : design , <EOL> '<STR_LIT>' : query_history , <EOL> '<STR_LIT>' : query_history , <EOL> '<STR_LIT>' : reverse ( get_app_name ( request ) + '<STR_LIT>' , kwargs = { } ) , <EOL> '<STR_LIT>' : reverse ( '<STR_LIT>' , kwargs = { } ) , <EOL> '<STR_LIT>' : design and design . id and not design . is_auto , <EOL> '<STR_LIT>' : doc and doc . id or - <NUM_LIT:1> , <EOL> '<STR_LIT>' : doc and doc . can_write ( request . user ) , <EOL> '<STR_LIT:action>' : action , <EOL> '<STR_LIT>' : request . GET . get ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' : '<STR_LIT>' in get_apps_dict ( request . user ) <EOL> } <EOL> return render ( '<STR_LIT>' , request , context ) <EOL> def view_results ( request , id , first_row = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> first_row = long ( first_row ) <EOL> start_over = ( first_row == <NUM_LIT:0> ) <EOL> results = type ( '<STR_LIT>' , ( object , ) , { <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : False , <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> } ) <EOL> data = [ ] <EOL> fetch_error = False <EOL> error_message = '<STR_LIT>' <EOL> log = '<STR_LIT>' <EOL> columns = [ ] <EOL> app_name = get_app_name ( request ) <EOL> query_history = authorized_get_query_history ( request , id , must_exist = True ) <EOL> query_server = query_history . get_query_server_config ( ) <EOL> db = dbms . get ( request . user , query_server ) <EOL> handle , state = _get_query_handle_and_state ( query_history ) <EOL> context_param = request . GET . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> query_context = parse_query_context ( context_param ) <EOL> expired = state == models . QueryHistory . STATE . expired <EOL> try : <EOL> if query_server [ '<STR_LIT>' ] == '<STR_LIT>' and not handle . has_result_set : <EOL> downloadable = False <EOL> else : <EOL> results = db . fetch ( handle , start_over , <NUM_LIT:100> ) <EOL> data = escape_rows ( results . rows ( ) ) <EOL> downloadable = first_row > <NUM_LIT:0> or data <EOL> log = db . get_log ( handle ) <EOL> columns = results . data_table . cols ( ) <EOL> except Exception , ex : <EOL> LOG . exception ( '<STR_LIT>' ) <EOL> fetch_error = True <EOL> error_message , log = expand_exception ( ex , db , handle ) <EOL> error = fetch_error or results is None or expired <EOL> context = { <EOL> '<STR_LIT:error>' : error , <EOL> '<STR_LIT:message>' : error_message , <EOL> '<STR_LIT>' : query_history , <EOL> '<STR_LIT>' : data , <EOL> '<STR_LIT>' : columns , <EOL> '<STR_LIT>' : first_row , <EOL> '<STR_LIT>' : log , <EOL> '<STR_LIT>' : app_name != '<STR_LIT>' and _parse_out_hadoop_jobs ( log ) , <EOL> '<STR_LIT>' : query_context , <EOL> '<STR_LIT>' : False , <EOL> '<STR_LIT>' : context_param , <EOL> '<STR_LIT>' : expired , <EOL> '<STR_LIT>' : app_name , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : query_history . is_finished ( ) <EOL> } <EOL> if not error : <EOL> download_urls = { } <EOL> if downloadable : <EOL> for format in common . DL_FORMATS : <EOL> download_urls [ format ] = reverse ( app_name + '<STR_LIT>' , kwargs = dict ( id = str ( id ) , format = format ) ) <EOL> results . start_row = first_row <EOL> context . update ( { <EOL> '<STR_LIT:id>' : id , <EOL> '<STR_LIT>' : data , <EOL> '<STR_LIT>' : results . has_more , <EOL> '<STR_LIT>' : results . start_row + len ( data ) , <EOL> '<STR_LIT>' : results . start_row , <EOL> '<STR_LIT>' : first_row , <EOL> '<STR_LIT>' : columns , <EOL> '<STR_LIT>' : download_urls , <EOL> '<STR_LIT>' : query_history . owner == request . user , <EOL> '<STR_LIT>' : <EOL> reverse ( get_app_name ( request ) + '<STR_LIT>' , kwargs = { <EOL> '<STR_LIT:id>' : str ( id ) , <EOL> '<STR_LIT>' : results . start_row + len ( data ) <EOL> } <EOL> ) <EOL> + ( '<STR_LIT>' + context_param or '<STR_LIT>' ) + '<STR_LIT>' <EOL> } ) <EOL> context [ '<STR_LIT>' ] = massage_columns_for_json ( columns ) <EOL> if '<STR_LIT>' in context : <EOL> del context [ '<STR_LIT>' ] <EOL> if '<STR_LIT>' in context : <EOL> del context [ '<STR_LIT>' ] <EOL> return JsonResponse ( context ) <EOL> def configuration ( request ) : <EOL> app_name = get_app_name ( request ) <EOL> query_server = get_query_server_config ( app_name ) <EOL> session = Session . objects . get_session ( request . user , query_server [ '<STR_LIT>' ] ) <EOL> if session : <EOL> properties = json . loads ( session . properties ) <EOL> for key , value in properties . items ( ) : <EOL> if '<STR_LIT:password>' in key . lower ( ) : <EOL> properties [ key ] = '<STR_LIT:*>' * len ( value ) <EOL> else : <EOL> properties = { } <EOL> return render ( "<STR_LIT>" , request , { '<STR_LIT>' : properties } ) <EOL> """<STR_LIT>""" <EOL> def install_examples ( request ) : <EOL> response = { '<STR_LIT:status>' : - <NUM_LIT:1> , '<STR_LIT:message>' : '<STR_LIT>' } <EOL> if request . method == '<STR_LIT:POST>' : <EOL> try : <EOL> app_name = get_app_name ( request ) <EOL> beeswax . management . commands . beeswax_install_examples . Command ( ) . handle ( app_name = app_name , user = request . user ) <EOL> response [ '<STR_LIT:status>' ] = <NUM_LIT:0> <EOL> except Exception , err : <EOL> LOG . exception ( err ) <EOL> response [ '<STR_LIT:message>' ] = str ( err ) <EOL> else : <EOL> response [ '<STR_LIT:message>' ] = _ ( '<STR_LIT>' ) <EOL> return JsonResponse ( response ) <EOL> @ login_notrequired <EOL> def query_done_cb ( request , server_id ) : <EOL> """<STR_LIT>""" <EOL> message_template = '<STR_LIT>' <EOL> message = { '<STR_LIT:message>' : '<STR_LIT:error>' } <EOL> try : <EOL> query_history = QueryHistory . objects . get ( server_id = server_id + '<STR_LIT:\n>' ) <EOL> query_history . set_to_available ( ) <EOL> if not query_history . notify : <EOL> message [ '<STR_LIT:message>' ] = '<STR_LIT>' <EOL> return HttpResponse ( message_template % message ) <EOL> design = query_history . design <EOL> user = query_history . owner <EOL> subject = _ ( "<STR_LIT>" ) <EOL> if design : <EOL> subject += "<STR_LIT>" % ( design . name , ) <EOL> link = "<STR_LIT>" % ( get_desktop_uri_prefix ( ) , <EOL> reverse ( get_app_name ( request ) + '<STR_LIT>' , kwargs = { '<STR_LIT>' : query_history . id } ) ) <EOL> body = _ ( "<STR_LIT>" ) % { <EOL> '<STR_LIT>' : subject , '<STR_LIT>' : link , '<STR_LIT>' : query_history . query <EOL> } <EOL> user . email_user ( subject , body ) <EOL> message [ '<STR_LIT:message>' ] = '<STR_LIT>' <EOL> except Exception , ex : <EOL> msg = "<STR_LIT>" % ( ex ) <EOL> LOG . error ( msg ) <EOL> message [ '<STR_LIT:message>' ] = msg <EOL> return HttpResponse ( message_template % message ) <EOL> """<STR_LIT>""" <EOL> def massage_columns_for_json ( cols ) : <EOL> massaged_cols = [ ] <EOL> for column in cols : <EOL> massaged_cols . append ( { <EOL> '<STR_LIT:name>' : column . name , <EOL> '<STR_LIT:type>' : column . type , <EOL> '<STR_LIT>' : column . comment <EOL> } ) <EOL> return massaged_cols <EOL> def authorized_get_design ( request , design_id , owner_only = False , must_exist = False ) : <EOL> if design_id is None and not must_exist : <EOL> return None <EOL> try : <EOL> design = SavedQuery . objects . get ( id = design_id ) <EOL> except SavedQuery . DoesNotExist : <EOL> if must_exist : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) % { '<STR_LIT:id>' : design_id } ) <EOL> else : <EOL> return None <EOL> if owner_only : <EOL> design . doc . get ( ) . can_write_or_exception ( request . user ) <EOL> else : <EOL> design . doc . get ( ) . can_read_or_exception ( request . user ) <EOL> return design <EOL> def authorized_get_query_history ( request , query_history_id , owner_only = False , must_exist = False ) : <EOL> if query_history_id is None and not must_exist : <EOL> return None <EOL> try : <EOL> query_history = QueryHistory . get ( id = query_history_id ) <EOL> except QueryHistory . DoesNotExist : <EOL> if must_exist : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) % { '<STR_LIT:id>' : query_history_id } ) <EOL> else : <EOL> return None <EOL> if query_history . design is None or not query_history . design . doc . exists ( ) : <EOL> if not request . user . is_superuser and request . user != query_history . owner : <EOL> raise PopupException ( _ ( '<STR_LIT>' ) % { '<STR_LIT:id>' : query_history_id } ) <EOL> else : <EOL> query_history . design . doc . get ( ) . can_read_or_exception ( request . user ) <EOL> return query_history <EOL> def safe_get_design ( request , design_type , design_id = None ) : <EOL> """<STR_LIT>""" <EOL> design = None <EOL> if design_id is not None : <EOL> design = authorized_get_design ( request , design_id ) <EOL> if design is None : <EOL> design = SavedQuery ( owner = request . user , type = design_type ) <EOL> return design <EOL> def make_parameterization_form ( query_str ) : <EOL> """<STR_LIT>""" <EOL> variables = find_variables ( query_str ) <EOL> if len ( variables ) > <NUM_LIT:0> : <EOL> class Form ( forms . Form ) : <EOL> for name in sorted ( variables ) : <EOL> locals ( ) [ name ] = forms . CharField ( required = True ) <EOL> return Form <EOL> else : <EOL> return None <EOL> def execute_directly ( request , query , query_server = None , <EOL> design = None , on_success_url = None , on_success_params = None , <EOL> ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if design is not None : <EOL> authorized_get_design ( request , design . id ) <EOL> db = dbms . get ( request . user , query_server ) <EOL> database = query . query . get ( '<STR_LIT>' , '<STR_LIT:default>' ) <EOL> db . use ( database ) <EOL> query_history = db . execute_query ( query , design ) <EOL> watch_url = reverse ( get_app_name ( request ) + '<STR_LIT>' , kwargs = { '<STR_LIT>' : query_history . id } ) <EOL> get_dict = QueryDict ( None , mutable = True ) <EOL> if on_success_url : <EOL> if callable ( on_success_url ) : <EOL> on_success_url = on_success_url ( query_history ) <EOL> get_dict [ '<STR_LIT>' ] = on_success_url <EOL> if on_success_params : <EOL> get_dict . update ( on_success_params ) <EOL> return format_preserving_redirect ( request , watch_url , get_dict ) <EOL> def _list_designs ( user , querydict , page_size , prefix = "<STR_LIT>" , is_trashed = False ) : <EOL> """<STR_LIT>""" <EOL> DEFAULT_SORT = ( '<STR_LIT:->' , '<STR_LIT:date>' ) <EOL> SORT_ATTR_TRANSLATION = dict ( <EOL> date = '<STR_LIT>' , <EOL> name = '<STR_LIT:name>' , <EOL> desc = '<STR_LIT:description>' , <EOL> type = '<STR_LIT>' , <EOL> ) <EOL> if is_trashed : <EOL> db_queryset = Document . objects . trashed_docs ( SavedQuery , user ) <EOL> else : <EOL> db_queryset = Document . objects . available_docs ( SavedQuery , user ) <EOL> filter_username = querydict . get ( prefix + '<STR_LIT:user>' ) <EOL> if filter_username : <EOL> try : <EOL> db_queryset = db_queryset . filter ( owner = User . objects . get ( username = filter_username ) ) <EOL> except User . DoesNotExist : <EOL> pass <EOL> d_type = querydict . get ( prefix + '<STR_LIT:type>' ) <EOL> if d_type and d_type in SavedQuery . TYPES_MAPPING . keys ( ) : <EOL> db_queryset = db_queryset . filter ( extra = str ( SavedQuery . TYPES_MAPPING [ d_type ] ) ) <EOL> frag = querydict . get ( prefix + '<STR_LIT:text>' ) <EOL> if frag : <EOL> db_queryset = db_queryset . filter ( Q ( name__icontains = frag ) | Q ( description__icontains = frag ) ) <EOL> sort_key = querydict . get ( prefix + '<STR_LIT>' ) <EOL> if sort_key : <EOL> if sort_key [ <NUM_LIT:0> ] == '<STR_LIT:->' : <EOL> sort_dir , sort_attr = '<STR_LIT:->' , sort_key [ <NUM_LIT:1> : ] <EOL> else : <EOL> sort_dir , sort_attr = '<STR_LIT>' , sort_key <EOL> if not SORT_ATTR_TRANSLATION . has_key ( sort_attr ) : <EOL> LOG . warn ( '<STR_LIT>' % ( sort_key , ) ) <EOL> sort_dir , sort_attr = DEFAULT_SORT <EOL> else : <EOL> sort_dir , sort_attr = DEFAULT_SORT <EOL> db_queryset = db_queryset . order_by ( sort_dir + SORT_ATTR_TRANSLATION [ sort_attr ] ) <EOL> designs = [ job . content_object for job in db_queryset . all ( ) if job . content_object and job . content_object . is_auto == False ] <EOL> pagenum = int ( querydict . get ( prefix + '<STR_LIT>' , <NUM_LIT:1> ) ) <EOL> paginator = Paginator ( designs , page_size ) <EOL> page = paginator . page ( pagenum ) <EOL> keys_to_copy = [ prefix + key for key in ( '<STR_LIT:user>' , '<STR_LIT:type>' , '<STR_LIT>' , '<STR_LIT:text>' ) ] <EOL> filter_params = copy_query_dict ( querydict , keys_to_copy ) <EOL> return page , filter_params <EOL> def _get_query_handle_and_state ( query_history ) : <EOL> """<STR_LIT>""" <EOL> handle = query_history . get_handle ( ) <EOL> if handle is None : <EOL> raise PopupException ( _ ( "<STR_LIT>" ) ) <EOL> state = dbms . get ( query_history . owner , query_history . get_query_server_config ( ) ) . get_state ( handle ) <EOL> if state is None : <EOL> raise PopupException ( _ ( "<STR_LIT>" ) ) <EOL> return ( handle , state ) <EOL> def parse_query_context ( context ) : <EOL> """<STR_LIT>""" <EOL> if not context : <EOL> return None <EOL> pair = context . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> if len ( pair ) != <NUM_LIT:2> or pair [ <NUM_LIT:0> ] not in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> LOG . error ( "<STR_LIT>" % ( context , ) ) <EOL> return None <EOL> if pair [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> pair [ <NUM_LIT:1> ] = models . SavedQuery . get ( int ( pair [ <NUM_LIT:1> ] ) ) <EOL> return pair <EOL> def _parse_out_hadoop_jobs ( log , engine = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> ret = [ ] <EOL> if engine . lower ( ) == '<STR_LIT>' : <EOL> pattern = HADOOP_JOBS_RE <EOL> elif engine . lower ( ) == '<STR_LIT>' : <EOL> pattern = SPARK_APPLICATION_RE <EOL> else : <EOL> raise ValueError ( _ ( '<STR_LIT>' ) % { '<STR_LIT>' : engine } ) <EOL> for match in pattern . finditer ( log ) : <EOL> job_id = match . group ( <NUM_LIT:1> ) <EOL> if job_id not in ret : <EOL> ret . append ( job_id ) <EOL> return ret <EOL> def _copy_prefix ( prefix , base_dict ) : <EOL> """<STR_LIT>""" <EOL> querydict = QueryDict ( None , mutable = True ) <EOL> for key , val in base_dict . iteritems ( ) : <EOL> if key . startswith ( prefix ) : <EOL> querydict [ key ] = val <EOL> return querydict <EOL> def _list_query_history ( user , querydict , page_size , prefix = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> DEFAULT_SORT = ( '<STR_LIT:->' , '<STR_LIT:date>' ) <EOL> SORT_ATTR_TRANSLATION = dict ( <EOL> date = '<STR_LIT>' , <EOL> state = '<STR_LIT>' , <EOL> name = '<STR_LIT>' , <EOL> type = '<STR_LIT>' , <EOL> ) <EOL> db_queryset = models . QueryHistory . objects . select_related ( ) <EOL> if querydict . get ( prefix + '<STR_LIT>' , '<STR_LIT>' ) != '<STR_LIT>' : <EOL> db_queryset = db_queryset . exclude ( design__isnull = False , design__is_auto = True ) <EOL> user_filter = querydict . get ( prefix + '<STR_LIT:user>' , user . username ) <EOL> if user_filter != '<STR_LIT>' : <EOL> db_queryset = db_queryset . filter ( owner__username = user_filter ) <EOL> design_id = querydict . get ( prefix + '<STR_LIT>' ) <EOL> if design_id : <EOL> db_queryset = db_queryset . filter ( design__id = int ( design_id ) ) <EOL> search_filter = querydict . get ( prefix + '<STR_LIT>' ) <EOL> if search_filter : <EOL> db_queryset = db_queryset . filter ( Q ( design__name__icontains = search_filter ) | Q ( query__icontains = search_filter ) | Q ( owner__username__icontains = search_filter ) ) <EOL> d_type = querydict . get ( prefix + '<STR_LIT:type>' ) <EOL> if d_type : <EOL> if d_type not in SavedQuery . TYPES_MAPPING . keys ( ) : <EOL> LOG . warn ( '<STR_LIT>' % ( d_type , ) ) <EOL> else : <EOL> db_queryset = db_queryset . filter ( design__type = SavedQuery . TYPES_MAPPING [ d_type ] ) <EOL> recent = querydict . get ( '<STR_LIT>' ) <EOL> if recent : <EOL> db_queryset = db_queryset . filter ( is_cleared = False ) <EOL> sort_key = querydict . get ( prefix + '<STR_LIT>' ) <EOL> if sort_key : <EOL> sort_dir , sort_attr = '<STR_LIT>' , sort_key <EOL> if sort_key [ <NUM_LIT:0> ] == '<STR_LIT:->' : <EOL> sort_dir , sort_attr = '<STR_LIT:->' , sort_key [ <NUM_LIT:1> : ] <EOL> if not SORT_ATTR_TRANSLATION . has_key ( sort_attr ) : <EOL> LOG . warn ( '<STR_LIT>' % ( sort_key , ) ) <EOL> sort_dir , sort_attr = DEFAULT_SORT <EOL> else : <EOL> sort_dir , sort_attr = DEFAULT_SORT <EOL> db_queryset = db_queryset . order_by ( sort_dir + SORT_ATTR_TRANSLATION [ sort_attr ] , '<STR_LIT>' ) <EOL> total_count = db_queryset . count ( ) <EOL> pagenum = int ( querydict . get ( prefix + '<STR_LIT>' , <NUM_LIT:1> ) ) <EOL> if pagenum < <NUM_LIT:1> : <EOL> pagenum = <NUM_LIT:1> <EOL> db_queryset = db_queryset [ page_size * ( pagenum - <NUM_LIT:1> ) : page_size * pagenum ] <EOL> paginator = Paginator ( db_queryset , page_size , total = total_count ) <EOL> page = paginator . page ( pagenum ) <EOL> for history in page . object_list : <EOL> _update_query_state ( history . get_full_object ( ) ) <EOL> keys_to_copy = [ prefix + key for key in ( '<STR_LIT:user>' , '<STR_LIT:type>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ] <EOL> filter_params = copy_query_dict ( querydict , keys_to_copy ) <EOL> return page , filter_params <EOL> def _update_query_state ( query_history ) : <EOL> """<STR_LIT>""" <EOL> if query_history . last_state <= models . QueryHistory . STATE . running . index : <EOL> try : <EOL> state_enum = dbms . get ( query_history . owner , query_history . get_query_server_config ( ) ) . get_state ( query_history . get_handle ( ) ) <EOL> if state_enum is None : <EOL> return False <EOL> except Exception , e : <EOL> LOG . error ( e ) <EOL> state_enum = models . QueryHistory . STATE . failed <EOL> query_history . save_state ( state_enum ) <EOL> return True <EOL> def get_db_choices ( request ) : <EOL> app_name = get_app_name ( request ) <EOL> query_server = get_query_server_config ( app_name ) <EOL> db = dbms . get ( request . user , query_server ) <EOL> dbs = db . get_databases ( ) <EOL> return [ ( db , db ) for db in dbs ] <EOL> WHITESPACE = re . compile ( "<STR_LIT>" , re . MULTILINE ) <EOL> def collapse_whitespace ( s ) : <EOL> return WHITESPACE . sub ( "<STR_LIT:U+0020>" , s ) . strip ( ) </s>
