<s> from django . conf import settings <EOL> from django . views . decorators . http import require_http_methods <EOL> from django . views . decorators . csrf import csrf_exempt <EOL> from django . shortcuts import render_to_response , redirect <EOL> from django . template import RequestContext , Context , loader <EOL> from django . utils import timezone <EOL> from django . http import HttpResponse <EOL> from django . db import transaction <EOL> from utils . quota_util import QuotaUpdater <EOL> from models import Table <EOL> import datetime <EOL> import dbutil <EOL> import json <EOL> import logging <EOL> import metric_helper <EOL> import time <EOL> import owl_config <EOL> logger = logging . getLogger ( __name__ ) <EOL> class Namespace : <EOL> def __init__ ( self , ** kwargs ) : <EOL> for name , value in kwargs . iteritems ( ) : <EOL> setattr ( self , name , value ) <EOL> def index ( request ) : <EOL> clusters = dbutil . get_clusters_by_service ( ) <EOL> service = Namespace ( name = "<STR_LIT>" ) <EOL> params = { <EOL> '<STR_LIT>' : service , <EOL> '<STR_LIT>' : clusters , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_service ( request , id ) : <EOL> service = dbutil . get_service ( id ) <EOL> clusters = dbutil . get_clusters_by_service ( id ) <EOL> params = { <EOL> '<STR_LIT>' : service , <EOL> '<STR_LIT>' : clusters , <EOL> } <EOL> if service . name == '<STR_LIT>' : <EOL> tsdb_read_query = [ ] <EOL> tsdb_write_query = [ ] <EOL> for cluster in clusters : <EOL> tsdb_read_query . append ( metric_helper . make_metric_query ( cluster . name , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> tsdb_write_query . append ( metric_helper . make_metric_query ( cluster . name , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> params . update ( { <EOL> '<STR_LIT>' : tsdb_read_query , <EOL> '<STR_LIT>' : tsdb_write_query , <EOL> } ) <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> else : <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_cluster ( request , id ) : <EOL> return redirect ( '<STR_LIT>' % id ) <EOL> def show_cluster_task_board ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> tasks = dbutil . get_tasks_by_cluster ( id ) <EOL> params = { '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : tasks } <EOL> if cluster . service . name == '<STR_LIT>' : <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> elif cluster . service . name == '<STR_LIT>' : <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> elif cluster . service . name == '<STR_LIT>' : <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> else : <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_cluster_user_board ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> if cluster . service . name == '<STR_LIT>' : <EOL> return show_hdfs_user_board ( request , cluster ) ; <EOL> return HttpResponse ( '<STR_LIT>' ) <EOL> def show_hdfs_user_board ( request , cluster ) : <EOL> if '<STR_LIT>' in request . GET : <EOL> quota_updater = QuotaUpdater ( ) <EOL> quota_updater . update_cluster ( cluster ) <EOL> return redirect ( '<STR_LIT>' % cluster . id ) <EOL> dirs = dbutil . get_quota_summary ( cluster ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : dirs , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_cluster_table_board ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> if cluster . service . name != '<STR_LIT>' : <EOL> return HttpResponse ( '<STR_LIT>' ) <EOL> read_requests_dist_by_table , write_requests_dist_by_table = dbutil . get_requests_distribution_groupby ( cluster , '<STR_LIT>' ) ; <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : read_requests_dist_by_table , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> read_requests_dist_by_table_chart = loader . get_template ( '<STR_LIT>' ) . render ( Context ( params ) ) <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : write_requests_dist_by_table , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> write_requests_dist_by_table_chart = loader . get_template ( '<STR_LIT>' ) . render ( <EOL> Context ( params ) ) <EOL> tables = dbutil . get_items_on_cluster ( cluster , '<STR_LIT>' , order_by = '<STR_LIT>' ) <EOL> system_tables = [ table for table in tables if is_system_table ( table ) ] <EOL> user_tables = [ table for table in tables if not is_system_table ( table ) ] <EOL> table_read_item_keys = '<STR_LIT:|>' . join ( [ '<STR_LIT>' % ( table . name ) for table in user_tables ] ) <EOL> table_write_item_keys = '<STR_LIT:|>' . join ( [ '<STR_LIT>' % ( table . name ) for table in user_tables ] ) <EOL> tsdb_read_query = [ ] <EOL> tsdb_write_query = [ ] <EOL> for table in user_tables : <EOL> tsdb_read_query . append ( metric_helper . make_metric_query ( cluster . name , table . name , '<STR_LIT>' ) ) <EOL> tsdb_write_query . append ( metric_helper . make_metric_query ( cluster . name , table . name , '<STR_LIT>' ) ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : read_requests_dist_by_table_chart , <EOL> '<STR_LIT>' : write_requests_dist_by_table_chart , <EOL> '<STR_LIT>' : system_tables , <EOL> '<STR_LIT>' : user_tables , <EOL> '<STR_LIT>' : table_read_item_keys , <EOL> '<STR_LIT>' : table_write_item_keys , <EOL> '<STR_LIT>' : tsdb_read_query , <EOL> '<STR_LIT>' : tsdb_write_query , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_quota_total_board ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> if cluster . service . name != '<STR_LIT>' : <EOL> return HttpResponse ( '<STR_LIT>' ) <EOL> tsdb_quota_total , tsdb_space_quota_total = dbutil . get_quota_distribution ( cluster ) <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : tsdb_quota_total , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> tsdb_quota_total_chart = loader . get_template ( '<STR_LIT>' ) . render ( Context ( params ) ) <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : tsdb_space_quota_total , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> tsdb_space_quota_total_chart = loader . get_template ( '<STR_LIT>' ) . render ( Context ( params ) ) <EOL> tsdb_quota_total_query = [ metric_helper . make_quota_query ( cluster . name , '<STR_LIT>' , '<STR_LIT>' ) ] <EOL> tsdb_space_quota_total_query = [ metric_helper . make_quota_query ( cluster . name , <EOL> '<STR_LIT>' , '<STR_LIT>' ) ] <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : tsdb_quota_total_chart , <EOL> '<STR_LIT>' : tsdb_space_quota_total_chart , <EOL> '<STR_LIT>' : tsdb_quota_total_query , <EOL> '<STR_LIT>' : tsdb_space_quota_total_query , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def is_system_table ( table ) : <EOL> system_table_names = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> return table . name in system_table_names <EOL> def show_cluster_basic_board ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> if cluster . service . name != '<STR_LIT>' : <EOL> return HttpResponse ( '<STR_LIT>' ) <EOL> basic_info = dbutil . get_hbase_basic_info ( cluster ) <EOL> hdfs_cluster = dbutil . get_hdfs_cluster_by_name ( cluster . name ) <EOL> group = '<STR_LIT>' <EOL> tsdb_read_query = [ metric_helper . make_metric_query ( cluster . name , group , '<STR_LIT>' ) ] <EOL> tsdb_write_query = [ metric_helper . make_metric_query ( cluster . name , group , '<STR_LIT>' ) ] <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : hdfs_cluster , <EOL> '<STR_LIT>' : basic_info , <EOL> '<STR_LIT>' : tsdb_read_query , <EOL> '<STR_LIT>' : tsdb_write_query , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_cluster_regionserver_board ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> if cluster . service . name != '<STR_LIT>' : <EOL> return HttpResponse ( '<STR_LIT>' ) <EOL> read_requests_dist_by_rs , write_requests_dist_by_rs = dbutil . get_requests_distribution_groupby ( cluster , '<STR_LIT>' ) ; <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : read_requests_dist_by_rs , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> read_requests_dist_by_rs_chart = loader . get_template ( '<STR_LIT>' ) . render ( Context ( params ) ) <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : write_requests_dist_by_rs , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> write_requests_dist_by_rs_chart = loader . get_template ( '<STR_LIT>' ) . render ( Context ( params ) ) <EOL> regionservers = dbutil . get_items_on_cluster ( cluster , '<STR_LIT>' , order_by = '<STR_LIT:name>' ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : read_requests_dist_by_rs_chart , <EOL> '<STR_LIT>' : write_requests_dist_by_rs_chart , <EOL> '<STR_LIT>' : regionservers , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_cluster_replication ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> region_servers = dbutil . get_regionservers_with_active_replication_metrics_by_cluster ( cluster ) <EOL> ( peer_id_endpoint_map , peer_id_cluster_map ) = metric_helper . get_peer_id_endpoint_map_and_cluster ( region_servers ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : metric_helper . make_metrics_query_for_replication ( peer_id_endpoint_map , peer_id_cluster_map ) , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_cluster_storm_builtin_metrics ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> storm_tasks = dbutil . get_storm_task_by_cluster ( cluster ) <EOL> type = request . GET . get ( '<STR_LIT:type>' ) <EOL> type_dict = { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> } <EOL> storm_metrics = [ ] ; <EOL> for storm_task in storm_tasks : <EOL> if storm_task . job . name != '<STR_LIT>' : <EOL> continue <EOL> try : <EOL> json_metrics = json . loads ( storm_task . last_metrics_raw ) <EOL> except : <EOL> logger . warning ( "<STR_LIT>" , storm_task ) <EOL> return HttpResponse ( '<STR_LIT>' ) <EOL> for storm_id , topology_metrics in json_metrics . iteritems ( ) : <EOL> element = { "<STR_LIT>" : storm_id } <EOL> for group_name , group_metrics in topology_metrics . iteritems ( ) : <EOL> if group_name == type_dict . get ( type ) : <EOL> for metrics_name , metrics in group_metrics . iteritems ( ) : <EOL> metrics_name = metrics_name . lstrip ( '<STR_LIT:_>' ) <EOL> metrics_name = metrics_name . replace ( '<STR_LIT:->' , '<STR_LIT:_>' ) <EOL> element [ metrics_name ] = metrics <EOL> storm_metrics . append ( element ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : storm_metrics , <EOL> } <EOL> if type == "<STR_LIT>" : <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> elif type == "<STR_LIT>" : <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> else : <EOL> return HttpResponse ( '<STR_LIT>' + type ) <EOL> def show_cluster_storm_system_metrics ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> storm_tasks = dbutil . get_storm_task_by_cluster ( cluster ) <EOL> storm_metrics = [ ] <EOL> for storm_task in storm_tasks : <EOL> try : <EOL> json_metrics = json . loads ( storm_task . last_metrics_raw ) <EOL> except : <EOL> logger . warning ( "<STR_LIT>" , storm_task . last_metrics_raw ) <EOL> return HttpResponse ( '<STR_LIT>' ) <EOL> for storm_id , topology_metrics in json_metrics . iteritems ( ) : <EOL> topology_element = [ ] <EOL> for group_name , group_metrics in topology_metrics . iteritems ( ) : <EOL> if group_name . find ( "<STR_LIT>" ) != <NUM_LIT:0> : <EOL> continue <EOL> group_name = group_name . lstrip ( "<STR_LIT>" ) <EOL> element = { "<STR_LIT>" : group_name } ; <EOL> gc_value = "<STR_LIT>" <EOL> memory_heap_value = "<STR_LIT>" <EOL> memory_non_heap_value = "<STR_LIT>" <EOL> for metrics_name , metrics in group_metrics . iteritems ( ) : <EOL> if metrics_name . find ( "<STR_LIT>" ) == <NUM_LIT:0> : <EOL> if len ( gc_value ) != <NUM_LIT:0> : <EOL> gc_value += "<STR_LIT>" <EOL> gc_value += metrics_name . lstrip ( "<STR_LIT>" ) + "<STR_LIT::>" + str ( metrics ) <EOL> if metrics_name . find ( "<STR_LIT>" ) == <NUM_LIT:0> : <EOL> if len ( memory_heap_value ) != <NUM_LIT:0> : <EOL> memory_heap_value += "<STR_LIT>" <EOL> memory_heap_value += metrics_name . lstrip ( "<STR_LIT>" ) + "<STR_LIT::>" + str ( metrics ) <EOL> if metrics_name . find ( "<STR_LIT>" ) == <NUM_LIT:0> : <EOL> if len ( memory_non_heap_value ) != <NUM_LIT:0> : <EOL> memory_non_heap_value += "<STR_LIT>" <EOL> memory_non_heap_value += metrics_name . lstrip ( "<STR_LIT>" ) + "<STR_LIT::>" + str ( metrics ) <EOL> if metrics_name == "<STR_LIT>" : <EOL> element [ "<STR_LIT>" ] = metrics <EOL> if metrics_name == "<STR_LIT>" : <EOL> element [ "<STR_LIT>" ] = metrics <EOL> element [ "<STR_LIT>" ] = gc_value <EOL> element [ "<STR_LIT>" ] = memory_heap_value <EOL> element [ "<STR_LIT>" ] = memory_non_heap_value <EOL> topology_element . append ( element ) <EOL> metrics = { <EOL> "<STR_LIT>" : storm_id , <EOL> "<STR_LIT>" : topology_element <EOL> } <EOL> storm_metrics . append ( metrics ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : storm_metrics , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_cluster_storm_user_metrics ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> storm_tasks = dbutil . get_storm_task_by_cluster ( cluster ) <EOL> storm_metrics = { } <EOL> for storm_task in storm_tasks : <EOL> if storm_task . job . name != '<STR_LIT>' : <EOL> continue <EOL> try : <EOL> json_metrics = json . loads ( storm_task . last_metrics_raw ) <EOL> except : <EOL> logger . warning ( "<STR_LIT>" , storm_task ) <EOL> return HttpResponse ( '<STR_LIT>' ) <EOL> for storm_id , topology_metrics in json_metrics . iteritems ( ) : <EOL> topology_metrics_dict = storm_metrics . setdefault ( storm_id , { } ) <EOL> for group_name , group_metrics in topology_metrics . iteritems ( ) : <EOL> if group_name . find ( "<STR_LIT>" ) == <NUM_LIT:0> or group_name == "<STR_LIT>" or group_name == "<STR_LIT>" : <EOL> continue <EOL> group_component_id = group_name . split ( "<STR_LIT::>" ) [ <NUM_LIT:0> ] <EOL> group_task_id = group_name . split ( "<STR_LIT::>" ) [ <NUM_LIT:1> ] <EOL> group_metrics_dict = topology_metrics_dict . setdefault ( group_component_id , { } ) <EOL> task_metrics_dict = group_metrics_dict . setdefault ( group_task_id , { } ) ; <EOL> for metrics_name , metrics in group_metrics . iteritems ( ) : <EOL> task_metrics_dict [ metrics_name ] = metrics <EOL> format_storm_metrics = { } <EOL> for storm_id in storm_metrics : <EOL> topology_metrics = storm_metrics . get ( storm_id ) <EOL> format_topology_metrics = format_storm_metrics . setdefault ( storm_id , { } ) <EOL> for component_id in topology_metrics : <EOL> group_metrics = topology_metrics . get ( component_id ) <EOL> format_group_metrics = format_topology_metrics . setdefault ( component_id , [ ] ) <EOL> for task_id in group_metrics : <EOL> metrics = group_metrics . get ( task_id ) <EOL> key_set , value_set = add_key_set_for_format_group_metrics ( format_group_metrics , metrics . keys ( ) ) <EOL> format_metrics_list = [ task_id ] <EOL> for key in key_set : <EOL> if key == "<STR_LIT>" : <EOL> continue <EOL> format_metrics_list . append ( metrics . get ( key , "<STR_LIT:U+0020>" ) ) <EOL> value_set . append ( format_metrics_list ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : format_storm_metrics , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def add_key_set_for_format_group_metrics ( format_group_metrics , task_key_set ) : <EOL> key_set = task_key_set [ : ] <EOL> key_set . sort ( ) <EOL> key_set . insert ( <NUM_LIT:0> , "<STR_LIT>" ) <EOL> for group_metrics in format_group_metrics : <EOL> if cmp ( key_set , group_metrics [ "<STR_LIT>" ] ) == <NUM_LIT:0> : <EOL> return ( key_set , group_metrics [ "<STR_LIT>" ] ) <EOL> new_group_metrics = { <EOL> "<STR_LIT>" : key_set , <EOL> "<STR_LIT>" : [ ] , <EOL> } <EOL> format_group_metrics . append ( new_group_metrics ) <EOL> return ( new_group_metrics [ "<STR_LIT>" ] , new_group_metrics [ "<STR_LIT>" ] ) <EOL> def show_storm_topology ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> storm_id = request . GET . get ( '<STR_LIT>' ) <EOL> spout_keys = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , ] <EOL> bolt_keys = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , ] <EOL> storm_metrics = { "<STR_LIT>" : storm_id } <EOL> storm_graphs = [ ] <EOL> for key in spout_keys : <EOL> title = storm_id + "<STR_LIT>" + key <EOL> query = ( "<STR_LIT>" % ( key , dbutil . format_storm_name ( storm_id ) ) ) <EOL> graph = { <EOL> "<STR_LIT:title>" : title , <EOL> "<STR_LIT>" : query , <EOL> } <EOL> storm_graphs . append ( graph ) <EOL> for key in bolt_keys : <EOL> title = storm_id + "<STR_LIT>" + key <EOL> query = ( "<STR_LIT>" % ( key , dbutil . format_storm_name ( storm_id ) ) ) <EOL> graph = { <EOL> "<STR_LIT:title>" : title , <EOL> "<STR_LIT>" : query , <EOL> } <EOL> storm_graphs . append ( graph ) <EOL> storm_metrics [ "<STR_LIT>" ] = storm_graphs <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : storm_metrics , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def is_test_table ( table ) : <EOL> if '<STR_LIT>' in table . cluster . name : <EOL> return True <EOL> if '<STR_LIT:test>' in table . cluster . name : <EOL> return True <EOL> if '<STR_LIT>' in table . name : <EOL> return True <EOL> if '<STR_LIT:test>' in table . name : <EOL> return True <EOL> return False <EOL> def show_all_tables ( request ) : <EOL> tables = dbutil . get_all_tables ( ) <EOL> tables = [ table for table in tables if not is_system_table ( table ) ] <EOL> tables = [ table for table in tables if not is_test_table ( table ) ] <EOL> params = { <EOL> '<STR_LIT>' : tables , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_table ( request , id ) : <EOL> table = dbutil . get_table ( id ) <EOL> cluster = table . cluster <EOL> read_requests_dist_by_rs , write_requests_dist_by_rs = dbutil . get_requests_distribution ( table ) <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : read_requests_dist_by_rs , <EOL> } <EOL> read_requests_dist_by_rs_chart = loader . get_template ( '<STR_LIT>' ) . render ( Context ( params ) ) <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : write_requests_dist_by_rs , <EOL> } <EOL> write_requests_dist_by_rs_chart = loader . get_template ( '<STR_LIT>' ) . render ( <EOL> Context ( params ) ) <EOL> memstore_size_dist_by_region , storefile_size_dist_by_region = dbutil . get_data_distribution ( table ) <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : memstore_size_dist_by_region , <EOL> } <EOL> memstore_size_dist_by_region_chart = loader . get_template ( '<STR_LIT>' ) . render ( Context ( params ) ) <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : storefile_size_dist_by_region , <EOL> } <EOL> storefile_size_dist_by_region_chart = loader . get_template ( '<STR_LIT>' ) . render ( Context ( params ) ) <EOL> group = str ( table ) <EOL> tsdb_read_query = [ metric_helper . make_metric_query ( cluster . name , group , '<STR_LIT>' ) ] <EOL> tsdb_write_query = [ metric_helper . make_metric_query ( cluster . name , group , '<STR_LIT>' ) ] <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : table , <EOL> '<STR_LIT>' : read_requests_dist_by_rs_chart , <EOL> '<STR_LIT>' : write_requests_dist_by_rs_chart , <EOL> '<STR_LIT>' : memstore_size_dist_by_region_chart , <EOL> '<STR_LIT>' : storefile_size_dist_by_region_chart , <EOL> '<STR_LIT>' : tsdb_read_query , <EOL> '<STR_LIT>' : tsdb_write_query , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_table_operation ( request , id ) : <EOL> table = dbutil . get_table ( id ) <EOL> cluster = table . cluster <EOL> endpoint = dbutil . map_cluster_to_endpoint ( cluster . name ) <EOL> group = str ( table ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : table , <EOL> '<STR_LIT>' : metric_helper . make_operation_metrics ( endpoint , table , group ) , <EOL> '<STR_LIT>' : endpoint <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_table_count_rows ( request ) : <EOL> tables_to_count = Table . objects . filter ( is_count_rows = True ) <EOL> tables_not_to_count = Table . objects . filter ( is_count_rows = False ) <EOL> params = { <EOL> '<STR_LIT>' : settings . COUNT_PERIOD , <EOL> '<STR_LIT>' : settings . COUNT_START_HOUR , <EOL> '<STR_LIT>' : settings . COUNT_END_HOUR , <EOL> '<STR_LIT>' : tables_to_count , <EOL> '<STR_LIT>' : tables_not_to_count <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def add_table_count_rows ( request , id ) : <EOL> table = dbutil . get_table ( id ) <EOL> table . is_count_rows = True <EOL> table . save ( ) <EOL> return HttpResponse ( ) <EOL> def cancel_table_count_rows ( request , id ) : <EOL> table = dbutil . get_table ( id ) <EOL> table . is_count_rows = False <EOL> table . save ( ) <EOL> return HttpResponse ( ) <EOL> def show_regionserver_operation ( request , id ) : <EOL> regionserver = dbutil . get_regionserver ( id ) <EOL> cluster = regionserver . cluster <EOL> endpoint = dbutil . map_cluster_to_endpoint ( cluster . name ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : regionserver , <EOL> '<STR_LIT>' : metric_helper . generate_operation_metric_for_regionserver ( regionserver ) , <EOL> '<STR_LIT>' : endpoint <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_cluster_operation ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> endpoint = dbutil . map_cluster_to_endpoint ( cluster . name ) <EOL> group = '<STR_LIT>' <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : metric_helper . make_operation_metrics ( endpoint , cluster . hbasecluster , group ) , <EOL> '<STR_LIT>' : endpoint <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_cluster_operation_table_comparison ( request , id ) : <EOL> cluster = dbutil . get_cluster ( id ) <EOL> endpoint = dbutil . map_cluster_to_endpoint ( cluster . name ) <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : metric_helper . make_operation_metrics_for_tables_in_cluster ( cluster ) , <EOL> '<STR_LIT>' : endpoint <EOL> } <EOL> print params [ '<STR_LIT>' ] <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_regionserver ( request , id ) : <EOL> rs = dbutil . get_regionserver ( id ) <EOL> cluster = rs . cluster <EOL> read_requests_dist_by_rs , write_requests_dist_by_rs = dbutil . get_requests_distribution ( rs ) ; <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : read_requests_dist_by_rs , <EOL> } <EOL> read_requests_dist_by_rs_chart = loader . get_template ( '<STR_LIT>' ) . render ( Context ( params ) ) <EOL> params = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : write_requests_dist_by_rs , <EOL> } <EOL> write_requests_dist_by_rs_chart = loader . get_template ( '<STR_LIT>' ) . render ( <EOL> Context ( params ) ) <EOL> group = str ( rs ) <EOL> tsdb_read_query = [ metric_helper . make_metric_query ( cluster . name , group , '<STR_LIT>' ) ] <EOL> tsdb_write_query = [ metric_helper . make_metric_query ( cluster . name , group , '<STR_LIT>' ) ] <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : rs , <EOL> '<STR_LIT>' : read_requests_dist_by_rs_chart , <EOL> '<STR_LIT>' : write_requests_dist_by_rs_chart , <EOL> '<STR_LIT>' : tsdb_read_query , <EOL> '<STR_LIT>' : tsdb_write_query , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_user_quota ( request , id ) : <EOL> quota = dbutil . get_quota ( id ) <EOL> cluster = quota . cluster <EOL> used_quota_query = [ metric_helper . make_quota_query ( cluster . name , quota . name , '<STR_LIT>' ) ] <EOL> used_space_quota_query = [ metric_helper . make_quota_query ( cluster . name , quota . name , '<STR_LIT>' ) ] <EOL> params = { <EOL> '<STR_LIT>' : cluster , <EOL> '<STR_LIT>' : used_quota_query , <EOL> '<STR_LIT>' : used_space_quota_query , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_job ( request , id ) : <EOL> tasks = dbutil . get_healthy_tasks_by_job ( id ) <EOL> job = dbutil . get_job ( id ) <EOL> endpoints = [ metric_helper . form_perf_counter_endpoint_name ( task ) for task in tasks ] <EOL> tsdb_metrics = metric_helper . make_metrics_query_for_job ( endpoints , job , tasks ) <EOL> print tsdb_metrics <EOL> params = { <EOL> '<STR_LIT>' : job , <EOL> '<STR_LIT>' : tasks , <EOL> '<STR_LIT>' : tsdb_metrics , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_task ( request , id ) : <EOL> task = dbutil . get_task ( id ) <EOL> job = task . job <EOL> tasks = dbutil . get_tasks_by_job ( job ) <EOL> tsdb_metrics = metric_helper . make_metrics_query_for_task ( <EOL> metric_helper . form_perf_counter_endpoint_name ( task ) , <EOL> task ) <EOL> params = { <EOL> '<STR_LIT>' : job , <EOL> '<STR_LIT>' : task , <EOL> '<STR_LIT>' : tasks , <EOL> '<STR_LIT>' : tsdb_metrics , <EOL> } <EOL> return respond ( request , '<STR_LIT>' , params ) <EOL> def show_all_metrics ( request ) : <EOL> result = { } <EOL> metrics = dbutil . get_all_metrics ( ) <EOL> if not metrics : <EOL> return HttpResponse ( '<STR_LIT>' , content_type = '<STR_LIT>' ) <EOL> result [ '<STR_LIT>' ] = int ( time . time ( ) ) <EOL> result [ '<STR_LIT:data>' ] = metrics <EOL> indent = None <EOL> if '<STR_LIT>' in request . GET : <EOL> indent = <NUM_LIT:1> <EOL> return HttpResponse ( json . dumps ( result , indent = indent ) , <EOL> content_type = '<STR_LIT>' ) <EOL> def show_all_metrics_config ( request ) : <EOL> metrics_config = metric_helper . get_all_metrics_config ( ) <EOL> indent = None <EOL> if '<STR_LIT>' in request . GET : <EOL> indent = <NUM_LIT:1> <EOL> return HttpResponse ( json . dumps ( metrics_config , indent = indent ) , <EOL> content_type = '<STR_LIT>' ) <EOL> def get_time_range ( request ) : <EOL> start_time = datetime . datetime . today ( ) + datetime . timedelta ( hours = - <NUM_LIT:1> ) <EOL> end_time = datetime . datetime . today ( ) <EOL> if '<STR_LIT>' in request . COOKIES : <EOL> start_time = datetime . datetime . strptime ( request . COOKIES [ '<STR_LIT>' ] , '<STR_LIT>' ) <EOL> if '<STR_LIT>' in request . GET : <EOL> start_time = datetime . datetime . strptime ( request . GET [ '<STR_LIT>' ] , '<STR_LIT>' ) <EOL> if '<STR_LIT>' in request . COOKIES : <EOL> end_time = datetime . datetime . strptime ( request . COOKIES [ '<STR_LIT>' ] , '<STR_LIT>' ) <EOL> if '<STR_LIT>' in request . GET : <EOL> end_time = datetime . datetime . strptime ( request . GET [ '<STR_LIT>' ] , '<STR_LIT>' ) <EOL> return start_time , end_time <EOL> @ transaction . commit_on_success <EOL> @ csrf_exempt <EOL> @ require_http_methods ( [ "<STR_LIT:POST>" ] ) <EOL> def add_counter ( request ) : <EOL> counters = json . loads ( request . body ) <EOL> remote_ip = request . META [ '<STR_LIT>' ] <EOL> update_time = datetime . datetime . utcfromtimestamp ( time . time ( ) ) . replace ( tzinfo = timezone . utc ) <EOL> for dict in counters : <EOL> group = dict [ '<STR_LIT>' ] <EOL> endpoint = remote_ip <EOL> if '<STR_LIT>' in dict : <EOL> endpoint = dict [ '<STR_LIT>' ] <EOL> label = '<STR_LIT>' <EOL> if '<STR_LIT:label>' in dict : <EOL> label = dict [ '<STR_LIT:label>' ] <EOL> name = dict [ '<STR_LIT:name>' ] <EOL> counter , create = dbutil . get_or_create_counter ( group , name ) <EOL> counter . host = endpoint <EOL> counter . value = ( float ) ( dict [ '<STR_LIT:value>' ] ) <EOL> counter . unit = dict [ '<STR_LIT>' ] <EOL> counter . last_update_time = update_time <EOL> counter . label = label <EOL> counter . save ( ) <EOL> return HttpResponse ( "<STR_LIT>" ) <EOL> def show_all_counters ( request ) : <EOL> result = { } <EOL> metrics = dbutil . get_all_counters ( ) <EOL> if not metrics : <EOL> return HttpResponse ( '<STR_LIT>' , content_type = '<STR_LIT>' ) <EOL> result [ '<STR_LIT>' ] = time . time ( ) <EOL> result [ '<STR_LIT:data>' ] = metrics <EOL> indent = None <EOL> if '<STR_LIT>' in request . GET : <EOL> indent = <NUM_LIT:1> <EOL> return HttpResponse ( json . dumps ( result , indent = indent ) , <EOL> content_type = '<STR_LIT>' ) <EOL> def respond ( request , template , params = None ) : <EOL> """<STR_LIT>""" <EOL> params [ '<STR_LIT>' ] = request <EOL> params [ '<STR_LIT:user>' ] = request . user <EOL> params [ '<STR_LIT>' ] = owl_config . CHART_URL_PREFIX <EOL> params [ '<STR_LIT>' ] = owl_config . TSDB_ADDR <EOL> params [ '<STR_LIT>' ] = owl_config . SUPERVISOR_PORT <EOL> params [ '<STR_LIT>' ] = ( datetime . datetime . now ( ) - datetime . timedelta ( minutes = <NUM_LIT:15> ) ) . strftime ( '<STR_LIT>' ) <EOL> params [ '<STR_LIT>' ] = ( datetime . datetime . now ( ) - datetime . timedelta ( hours = <NUM_LIT:20> ) ) . strftime ( '<STR_LIT>' ) <EOL> params . update ( request . GET ) <EOL> response = render_to_response ( template , params , <EOL> context_instance = RequestContext ( request ) ) <EOL> return response </s>
