<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> from __future__ import division <EOL> from __future__ import print_function <EOL> from __future__ import unicode_literals <EOL> import functools <EOL> import json <EOL> import logging <EOL> import textwrap <EOL> from collections import namedtuple <EOL> from copy import deepcopy , copy <EOL> from datetime import timedelta , datetime , date <EOL> import humanize <EOL> import pandas as pd <EOL> import requests <EOL> import sqlalchemy as sqla <EOL> import sqlparse <EOL> from dateutil . parser import parse <EOL> from flask import request , g <EOL> from flask . ext . appbuilder import Model <EOL> from flask . ext . appbuilder . models . mixins import AuditMixin <EOL> from pydruid . client import PyDruid <EOL> from flask . ext . appbuilder . models . decorators import renders <EOL> from pydruid . utils . filters import Dimension , Filter <EOL> from six import string_types <EOL> from sqlalchemy import ( <EOL> Column , Integer , String , ForeignKey , Text , Boolean , DateTime , Date , <EOL> Table , create_engine , MetaData , desc , select , and_ , func ) <EOL> from sqlalchemy . engine import reflection <EOL> from sqlalchemy . ext . declarative import declared_attr <EOL> from sqlalchemy . orm import relationship <EOL> from sqlalchemy . sql import table , literal_column , text , column <EOL> from sqlalchemy . sql . elements import ColumnClause <EOL> from sqlalchemy_utils import EncryptedType <EOL> from caravel import app , db , get_session , utils <EOL> from caravel . viz import viz_types <EOL> from caravel . utils import flasher <EOL> config = app . config <EOL> QueryResult = namedtuple ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> class AuditMixinNullable ( AuditMixin ) : <EOL> """<STR_LIT>""" <EOL> created_on = Column ( DateTime , default = datetime . now , nullable = True ) <EOL> changed_on = Column ( <EOL> DateTime , default = datetime . now , <EOL> onupdate = datetime . now , nullable = True ) <EOL> @ declared_attr <EOL> def created_by_fk ( cls ) : <EOL> return Column ( Integer , ForeignKey ( '<STR_LIT>' ) , <EOL> default = cls . get_user_id , nullable = True ) <EOL> @ declared_attr <EOL> def changed_by_fk ( cls ) : <EOL> return Column ( <EOL> Integer , ForeignKey ( '<STR_LIT>' ) , <EOL> default = cls . get_user_id , onupdate = cls . get_user_id , nullable = True ) <EOL> @ renders ( '<STR_LIT>' ) <EOL> def creator ( self ) : <EOL> return '<STR_LIT:{}>' . format ( self . created_by or '<STR_LIT>' ) <EOL> @ renders ( '<STR_LIT>' ) <EOL> def changed_by_ ( self ) : <EOL> return '<STR_LIT:{}>' . format ( self . changed_by or '<STR_LIT>' ) <EOL> @ renders ( '<STR_LIT>' ) <EOL> def modified ( self ) : <EOL> s = humanize . naturaltime ( datetime . now ( ) - self . changed_on ) <EOL> return '<STR_LIT>' . format ( s ) <EOL> @ property <EOL> def icons ( self ) : <EOL> return """<STR_LIT>""" . format ( ** locals ( ) ) <EOL> class Url ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT:url>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> url = Column ( Text ) <EOL> class CssTemplate ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> template_name = Column ( String ( <NUM_LIT> ) ) <EOL> css = Column ( Text , default = '<STR_LIT>' ) <EOL> slice_user = Table ( '<STR_LIT>' , Model . metadata , <EOL> Column ( '<STR_LIT:id>' , Integer , primary_key = True ) , <EOL> Column ( '<STR_LIT>' , Integer , ForeignKey ( '<STR_LIT>' ) ) , <EOL> Column ( '<STR_LIT>' , Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> ) <EOL> class Slice ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> slice_name = Column ( String ( <NUM_LIT> ) ) <EOL> druid_datasource_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> table_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> datasource_type = Column ( String ( <NUM_LIT:200> ) ) <EOL> datasource_name = Column ( String ( <NUM_LIT> ) ) <EOL> viz_type = Column ( String ( <NUM_LIT> ) ) <EOL> params = Column ( Text ) <EOL> description = Column ( Text ) <EOL> cache_timeout = Column ( Integer ) <EOL> perm = Column ( String ( <NUM_LIT> ) ) <EOL> table = relationship ( <EOL> '<STR_LIT>' , foreign_keys = [ table_id ] , backref = '<STR_LIT>' ) <EOL> druid_datasource = relationship ( <EOL> '<STR_LIT>' , foreign_keys = [ druid_datasource_id ] , backref = '<STR_LIT>' ) <EOL> owners = relationship ( "<STR_LIT>" , secondary = slice_user ) <EOL> def __repr__ ( self ) : <EOL> return self . slice_name <EOL> @ property <EOL> def datasource ( self ) : <EOL> return self . table or self . druid_datasource <EOL> @ property <EOL> def datasource_link ( self ) : <EOL> if self . table : <EOL> return self . table . link <EOL> elif self . druid_datasource : <EOL> return self . druid_datasource . link <EOL> @ property <EOL> def datasource_edit_url ( self ) : <EOL> if self . table : <EOL> return self . table . url <EOL> elif self . druid_datasource : <EOL> return self . druid_datasource . url <EOL> @ property <EOL> @ utils . memoized <EOL> def viz ( self ) : <EOL> d = json . loads ( self . params ) <EOL> viz_class = viz_types [ self . viz_type ] <EOL> return viz_class ( self . datasource , form_data = d ) <EOL> @ property <EOL> def description_markeddown ( self ) : <EOL> return utils . markdown ( self . description ) <EOL> @ property <EOL> def datasource_id ( self ) : <EOL> return self . table_id or self . druid_datasource_id <EOL> @ property <EOL> def data ( self ) : <EOL> d = { } <EOL> self . token = '<STR_LIT>' <EOL> try : <EOL> d = self . viz . data <EOL> self . token = d . get ( '<STR_LIT>' ) <EOL> except Exception as e : <EOL> d [ '<STR_LIT:error>' ] = str ( e ) <EOL> d [ '<STR_LIT>' ] = self . id <EOL> return d <EOL> @ property <EOL> def json_data ( self ) : <EOL> return json . dumps ( self . data ) <EOL> @ property <EOL> def slice_url ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> slice_params = json . loads ( self . params ) <EOL> except Exception as e : <EOL> logging . exception ( e ) <EOL> slice_params = { } <EOL> slice_params [ '<STR_LIT>' ] = self . id <EOL> slice_params [ '<STR_LIT>' ] = "<STR_LIT:false>" <EOL> slice_params [ '<STR_LIT>' ] = self . slice_name <EOL> from werkzeug . urls import Href <EOL> href = Href ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" . format ( obj = self ) ) <EOL> return href ( slice_params ) <EOL> @ property <EOL> def edit_url ( self ) : <EOL> return "<STR_LIT>" . format ( self . id ) <EOL> @ property <EOL> def slice_link ( self ) : <EOL> url = self . slice_url <EOL> return '<STR_LIT>' . format ( <EOL> url = url , obj = self ) <EOL> def set_perm ( mapper , connection , target ) : <EOL> if target . table_id : <EOL> src_class = SqlaTable <EOL> id_ = target . table_id <EOL> elif target . druid_datasource_id : <EOL> src_class = DruidDatasource <EOL> id_ = target . druid_datasource_id <EOL> ds = db . session . query ( src_class ) . filter_by ( id = int ( id_ ) ) . first ( ) <EOL> target . perm = ds . perm <EOL> sqla . event . listen ( Slice , '<STR_LIT>' , set_perm ) <EOL> sqla . event . listen ( Slice , '<STR_LIT>' , set_perm ) <EOL> dashboard_slices = Table ( <EOL> '<STR_LIT>' , Model . metadata , <EOL> Column ( '<STR_LIT:id>' , Integer , primary_key = True ) , <EOL> Column ( '<STR_LIT>' , Integer , ForeignKey ( '<STR_LIT>' ) ) , <EOL> Column ( '<STR_LIT>' , Integer , ForeignKey ( '<STR_LIT>' ) ) , <EOL> ) <EOL> dashboard_user = Table ( <EOL> '<STR_LIT>' , Model . metadata , <EOL> Column ( '<STR_LIT:id>' , Integer , primary_key = True ) , <EOL> Column ( '<STR_LIT>' , Integer , ForeignKey ( '<STR_LIT>' ) ) , <EOL> Column ( '<STR_LIT>' , Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> ) <EOL> class Dashboard ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> dashboard_title = Column ( String ( <NUM_LIT> ) ) <EOL> position_json = Column ( Text ) <EOL> description = Column ( Text ) <EOL> css = Column ( Text ) <EOL> json_metadata = Column ( Text ) <EOL> slug = Column ( String ( <NUM_LIT:255> ) , unique = True ) <EOL> slices = relationship ( <EOL> '<STR_LIT>' , secondary = dashboard_slices , backref = '<STR_LIT>' ) <EOL> owners = relationship ( "<STR_LIT>" , secondary = dashboard_user ) <EOL> def __repr__ ( self ) : <EOL> return self . dashboard_title <EOL> @ property <EOL> def url ( self ) : <EOL> return "<STR_LIT>" . format ( self . slug or self . id ) <EOL> @ property <EOL> def metadata_dejson ( self ) : <EOL> if self . json_metadata : <EOL> return json . loads ( self . json_metadata ) <EOL> else : <EOL> return { } <EOL> def dashboard_link ( self ) : <EOL> return '<STR_LIT>' . format ( obj = self ) <EOL> @ property <EOL> def json_data ( self ) : <EOL> d = { <EOL> '<STR_LIT:id>' : self . id , <EOL> '<STR_LIT>' : self . metadata_dejson , <EOL> '<STR_LIT>' : self . dashboard_title , <EOL> '<STR_LIT>' : self . slug , <EOL> '<STR_LIT>' : [ slc . data for slc in self . slices ] , <EOL> } <EOL> return json . dumps ( d ) <EOL> class Queryable ( object ) : <EOL> """<STR_LIT>""" <EOL> @ property <EOL> def column_names ( self ) : <EOL> return sorted ( [ c . column_name for c in self . columns ] ) <EOL> @ property <EOL> def main_dttm_col ( self ) : <EOL> return "<STR_LIT>" <EOL> @ property <EOL> def groupby_column_names ( self ) : <EOL> return sorted ( [ c . column_name for c in self . columns if c . groupby ] ) <EOL> @ property <EOL> def filterable_column_names ( self ) : <EOL> return sorted ( [ c . column_name for c in self . columns if c . filterable ] ) <EOL> @ property <EOL> def dttm_cols ( self ) : <EOL> return [ ] <EOL> class Database ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> database_name = Column ( String ( <NUM_LIT> ) , unique = True ) <EOL> sqlalchemy_uri = Column ( String ( <NUM_LIT> ) ) <EOL> password = Column ( EncryptedType ( String ( <NUM_LIT> ) , config . get ( '<STR_LIT>' ) ) ) <EOL> cache_timeout = Column ( Integer ) <EOL> extra = Column ( Text , default = textwrap . dedent ( """<STR_LIT>""" ) ) <EOL> def __repr__ ( self ) : <EOL> return self . database_name <EOL> def get_sqla_engine ( self ) : <EOL> extra = self . get_extra ( ) <EOL> params = extra . get ( '<STR_LIT>' , { } ) <EOL> return create_engine ( self . sqlalchemy_uri_decrypted , ** params ) <EOL> def safe_sqlalchemy_uri ( self ) : <EOL> return self . sqlalchemy_uri <EOL> def grains ( self ) : <EOL> """<STR_LIT>""" <EOL> Grain = namedtuple ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> db_time_grains = { <EOL> '<STR_LIT>' : ( <EOL> Grain ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> Grain ( '<STR_LIT>' , "<STR_LIT>" ) , <EOL> Grain ( '<STR_LIT>' , "<STR_LIT>" ) , <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> ) , <EOL> '<STR_LIT>' : ( <EOL> Grain ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> Grain ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> ) , <EOL> '<STR_LIT>' : ( <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> Grain ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ) , <EOL> } <EOL> db_time_grains [ '<STR_LIT>' ] = db_time_grains [ '<STR_LIT>' ] <EOL> for db_type , grains in db_time_grains . items ( ) : <EOL> if self . sqlalchemy_uri . startswith ( db_type ) : <EOL> return grains <EOL> def grains_dict ( self ) : <EOL> return { grain . name : grain for grain in self . grains ( ) } <EOL> def get_extra ( self ) : <EOL> extra = { } <EOL> if self . extra : <EOL> try : <EOL> extra = json . loads ( self . extra ) <EOL> except Exception as e : <EOL> logging . error ( e ) <EOL> return extra <EOL> def get_table ( self , table_name , schema = None ) : <EOL> extra = self . get_extra ( ) <EOL> meta = MetaData ( ** extra . get ( '<STR_LIT>' , { } ) ) <EOL> return Table ( <EOL> table_name , meta , <EOL> schema = schema or None , <EOL> autoload = True , <EOL> autoload_with = self . get_sqla_engine ( ) ) <EOL> def get_columns ( self , table_name ) : <EOL> engine = self . get_sqla_engine ( ) <EOL> insp = reflection . Inspector . from_engine ( engine ) <EOL> return insp . get_columns ( table_name ) <EOL> @ property <EOL> def sqlalchemy_uri_decrypted ( self ) : <EOL> conn = sqla . engine . url . make_url ( self . sqlalchemy_uri ) <EOL> conn . password = self . password <EOL> return str ( conn ) <EOL> @ property <EOL> def sql_url ( self ) : <EOL> return '<STR_LIT>' . format ( self . id ) <EOL> @ property <EOL> def sql_link ( self ) : <EOL> return '<STR_LIT>' . format ( self . sql_url ) <EOL> class SqlaTable ( Model , Queryable , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> type = "<STR_LIT>" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> table_name = Column ( String ( <NUM_LIT> ) ) <EOL> main_dttm_col = Column ( String ( <NUM_LIT> ) ) <EOL> description = Column ( Text ) <EOL> default_endpoint = Column ( Text ) <EOL> database_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) , nullable = False ) <EOL> is_featured = Column ( Boolean , default = False ) <EOL> user_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> owner = relationship ( '<STR_LIT>' , backref = '<STR_LIT>' , foreign_keys = [ user_id ] ) <EOL> database = relationship ( <EOL> '<STR_LIT>' , backref = '<STR_LIT>' , foreign_keys = [ database_id ] ) <EOL> offset = Column ( Integer , default = <NUM_LIT:0> ) <EOL> cache_timeout = Column ( Integer ) <EOL> schema = Column ( String ( <NUM_LIT> ) ) <EOL> baselink = "<STR_LIT>" <EOL> __table_args__ = ( <EOL> sqla . UniqueConstraint ( <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> name = '<STR_LIT>' ) , ) <EOL> def __repr__ ( self ) : <EOL> return self . table_name <EOL> @ property <EOL> def description_markeddown ( self ) : <EOL> return utils . markdown ( self . description ) <EOL> @ property <EOL> def url ( self ) : <EOL> return '<STR_LIT>' . format ( self . id ) <EOL> @ property <EOL> def link ( self ) : <EOL> return '<STR_LIT>' . format ( ** locals ( ) ) <EOL> @ property <EOL> def perm ( self ) : <EOL> return ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) . format ( obj = self ) <EOL> @ property <EOL> def full_name ( self ) : <EOL> return "<STR_LIT>" . format ( obj = self ) <EOL> @ property <EOL> def dttm_cols ( self ) : <EOL> l = [ c . column_name for c in self . columns if c . is_dttm ] <EOL> if self . main_dttm_col not in l : <EOL> l . append ( self . main_dttm_col ) <EOL> return l <EOL> @ property <EOL> def any_dttm_col ( self ) : <EOL> cols = self . dttm_cols <EOL> if cols : <EOL> return cols [ <NUM_LIT:0> ] <EOL> @ property <EOL> def html ( self ) : <EOL> t = ( ( c . column_name , c . type ) for c in self . columns ) <EOL> df = pd . DataFrame ( t ) <EOL> df . columns = [ '<STR_LIT>' , '<STR_LIT:type>' ] <EOL> return df . to_html ( <EOL> index = False , <EOL> classes = ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) ) <EOL> @ property <EOL> def name ( self ) : <EOL> return self . table_name <EOL> @ property <EOL> def explore_url ( self ) : <EOL> if self . default_endpoint : <EOL> return self . default_endpoint <EOL> else : <EOL> return "<STR_LIT>" . format ( obj = self ) <EOL> @ property <EOL> def table_link ( self ) : <EOL> return '<STR_LIT>' . format ( obj = self ) <EOL> @ property <EOL> def metrics_combo ( self ) : <EOL> return sorted ( <EOL> [ <EOL> ( m . metric_name , m . verbose_name or m . metric_name ) <EOL> for m in self . metrics ] , <EOL> key = lambda x : x [ <NUM_LIT:1> ] ) <EOL> @ property <EOL> def sql_url ( self ) : <EOL> return self . database . sql_url + "<STR_LIT>" + str ( self . table_name ) <EOL> @ property <EOL> def sql_link ( self ) : <EOL> return '<STR_LIT>' . format ( self . sql_url ) <EOL> def query ( <EOL> self , groupby , metrics , <EOL> granularity , <EOL> from_dttm , to_dttm , <EOL> filter = None , <EOL> is_timeseries = True , <EOL> timeseries_limit = <NUM_LIT:15> , row_limit = None , <EOL> inner_from_dttm = None , inner_to_dttm = None , <EOL> extras = None , <EOL> columns = None ) : <EOL> """<STR_LIT>""" <EOL> if granularity not in self . dttm_cols : <EOL> granularity = self . main_dttm_col <EOL> cols = { col . column_name : col for col in self . columns } <EOL> qry_start_dttm = datetime . now ( ) <EOL> if not granularity and is_timeseries : <EOL> raise Exception ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> metrics_exprs = [ <EOL> literal_column ( m . expression ) . label ( m . metric_name ) <EOL> for m in self . metrics if m . metric_name in metrics ] <EOL> if metrics : <EOL> main_metric_expr = literal_column ( [ <EOL> m . expression for m in self . metrics <EOL> if m . metric_name == metrics [ <NUM_LIT:0> ] ] [ <NUM_LIT:0> ] ) <EOL> else : <EOL> main_metric_expr = literal_column ( "<STR_LIT>" ) <EOL> select_exprs = [ ] <EOL> groupby_exprs = [ ] <EOL> if groupby : <EOL> select_exprs = [ ] <EOL> inner_select_exprs = [ ] <EOL> inner_groupby_exprs = [ ] <EOL> for s in groupby : <EOL> col = cols [ s ] <EOL> expr = col . expression <EOL> if expr : <EOL> outer = literal_column ( expr ) . label ( s ) <EOL> inner = literal_column ( expr ) . label ( '<STR_LIT>' + s ) <EOL> else : <EOL> outer = column ( s ) . label ( s ) <EOL> inner = column ( s ) . label ( '<STR_LIT>' + s ) <EOL> groupby_exprs . append ( outer ) <EOL> select_exprs . append ( outer ) <EOL> inner_groupby_exprs . append ( inner ) <EOL> inner_select_exprs . append ( inner ) <EOL> elif columns : <EOL> for s in columns : <EOL> select_exprs . append ( s ) <EOL> metrics_exprs = [ ] <EOL> if granularity : <EOL> dttm_expr = cols [ granularity ] . expression or granularity <EOL> timestamp = literal_column ( dttm_expr ) . label ( '<STR_LIT>' ) <EOL> time_grain_sqla = extras . get ( '<STR_LIT>' ) <EOL> if time_grain_sqla : <EOL> udf = self . database . grains_dict ( ) . get ( time_grain_sqla , '<STR_LIT>' ) <EOL> timestamp_grain = literal_column ( <EOL> udf . function . format ( col = dttm_expr ) ) . label ( '<STR_LIT>' ) <EOL> else : <EOL> timestamp_grain = timestamp <EOL> if is_timeseries : <EOL> select_exprs += [ timestamp_grain ] <EOL> groupby_exprs += [ timestamp_grain ] <EOL> tf = '<STR_LIT>' <EOL> time_filter = [ <EOL> timestamp >= from_dttm . strftime ( tf ) , <EOL> timestamp <= to_dttm . strftime ( tf ) , <EOL> ] <EOL> inner_time_filter = copy ( time_filter ) <EOL> if inner_from_dttm : <EOL> inner_time_filter [ <NUM_LIT:0> ] = timestamp >= inner_from_dttm . strftime ( tf ) <EOL> if inner_to_dttm : <EOL> inner_time_filter [ <NUM_LIT:1> ] = timestamp <= inner_to_dttm . strftime ( tf ) <EOL> else : <EOL> inner_time_filter = [ ] <EOL> select_exprs += metrics_exprs <EOL> qry = select ( select_exprs ) <EOL> tbl = table ( self . table_name ) <EOL> if self . schema : <EOL> tbl . schema = self . schema <EOL> if not columns : <EOL> qry = qry . group_by ( * groupby_exprs ) <EOL> where_clause_and = [ ] <EOL> having_clause_and = [ ] <EOL> for col , op , eq in filter : <EOL> col_obj = cols [ col ] <EOL> if op in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> values = eq . split ( "<STR_LIT:U+002C>" ) <EOL> if col_obj . expression : <EOL> cond = ColumnClause ( <EOL> col_obj . expression , is_literal = True ) . in_ ( values ) <EOL> else : <EOL> cond = column ( col ) . in_ ( values ) <EOL> if op == '<STR_LIT>' : <EOL> cond = ~ cond <EOL> where_clause_and . append ( cond ) <EOL> if extras and '<STR_LIT>' in extras : <EOL> where_clause_and += [ text ( extras [ '<STR_LIT>' ] ) ] <EOL> if extras and '<STR_LIT>' in extras : <EOL> having_clause_and += [ text ( extras [ '<STR_LIT>' ] ) ] <EOL> if granularity : <EOL> qry = qry . where ( and_ ( * ( time_filter + where_clause_and ) ) ) <EOL> else : <EOL> qry = qry . where ( and_ ( * where_clause_and ) ) <EOL> qry = qry . having ( and_ ( * having_clause_and ) ) <EOL> if groupby : <EOL> qry = qry . order_by ( desc ( main_metric_expr ) ) <EOL> qry = qry . limit ( row_limit ) <EOL> if timeseries_limit and groupby : <EOL> subq = select ( inner_select_exprs ) <EOL> subq = subq . select_from ( tbl ) <EOL> subq = subq . where ( and_ ( * ( where_clause_and + inner_time_filter ) ) ) <EOL> subq = subq . group_by ( * inner_groupby_exprs ) <EOL> subq = subq . order_by ( desc ( main_metric_expr ) ) <EOL> subq = subq . limit ( timeseries_limit ) <EOL> on_clause = [ ] <EOL> for i , gb in enumerate ( groupby ) : <EOL> on_clause . append ( <EOL> groupby_exprs [ i ] == column ( "<STR_LIT>" + gb ) ) <EOL> tbl = tbl . join ( subq . alias ( ) , and_ ( * on_clause ) ) <EOL> qry = qry . select_from ( tbl ) <EOL> engine = self . database . get_sqla_engine ( ) <EOL> sql = "<STR_LIT:{}>" . format ( <EOL> qry . compile ( engine , compile_kwargs = { "<STR_LIT>" : True } ) ) <EOL> df = pd . read_sql_query ( <EOL> sql = sql , <EOL> con = engine <EOL> ) <EOL> sql = sqlparse . format ( sql , reindent = True ) <EOL> return QueryResult ( <EOL> df = df , duration = datetime . now ( ) - qry_start_dttm , query = sql ) <EOL> def fetch_metadata ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> table = self . database . get_table ( self . table_name , schema = self . schema ) <EOL> except Exception as e : <EOL> flasher ( str ( e ) ) <EOL> flasher ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" , "<STR_LIT>" ) <EOL> return <EOL> TC = TableColumn <EOL> M = SqlMetric <EOL> metrics = [ ] <EOL> any_date_col = None <EOL> for col in table . columns : <EOL> try : <EOL> datatype = str ( col . type ) <EOL> except Exception as e : <EOL> datatype = "<STR_LIT>" <EOL> dbcol = ( <EOL> db . session <EOL> . query ( TC ) <EOL> . filter ( TC . table == self ) <EOL> . filter ( TC . column_name == col . name ) <EOL> . first ( ) <EOL> ) <EOL> db . session . flush ( ) <EOL> if not dbcol : <EOL> dbcol = TableColumn ( column_name = col . name ) <EOL> num_types = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> date_types = ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> str_types = ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> datatype = str ( datatype ) . upper ( ) <EOL> if any ( [ t in datatype for t in str_types ] ) : <EOL> dbcol . groupby = True <EOL> dbcol . filterable = True <EOL> elif any ( [ t in datatype for t in num_types ] ) : <EOL> dbcol . sum = True <EOL> elif any ( [ t in datatype for t in date_types ] ) : <EOL> dbcol . is_dttm = True <EOL> db . session . merge ( self ) <EOL> self . columns . append ( dbcol ) <EOL> if not any_date_col and '<STR_LIT:date>' in datatype . lower ( ) : <EOL> any_date_col = col . name <EOL> quoted = "<STR_LIT:{}>" . format ( <EOL> column ( dbcol . column_name ) . compile ( dialect = db . engine . dialect ) ) <EOL> if dbcol . sum : <EOL> metrics . append ( M ( <EOL> metric_name = '<STR_LIT>' + dbcol . column_name , <EOL> verbose_name = '<STR_LIT>' + dbcol . column_name , <EOL> metric_type = '<STR_LIT>' , <EOL> expression = "<STR_LIT>" . format ( quoted ) <EOL> ) ) <EOL> if dbcol . max : <EOL> metrics . append ( M ( <EOL> metric_name = '<STR_LIT>' + dbcol . column_name , <EOL> verbose_name = '<STR_LIT>' + dbcol . column_name , <EOL> metric_type = '<STR_LIT>' , <EOL> expression = "<STR_LIT>" . format ( quoted ) <EOL> ) ) <EOL> if dbcol . min : <EOL> metrics . append ( M ( <EOL> metric_name = '<STR_LIT>' + dbcol . column_name , <EOL> verbose_name = '<STR_LIT>' + dbcol . column_name , <EOL> metric_type = '<STR_LIT>' , <EOL> expression = "<STR_LIT>" . format ( quoted ) <EOL> ) ) <EOL> if dbcol . count_distinct : <EOL> metrics . append ( M ( <EOL> metric_name = '<STR_LIT>' + dbcol . column_name , <EOL> verbose_name = '<STR_LIT>' + dbcol . column_name , <EOL> metric_type = '<STR_LIT>' , <EOL> expression = "<STR_LIT>" . format ( quoted ) <EOL> ) ) <EOL> dbcol . type = datatype <EOL> db . session . merge ( self ) <EOL> db . session . commit ( ) <EOL> metrics . append ( M ( <EOL> metric_name = '<STR_LIT:count>' , <EOL> verbose_name = '<STR_LIT>' , <EOL> metric_type = '<STR_LIT:count>' , <EOL> expression = "<STR_LIT>" <EOL> ) ) <EOL> for metric in metrics : <EOL> m = ( <EOL> db . session . query ( M ) <EOL> . filter ( M . metric_name == metric . metric_name ) <EOL> . filter ( M . table_id == self . id ) <EOL> . first ( ) <EOL> ) <EOL> metric . table_id = self . id <EOL> if not m : <EOL> db . session . add ( metric ) <EOL> db . session . commit ( ) <EOL> if not self . main_dttm_col : <EOL> self . main_dttm_col = any_date_col <EOL> class SqlMetric ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> metric_name = Column ( String ( <NUM_LIT> ) ) <EOL> verbose_name = Column ( String ( <NUM_LIT> ) ) <EOL> metric_type = Column ( String ( <NUM_LIT:32> ) ) <EOL> table_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> table = relationship ( <EOL> '<STR_LIT>' , backref = '<STR_LIT>' , foreign_keys = [ table_id ] ) <EOL> expression = Column ( Text ) <EOL> description = Column ( Text ) <EOL> class TableColumn ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> table_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> table = relationship ( <EOL> '<STR_LIT>' , backref = '<STR_LIT>' , foreign_keys = [ table_id ] ) <EOL> column_name = Column ( String ( <NUM_LIT> ) ) <EOL> is_dttm = Column ( Boolean , default = False ) <EOL> is_active = Column ( Boolean , default = True ) <EOL> type = Column ( String ( <NUM_LIT:32> ) , default = '<STR_LIT>' ) <EOL> groupby = Column ( Boolean , default = False ) <EOL> count_distinct = Column ( Boolean , default = False ) <EOL> sum = Column ( Boolean , default = False ) <EOL> max = Column ( Boolean , default = False ) <EOL> min = Column ( Boolean , default = False ) <EOL> filterable = Column ( Boolean , default = False ) <EOL> expression = Column ( Text , default = '<STR_LIT>' ) <EOL> description = Column ( Text , default = '<STR_LIT>' ) <EOL> def __repr__ ( self ) : <EOL> return self . column_name <EOL> @ property <EOL> def isnum ( self ) : <EOL> types = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> return any ( [ t in self . type . upper ( ) for t in types ] ) <EOL> class DruidCluster ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> cluster_name = Column ( String ( <NUM_LIT> ) , unique = True ) <EOL> coordinator_host = Column ( String ( <NUM_LIT> ) ) <EOL> coordinator_port = Column ( Integer ) <EOL> coordinator_endpoint = Column ( <EOL> String ( <NUM_LIT> ) , default = '<STR_LIT>' ) <EOL> broker_host = Column ( String ( <NUM_LIT> ) ) <EOL> broker_port = Column ( Integer ) <EOL> broker_endpoint = Column ( String ( <NUM_LIT> ) , default = '<STR_LIT>' ) <EOL> metadata_last_refreshed = Column ( DateTime ) <EOL> def __repr__ ( self ) : <EOL> return self . cluster_name <EOL> def get_pydruid_client ( self ) : <EOL> cli = PyDruid ( <EOL> "<STR_LIT>" . format ( self . broker_host , self . broker_port ) , <EOL> self . broker_endpoint ) <EOL> return cli <EOL> def get_datasources ( self ) : <EOL> endpoint = ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> ) . format ( obj = self ) <EOL> return json . loads ( requests . get ( endpoint ) . text ) <EOL> def refresh_datasources ( self ) : <EOL> for datasource in self . get_datasources ( ) : <EOL> DruidDatasource . sync_to_db ( datasource , self ) <EOL> class DruidDatasource ( Model , AuditMixinNullable , Queryable ) : <EOL> """<STR_LIT>""" <EOL> type = "<STR_LIT>" <EOL> baselink = "<STR_LIT>" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> datasource_name = Column ( String ( <NUM_LIT> ) , unique = True ) <EOL> is_featured = Column ( Boolean , default = False ) <EOL> is_hidden = Column ( Boolean , default = False ) <EOL> description = Column ( Text ) <EOL> default_endpoint = Column ( Text ) <EOL> user_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> owner = relationship ( '<STR_LIT>' , backref = '<STR_LIT>' , foreign_keys = [ user_id ] ) <EOL> cluster_name = Column ( <EOL> String ( <NUM_LIT> ) , ForeignKey ( '<STR_LIT>' ) ) <EOL> cluster = relationship ( <EOL> '<STR_LIT>' , backref = '<STR_LIT>' , foreign_keys = [ cluster_name ] ) <EOL> offset = Column ( Integer , default = <NUM_LIT:0> ) <EOL> cache_timeout = Column ( Integer ) <EOL> @ property <EOL> def metrics_combo ( self ) : <EOL> return sorted ( <EOL> [ ( m . metric_name , m . verbose_name ) for m in self . metrics ] , <EOL> key = lambda x : x [ <NUM_LIT:1> ] ) <EOL> @ property <EOL> def name ( self ) : <EOL> return self . datasource_name <EOL> @ property <EOL> def perm ( self ) : <EOL> return ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) . format ( obj = self ) <EOL> @ property <EOL> def url ( self ) : <EOL> return '<STR_LIT>' . format ( self . id ) <EOL> @ property <EOL> def link ( self ) : <EOL> return ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) . format ( ** locals ( ) ) <EOL> @ property <EOL> def full_name ( self ) : <EOL> return ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) . format ( obj = self ) <EOL> def __repr__ ( self ) : <EOL> return self . datasource_name <EOL> @ property <EOL> def datasource_link ( self ) : <EOL> url = "<STR_LIT>" . format ( obj = self ) <EOL> return '<STR_LIT>' . format ( <EOL> url = url , obj = self ) <EOL> def get_metric_obj ( self , metric_name ) : <EOL> return [ <EOL> m . json_obj for m in self . metrics <EOL> if m . metric_name == metric_name <EOL> ] [ <NUM_LIT:0> ] <EOL> def latest_metadata ( self ) : <EOL> """<STR_LIT>""" <EOL> client = self . cluster . get_pydruid_client ( ) <EOL> results = client . time_boundary ( datasource = self . datasource_name ) <EOL> if not results : <EOL> return <EOL> max_time = results [ <NUM_LIT:0> ] [ '<STR_LIT:result>' ] [ '<STR_LIT>' ] <EOL> max_time = parse ( max_time ) <EOL> intervals = ( max_time - timedelta ( days = <NUM_LIT:7> ) ) . isoformat ( ) + '<STR_LIT:/>' <EOL> intervals += ( max_time - timedelta ( days = <NUM_LIT:1> ) ) . isoformat ( ) <EOL> segment_metadata = client . segment_metadata ( <EOL> datasource = self . datasource_name , <EOL> intervals = intervals ) <EOL> if segment_metadata : <EOL> return segment_metadata [ - <NUM_LIT:1> ] [ '<STR_LIT>' ] <EOL> def generate_metrics ( self ) : <EOL> for col in self . columns : <EOL> col . generate_metrics ( ) <EOL> @ classmethod <EOL> def sync_to_db ( cls , name , cluster ) : <EOL> """<STR_LIT>""" <EOL> print ( "<STR_LIT>" . format ( name ) ) <EOL> session = get_session ( ) <EOL> datasource = session . query ( cls ) . filter_by ( datasource_name = name ) . first ( ) <EOL> if not datasource : <EOL> datasource = cls ( datasource_name = name ) <EOL> session . add ( datasource ) <EOL> flasher ( "<STR_LIT>" . format ( name ) , "<STR_LIT:success>" ) <EOL> else : <EOL> flasher ( "<STR_LIT>" . format ( name ) , "<STR_LIT:info>" ) <EOL> datasource . cluster = cluster <EOL> cols = datasource . latest_metadata ( ) <EOL> if not cols : <EOL> return <EOL> for col in cols : <EOL> col_obj = ( <EOL> session <EOL> . query ( DruidColumn ) <EOL> . filter_by ( datasource_name = name , column_name = col ) <EOL> . first ( ) <EOL> ) <EOL> datatype = cols [ col ] [ '<STR_LIT:type>' ] <EOL> if not col_obj : <EOL> col_obj = DruidColumn ( datasource_name = name , column_name = col ) <EOL> session . add ( col_obj ) <EOL> if datatype == "<STR_LIT>" : <EOL> col_obj . groupby = True <EOL> col_obj . filterable = True <EOL> if col_obj : <EOL> col_obj . type = cols [ col ] [ '<STR_LIT:type>' ] <EOL> col_obj . datasource = datasource <EOL> col_obj . generate_metrics ( ) <EOL> def query ( <EOL> self , groupby , metrics , <EOL> granularity , <EOL> from_dttm , to_dttm , <EOL> filter = None , <EOL> is_timeseries = True , <EOL> timeseries_limit = None , <EOL> row_limit = None , <EOL> inner_from_dttm = None , inner_to_dttm = None , <EOL> extras = None , <EOL> select = None , ) : <EOL> """<STR_LIT>""" <EOL> qry_start_dttm = datetime . now ( ) <EOL> inner_from_dttm = inner_from_dttm or from_dttm <EOL> inner_to_dttm = inner_to_dttm or to_dttm <EOL> from_dttm = from_dttm . replace ( tzinfo = config . get ( "<STR_LIT>" ) ) <EOL> to_dttm = to_dttm . replace ( tzinfo = config . get ( "<STR_LIT>" ) ) <EOL> query_str = "<STR_LIT>" <EOL> aggregations = { <EOL> m . metric_name : m . json_obj <EOL> for m in self . metrics if m . metric_name in metrics <EOL> } <EOL> granularity = granularity or "<STR_LIT:all>" <EOL> if granularity != "<STR_LIT:all>" : <EOL> granularity = utils . parse_human_timedelta ( <EOL> granularity ) . total_seconds ( ) * <NUM_LIT:1000> <EOL> if not isinstance ( granularity , string_types ) : <EOL> granularity = { "<STR_LIT:type>" : "<STR_LIT>" , "<STR_LIT>" : granularity } <EOL> origin = extras . get ( '<STR_LIT>' ) <EOL> if origin : <EOL> dttm = utils . parse_human_datetime ( origin ) <EOL> granularity [ '<STR_LIT>' ] = dttm . isoformat ( ) <EOL> qry = dict ( <EOL> datasource = self . datasource_name , <EOL> dimensions = groupby , <EOL> aggregations = aggregations , <EOL> granularity = granularity , <EOL> intervals = from_dttm . isoformat ( ) + '<STR_LIT:/>' + to_dttm . isoformat ( ) , <EOL> ) <EOL> filters = None <EOL> for col , op , eq in filter : <EOL> cond = None <EOL> if op == '<STR_LIT>' : <EOL> cond = Dimension ( col ) == eq <EOL> elif op == '<STR_LIT>' : <EOL> cond = ~ ( Dimension ( col ) == eq ) <EOL> elif op in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> fields = [ ] <EOL> splitted = eq . split ( '<STR_LIT:U+002C>' ) <EOL> if len ( splitted ) > <NUM_LIT:1> : <EOL> for s in eq . split ( '<STR_LIT:U+002C>' ) : <EOL> s = s . strip ( ) <EOL> fields . append ( Filter . build_filter ( Dimension ( col ) == s ) ) <EOL> cond = Filter ( type = "<STR_LIT>" , fields = fields ) <EOL> else : <EOL> cond = Dimension ( col ) == eq <EOL> if op == '<STR_LIT>' : <EOL> cond = ~ cond <EOL> if filters : <EOL> filters = Filter ( type = "<STR_LIT>" , fields = [ <EOL> Filter . build_filter ( cond ) , <EOL> Filter . build_filter ( filters ) <EOL> ] ) <EOL> else : <EOL> filters = cond <EOL> if filters : <EOL> qry [ '<STR_LIT>' ] = filters <EOL> client = self . cluster . get_pydruid_client ( ) <EOL> orig_filters = filters <EOL> if timeseries_limit and is_timeseries : <EOL> pre_qry = deepcopy ( qry ) <EOL> pre_qry [ '<STR_LIT>' ] = "<STR_LIT:all>" <EOL> pre_qry [ '<STR_LIT>' ] = { <EOL> "<STR_LIT:type>" : "<STR_LIT:default>" , <EOL> "<STR_LIT>" : timeseries_limit , <EOL> '<STR_LIT>' : ( <EOL> inner_from_dttm . isoformat ( ) + '<STR_LIT:/>' + <EOL> inner_to_dttm . isoformat ( ) ) , <EOL> "<STR_LIT>" : [ { <EOL> "<STR_LIT>" : metrics [ <NUM_LIT:0> ] if metrics else self . metrics [ <NUM_LIT:0> ] , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> } ] , <EOL> } <EOL> client . groupby ( ** pre_qry ) <EOL> query_str += "<STR_LIT>" <EOL> query_str += json . dumps ( client . query_dict , indent = <NUM_LIT:2> ) + "<STR_LIT:\n>" <EOL> query_str += "<STR_LIT>" <EOL> df = client . export_pandas ( ) <EOL> if df is not None and not df . empty : <EOL> dims = qry [ '<STR_LIT>' ] <EOL> filters = [ ] <EOL> for _ , row in df . iterrows ( ) : <EOL> fields = [ ] <EOL> for dim in dims : <EOL> f = Filter . build_filter ( Dimension ( dim ) == row [ dim ] ) <EOL> fields . append ( f ) <EOL> if len ( fields ) > <NUM_LIT:1> : <EOL> filt = Filter ( type = "<STR_LIT>" , fields = fields ) <EOL> filters . append ( Filter . build_filter ( filt ) ) <EOL> elif fields : <EOL> filters . append ( fields [ <NUM_LIT:0> ] ) <EOL> if filters : <EOL> ff = Filter ( type = "<STR_LIT>" , fields = filters ) <EOL> if not orig_filters : <EOL> qry [ '<STR_LIT>' ] = ff <EOL> else : <EOL> qry [ '<STR_LIT>' ] = Filter ( type = "<STR_LIT>" , fields = [ <EOL> Filter . build_filter ( ff ) , <EOL> Filter . build_filter ( orig_filters ) ] ) <EOL> qry [ '<STR_LIT>' ] = None <EOL> if row_limit : <EOL> qry [ '<STR_LIT>' ] = { <EOL> "<STR_LIT:type>" : "<STR_LIT:default>" , <EOL> "<STR_LIT>" : row_limit , <EOL> "<STR_LIT>" : [ { <EOL> "<STR_LIT>" : metrics [ <NUM_LIT:0> ] if metrics else self . metrics [ <NUM_LIT:0> ] , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> } ] , <EOL> } <EOL> client . groupby ( ** qry ) <EOL> query_str += json . dumps ( client . query_dict , indent = <NUM_LIT:2> ) <EOL> df = client . export_pandas ( ) <EOL> if df is None or df . size == <NUM_LIT:0> : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> if ( <EOL> not is_timeseries and <EOL> granularity == "<STR_LIT:all>" and <EOL> '<STR_LIT>' in df . columns ) : <EOL> del df [ '<STR_LIT>' ] <EOL> cols = [ ] <EOL> if '<STR_LIT>' in df . columns : <EOL> cols += [ '<STR_LIT>' ] <EOL> cols += [ col for col in groupby if col in df . columns ] <EOL> cols += [ col for col in metrics if col in df . columns ] <EOL> cols += [ col for col in df . columns if col not in cols ] <EOL> df = df [ cols ] <EOL> return QueryResult ( <EOL> df = df , <EOL> query = query_str , <EOL> duration = datetime . now ( ) - qry_start_dttm ) <EOL> class Log ( Model ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> action = Column ( String ( <NUM_LIT> ) ) <EOL> user_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> dashboard_id = Column ( Integer ) <EOL> slice_id = Column ( Integer ) <EOL> user_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> json = Column ( Text ) <EOL> user = relationship ( '<STR_LIT>' , backref = '<STR_LIT>' , foreign_keys = [ user_id ] ) <EOL> dttm = Column ( DateTime , default = func . now ( ) ) <EOL> dt = Column ( Date , default = date . today ( ) ) <EOL> @ classmethod <EOL> def log_this ( cls , f ) : <EOL> """<STR_LIT>""" <EOL> @ functools . wraps ( f ) <EOL> def wrapper ( * args , ** kwargs ) : <EOL> user_id = None <EOL> if g . user : <EOL> user_id = g . user . id <EOL> d = request . args . to_dict ( ) <EOL> d . update ( kwargs ) <EOL> slice_id = d . get ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> slice_id = int ( slice_id ) if slice_id else <NUM_LIT:0> <EOL> log = cls ( <EOL> action = f . __name__ , <EOL> json = json . dumps ( d ) , <EOL> dashboard_id = d . get ( '<STR_LIT>' ) or None , <EOL> slice_id = slice_id , <EOL> user_id = user_id ) <EOL> db . session . add ( log ) <EOL> db . session . commit ( ) <EOL> return f ( * args , ** kwargs ) <EOL> return wrapper <EOL> class DruidMetric ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> metric_name = Column ( String ( <NUM_LIT> ) ) <EOL> verbose_name = Column ( String ( <NUM_LIT> ) ) <EOL> metric_type = Column ( String ( <NUM_LIT:32> ) ) <EOL> datasource_name = Column ( <EOL> String ( <NUM_LIT> ) , <EOL> ForeignKey ( '<STR_LIT>' ) ) <EOL> datasource = relationship ( '<STR_LIT>' , backref = '<STR_LIT>' , <EOL> enable_typechecks = False ) <EOL> json = Column ( Text ) <EOL> description = Column ( Text ) <EOL> @ property <EOL> def json_obj ( self ) : <EOL> try : <EOL> obj = json . loads ( self . json ) <EOL> except Exception : <EOL> obj = { } <EOL> return obj <EOL> class DruidColumn ( Model , AuditMixinNullable ) : <EOL> """<STR_LIT>""" <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> datasource_name = Column ( <EOL> String ( <NUM_LIT> ) , <EOL> ForeignKey ( '<STR_LIT>' ) ) <EOL> datasource = relationship ( '<STR_LIT>' , backref = '<STR_LIT>' , <EOL> enable_typechecks = False ) <EOL> column_name = Column ( String ( <NUM_LIT> ) ) <EOL> is_active = Column ( Boolean , default = True ) <EOL> type = Column ( String ( <NUM_LIT:32> ) ) <EOL> groupby = Column ( Boolean , default = False ) <EOL> count_distinct = Column ( Boolean , default = False ) <EOL> sum = Column ( Boolean , default = False ) <EOL> max = Column ( Boolean , default = False ) <EOL> min = Column ( Boolean , default = False ) <EOL> filterable = Column ( Boolean , default = False ) <EOL> description = Column ( Text ) <EOL> def __repr__ ( self ) : <EOL> return self . column_name <EOL> @ property <EOL> def isnum ( self ) : <EOL> return self . type in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def generate_metrics ( self ) : <EOL> """<STR_LIT>""" <EOL> M = DruidMetric <EOL> metrics = [ ] <EOL> metrics . append ( DruidMetric ( <EOL> metric_name = '<STR_LIT:count>' , <EOL> verbose_name = '<STR_LIT>' , <EOL> metric_type = '<STR_LIT:count>' , <EOL> json = json . dumps ( { '<STR_LIT:type>' : '<STR_LIT:count>' , '<STR_LIT:name>' : '<STR_LIT:count>' } ) <EOL> ) ) <EOL> if self . type in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> corrected_type = '<STR_LIT>' <EOL> else : <EOL> corrected_type = self . type <EOL> if self . sum and self . isnum : <EOL> mt = corrected_type . lower ( ) + '<STR_LIT>' <EOL> name = '<STR_LIT>' + self . column_name <EOL> metrics . append ( DruidMetric ( <EOL> metric_name = name , <EOL> metric_type = '<STR_LIT>' , <EOL> verbose_name = '<STR_LIT>' . format ( self . column_name ) , <EOL> json = json . dumps ( { <EOL> '<STR_LIT:type>' : mt , '<STR_LIT:name>' : name , '<STR_LIT>' : self . column_name } ) <EOL> ) ) <EOL> if self . min and self . isnum : <EOL> mt = corrected_type . lower ( ) + '<STR_LIT>' <EOL> name = '<STR_LIT>' + self . column_name <EOL> metrics . append ( DruidMetric ( <EOL> metric_name = name , <EOL> metric_type = '<STR_LIT>' , <EOL> verbose_name = '<STR_LIT>' . format ( self . column_name ) , <EOL> json = json . dumps ( { <EOL> '<STR_LIT:type>' : mt , '<STR_LIT:name>' : name , '<STR_LIT>' : self . column_name } ) <EOL> ) ) <EOL> if self . max and self . isnum : <EOL> mt = corrected_type . lower ( ) + '<STR_LIT>' <EOL> name = '<STR_LIT>' + self . column_name <EOL> metrics . append ( DruidMetric ( <EOL> metric_name = name , <EOL> metric_type = '<STR_LIT>' , <EOL> verbose_name = '<STR_LIT>' . format ( self . column_name ) , <EOL> json = json . dumps ( { <EOL> '<STR_LIT:type>' : mt , '<STR_LIT:name>' : name , '<STR_LIT>' : self . column_name } ) <EOL> ) ) <EOL> if self . count_distinct : <EOL> mt = '<STR_LIT>' <EOL> name = '<STR_LIT>' + self . column_name <EOL> metrics . append ( DruidMetric ( <EOL> metric_name = name , <EOL> verbose_name = '<STR_LIT>' . format ( self . column_name ) , <EOL> metric_type = '<STR_LIT>' , <EOL> json = json . dumps ( { <EOL> '<STR_LIT:type>' : '<STR_LIT>' , <EOL> '<STR_LIT:name>' : name , <EOL> '<STR_LIT>' : [ self . column_name ] } ) <EOL> ) ) <EOL> session = get_session ( ) <EOL> for metric in metrics : <EOL> m = ( <EOL> session . query ( M ) <EOL> . filter ( M . metric_name == metric . metric_name ) <EOL> . filter ( M . datasource_name == self . datasource_name ) <EOL> . filter ( DruidCluster . cluster_name == self . datasource . cluster_name ) <EOL> . first ( ) <EOL> ) <EOL> metric . datasource_name = self . datasource_name <EOL> if not m : <EOL> session . add ( metric ) <EOL> session . commit ( ) <EOL> class FavStar ( Model ) : <EOL> __tablename__ = '<STR_LIT>' <EOL> id = Column ( Integer , primary_key = True ) <EOL> user_id = Column ( Integer , ForeignKey ( '<STR_LIT>' ) ) <EOL> class_name = Column ( String ( <NUM_LIT:50> ) ) <EOL> obj_id = Column ( Integer ) <EOL> dttm = Column ( DateTime , default = func . now ( ) ) </s>
