"""
    Copyright (c) 2016-2017 Snowplow Analytics Ltd. All rights reserved.
    This program is licensed to you under the Apache License Version 2.0,
    and you may not use this file except in compliance with the Apache License
    Version 2.0. You may obtain a copy of the Apache License Version 2.0 at
    http://www.apache.org/licenses/LICENSE-2.0.
    Unless required by applicable law or agreed to in writing,
    software distributed under the Apache License Version 2.0 is distributed on
    an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
    express or implied. See the Apache License Version 2.0 for the specific
    language governing permissions and limitations there under.
"""

import sys

from datetime import datetime
from botocore.exceptions import ClientError

DYNAMODB_RUNID_ATTRIBUTE = 'RunId'

class RunManifests(object):
    """Wrapper class"""
    def __init__(self, dynamodb_client, table_name):
        self.dynamodb = dynamodb_client
        self.table_name = table_name

    def create(self):
        return create_manifest_table(self.dynamodb, self.table_name)

    def add(self, run_id):
        return add_to_manifest(self.dynamodb, self.table_name, run_id)

    def contains(self, run_id):
        return is_in_manifest(self.dynamodb, self.table_name, run_id)

def create_manifest_table(dynamodb_client, table_name):
    """Create DynamoDB table for run manifests

    Arguments:
    dynamodb_client - boto3 DynamoDB client (not service)
    table_name - string representing existing table name
    """
    try:
        dynamodb_client.create_table(
            AttributeDefinitions=[
                {
                    'AttributeName': DYNAMODB_RUNID_ATTRIBUTE,
                    'AttributeType': 'S'
                },
            ],
            TableName=table_name,
            KeySchema=[
                {
                    'AttributeName': DYNAMODB_RUNID_ATTRIBUTE,
                    'KeyType': 'HASH'
                },
            ],
            ProvisionedThroughput={
                'ReadCapacityUnits': 5,
                'WriteCapacityUnits': 5
            }
        )
        dynamodb_client.get_waiter('table_exists').wait(TableName=table_name)
    except ClientError as e:
        # Table already exists
        if e.response['Error']['Code'] == 'ResourceInUseException':
            pass
        else:
            raise e

def list_runids(s3_client, full_path):
    """Return list of all run ids inside S3 folder. It does not respect
    S3 pagination (`MaxKeys`) and returns **all** keys from bucket
    and won't list any prefixes with object archived to AWS Glacier

    Arguments:
    s3_client - boto3 S3 client (not service)
    full_path - full valid S3 path to events (such as enriched-archive)
                example: s3://acme-events-bucket/main-pipeline/enriched-archive
    """
    listing_finished = False                 # last response was not truncated
    run_ids_buffer = []
    last_continuation_token = None

    (bucket, prefix) = split_full_path(full_path)

    while not listing_finished:
        options = clean_dict({
            'Bucket': bucket,
            'Prefix': prefix,
            'Delimiter': '/',
            'ContinuationToken': last_continuation_token
        })

        response = s3_client.list_objects_v2(**options)
        keys = [extract_run_id(key['Prefix']) for key
                in response.get('CommonPrefixes', [])]
        run_ids_buffer.extend([key for key in keys if key is not None])
        last_continuation_token = response.get('NextContinuationToken', None)

        if not response['IsTruncated']:
            listing_finished = True

    non_archived_run_ids = [run_id for run_id in run_ids_buffer
                            if not is_glacier(s3_client, bucket, run_id)]
    return non_archived_run_ids

def split_full_path(path):
    """Return pair of bucket without protocol and path

    Arguments:
    path - valid S3 path, such as s3://somebucket/events

    >>> split_full_path('s3://mybucket/path-to-events')
    ('mybucket', 'path-to-events/')
    >>> split_full_path('s3://mybucket')
    ('mybucket', None)
    >>> split_full_path('s3n://snowplow-bucket/some/prefix/')
    ('snowplow-bucket', 'some/prefix/')
    """
    if path.startswith('s3://'):
        path = path[5:]
    elif path.startswith('s3n://'):
        path = path[6:]
    elif path.startswith('s3a://'):
        path = path[6:]
    else:
        raise ValueError("S3 path should start with s3://, s3n:// or "
                         "s3a:// prefix")
    parts = path.split('/')
    bucket = parts[0]
    path = '/'.join(parts[1:])
    return bucket, normalize_prefix(path)

def is_glacier(s3_client, bucket, prefix):
    """Check if prefix is archived in Glacier, by checking storage class of
    first object inside that prefix

    Arguments:
    s3_client - boto3 S3 client (not service)
    bucket - valid extracted bucket (without protocol and prefix)
             example: sowplow-events-data
    prefix - valid S3 prefix (usually, run_id)
             example: snowplow-archive/enriched/archive/
    """
    response = s3_client.list_objects_v2(Bucket=bucket,
                                         Prefix=prefix,
                                         MaxKeys=3)  # 3 to not fetch _SUCCESS

    for key in response['Contents']:
        if key.get('StorageClass', 'STANDARD') == 'GLACIER':
            return True
    return False

def extract_run_id(key):
    """Extract date part from run id

    Arguments:
    key - full key name, such as shredded-archive/run=2012-12-11-01-31-33/
          (trailing slash is required)

    >>> extract_run_id('shredded-archive/run=2012-12-11-01-11-33/')
    'shredded-archive/run=2012-12-11-01-11-33/'
    >>> extract_run_id('shredded-archive/run=2012-12-11-01-11-33')
    >>> extract_run_id('shredded-archive/run=2012-13-11-01-11-33/')
    """
    filename = key.split('/')[-2]  # -1 element is empty string
    run_id = filename.lstrip('run=')
    try:
        datetime.strptime(run_id, '%Y-%m-%d-%H-%M-%S')
        return key
    except ValueError:
        return None

def normalize_prefix(path):
    """Add trailing slash to prefix if it is not present

    >>> normalize_prefix("somepath")
    'somepath/'
    >>> normalize_prefix("somepath/")
    'somepath/'
    """
    if path is None or path is '' or path is '/':
        return None
    elif path.endswith('/'):
        return path
    else:
        return path + '/'

def clean_dict(dict):
    """Remove all keys with Nones as values

    >>> clean_dict({'key': None})
    {}
    >>> clean_dict({'empty_s': ''})
    {'empty_s': ''}
    """
    if sys.version_info[0] < 3:
        return {k: v for k, v in dict.iteritems() if v is not None}
    else:
        return {k: v for k, v in dict.items() if v is not None}

def add_to_manifest(dynamodb_client, table_name, run_id):
    """Add run_id into DynamoDB manifest table

    Arguments:
    dynamodb_client - boto3 DynamoDB client (not service)
    table_name - string representing existing table name
    run_id - string representing run_id to store
    """
    dynamodb_client.put_item(
        TableName=table_name,
        Item={
            DYNAMODB_RUNID_ATTRIBUTE: {
                'S': run_id
            }
        }
    )

def is_in_manifest(dynamodb_client, table_name, run_id):
    """Check if run_id is stored in DynamoDB table.
    Return True if run_id is stored or False otherwise.

    Arguments:
    dynamodb_client - boto3 DynamoDB client (not service)
    table_name - string representing existing table name
    run_id - string representing run_id to store
    """
    response = dynamodb_client.get_item(
        TableName=table_name,
        Key={
            DYNAMODB_RUNID_ATTRIBUTE: {
                'S': run_id
            }
        }
    )
    return response.get('Item') is not None


"""
    Copyright (c) 2016 Snowplow Analytics Ltd. All rights reserved.
    This program is licensed to you under the Apache License Version 2.0,
    and you may not use this file except in compliance with the Apache License
    Version 2.0. You may obtain a copy of the Apache License Version 2.0 at
    http://www.apache.org/licenses/LICENSE-2.0.
    Unless required by applicable law or agreed to in writing,
    software distributed under the Apache License Version 2.0 is distributed on
    an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
    express or implied. See the Apache License Version 2.0 for the specific
    language governing permissions and limitations there under.
    Authors: Fred Blundun
    Copyright: Copyright (c) 2016 Snowplow Analytics Ltd
    License: Apache License Version 2.0
"""

import re
import json

from snowplow_analytics_sdk.snowplow_event_transformation_exception import SnowplowEventTransformationException

# TODO: remove in 0.3.0
# See: https://github.com/snowplow/snowplow-python-analytics-sdk/issues/27
SCHEMA_PATTERN = re.compile(""".+:([a-zA-Z0-9_\.]+)/([a-zA-Z0-9_]+)/[^/]+/(.*)""")

SCHEMA_URI = ("^iglu:"                        # Protocol
              "([a-zA-Z0-9-_.]+)/"            # Vendor
              "([a-zA-Z0-9-_]+)/"             # Name
              "([a-zA-Z0-9-_]+)/"             # Format
              "([1-9][0-9]*"                  # MODEL (cannot start with 0)
              "(?:-(?:0|[1-9][0-9]*)){2})$")  # REVISION and ADDITION

SCHEMA_URI_REGEX = re.compile(SCHEMA_URI)

def extract_schema(uri):
    """
    Extracts Schema information from Iglu URI

    >>> extract_schema("iglu:com.acme-corporation_underscore/event_name-dash/jsonschema/1-10-1")['vendor']
    'com.acme-corporation_underscore'
    """
    match = re.match(SCHEMA_URI_REGEX, uri)
    if match:
        return {
            'vendor': match.group(1),
            'name': match.group(2),
            'format': match.group(3),
            'version': match.group(4)

        }
    else:
        raise SnowplowEventTransformationException([
            "Schema {} does not conform to regular expression {}".format(uri, SCHEMA_URI)
        ])

def fix_schema(prefix, schema):
    """
    Create an Elasticsearch field name from a schema string
    """
    schema_dict = extract_schema(schema)
    snake_case_organization = schema_dict['vendor'].replace('.', '_').lower()
    snake_case_name = re.sub('([^A-Z_])([A-Z])', '\g<1>_\g<2>', schema_dict['name']).lower()
    model = schema_dict['version'].split('-')[0]
    return "{}_{}_{}_{}".format(prefix, snake_case_organization, snake_case_name, model)

def parse_contexts(contexts):
    """
    Convert a contexts JSON to an Elasticsearch-compatible list of key-value pairs
    For example, the JSON

    {
      "data": [
        {
          "data": {
            "unique": true
          },
          "schema": "iglu:com.acme/unduplicated/jsonschema/1-0-0"
        },
        {
          "data": {
            "value": 1
          },
          "schema": "iglu:com.acme/duplicated/jsonschema/1-0-0"
        },
        {
          "data": {
            "value": 2
          },
          "schema": "iglu:com.acme/duplicated/jsonschema/1-0-0"
        }
      ],
      "schema": "iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"
    }

    would become

    [
      ("context_com_acme_duplicated_1", [{"value": 1}, {"value": 2}]),
      ("context_com_acme_unduplicated_1", [{"unique": true}])
    ]
    """
    my_json = json.loads(contexts)
    data = my_json['data']
    distinct_contexts = {}
    for context in data:
        schema = fix_schema("contexts", context['schema'])
        inner_data = context['data']
        if schema not in distinct_contexts:
            distinct_contexts[schema] = [inner_data]
        else:
            distinct_contexts[schema].append(inner_data)
    output = []
    for key in distinct_contexts:
        output.append((key, distinct_contexts[key]))
    return output

def parse_unstruct(unstruct):
    """
    Convert an unstructured event JSON to a list containing one Elasticsearch-compatible key-value pair
    For example, the JSON

    {
      "data": {
        "data": {
          "key": "value"
        },
        "schema": "iglu:com.snowplowanalytics.snowplow/link_click/jsonschema/1-0-1"
      },
      "schema": "iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0"
    }

    would become

    [
      (
        "unstruct_com_snowplowanalytics_snowplow_link_click_1", {
          "key": "value"
        }
      )
    ]
    """
    my_json = json.loads(unstruct)
    data = my_json['data']
    schema = data['schema']
    if 'data' in data:
        inner_data = data['data']
    else:
        raise SnowplowEventTransformationException(["Could not extract inner data field from unstructured event"])
    fixed_schema = fix_schema("unstruct_event", schema)
    return [(fixed_schema, inner_data)]


"""
    Copyright (c) 2016 Snowplow Analytics Ltd. All rights reserved.
    This program is licensed to you under the Apache License Version 2.0,
    and you may not use this file except in compliance with the Apache License
    Version 2.0. You may obtain a copy of the Apache License Version 2.0 at
    http://www.apache.org/licenses/LICENSE-2.0.
    Unless required by applicable law or agreed to in writing,
    software distributed under the Apache License Version 2.0 is distributed on
    an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
    express or implied. See the Apache License Version 2.0 for the specific
    language governing permissions and limitations there under.
    Authors: Fred Blundun
    Copyright: Copyright (c) 2016 Snowplow Analytics Ltd
    License: Apache License Version 2.0
"""

from snowplow_analytics_sdk import json_shredder
from snowplow_analytics_sdk.snowplow_event_transformation_exception import SnowplowEventTransformationException

LATITUDE_INDEX = 22
LONGITUDE_INDEX = 23

def convert_string(key, value):
    return [(key, value)]

def convert_int(key, value):
    return [(key, int(value))]

def convert_bool(key, value):
    if value == '1':
        return [(key, True)]
    elif value == '0':
        return [(key, False)]
    raise SnowplowEventTransformationException(["Invalid value {} for field {}".format(value, key)])

def convert_double(key, value):
    return [(key, float(value))]

def convert_tstamp(key, value):
    return [(key, value.replace(' ', 'T') + 'Z')]

def convert_contexts(key, value):
    return json_shredder.parse_contexts(value)

def convert_unstruct(key, value):
    return json_shredder.parse_unstruct(value)

# Ordered list of names of enriched event fields together with the function required to convert them to JSON
ENRICHED_EVENT_FIELD_TYPES = (
    ("app_id", convert_string),
    ("platform", convert_string),
    ("etl_tstamp", convert_tstamp),
    ("collector_tstamp", convert_tstamp),
    ("dvce_created_tstamp", convert_tstamp),
    ("event", convert_string),
    ("event_id", convert_string),
    ("txn_id", convert_int),
    ("name_tracker", convert_string),
    ("v_tracker", convert_string),
    ("v_collector", convert_string),
    ("v_etl", convert_string),
    ("user_id", convert_string),
    ("user_ipaddress", convert_string),
    ("user_fingerprint", convert_string),
    ("domain_userid", convert_string),
    ("domain_sessionidx", convert_int),
    ("network_userid", convert_string),
    ("geo_country", convert_string),
    ("geo_region", convert_string),
    ("geo_city", convert_string),
    ("geo_zipcode", convert_string),
    ("geo_latitude", convert_double),
    ("geo_longitude", convert_double),
    ("geo_region_name", convert_string),
    ("ip_isp", convert_string),
    ("ip_organization", convert_string),
    ("ip_domain", convert_string),
    ("ip_netspeed", convert_string),
    ("page_url", convert_string),
    ("page_title", convert_string),
    ("page_referrer", convert_string),
    ("page_urlscheme", convert_string),
    ("page_urlhost", convert_string),
    ("page_urlport", convert_int),
    ("page_urlpath", convert_string),
    ("page_urlquery", convert_string),
    ("page_urlfragment", convert_string),
    ("refr_urlscheme", convert_string),
    ("refr_urlhost", convert_string),
    ("refr_urlport", convert_int),
    ("refr_urlpath", convert_string),
    ("refr_urlquery", convert_string),
    ("refr_urlfragment", convert_string),
    ("refr_medium", convert_string),
    ("refr_source", convert_string),
    ("refr_term", convert_string),
    ("mkt_medium", convert_string),
    ("mkt_source", convert_string),
    ("mkt_term", convert_string),
    ("mkt_content", convert_string),
    ("mkt_campaign", convert_string),
    ("contexts", convert_contexts),
    ("se_category", convert_string),
    ("se_action", convert_string),
    ("se_label", convert_string),
    ("se_property", convert_string),
    ("se_value", convert_string),
    ("unstruct_event", convert_unstruct),
    ("tr_orderid", convert_string),
    ("tr_affiliation", convert_string),
    ("tr_total", convert_double),
    ("tr_tax", convert_double),
    ("tr_shipping", convert_double),
    ("tr_city", convert_string),
    ("tr_state", convert_string),
    ("tr_country", convert_string),
    ("ti_orderid", convert_string),
    ("ti_sku", convert_string),
    ("ti_name", convert_string),
    ("ti_category", convert_string),
    ("ti_price", convert_double),
    ("ti_quantity", convert_int),
    ("pp_xoffset_min", convert_int),
    ("pp_xoffset_max", convert_int),
    ("pp_yoffset_min", convert_int),
    ("pp_yoffset_max", convert_int),
    ("useragent", convert_string),
    ("br_name", convert_string),
    ("br_family", convert_string),
    ("br_version", convert_string),
    ("br_type", convert_string),
    ("br_renderengine", convert_string),
    ("br_lang", convert_string),
    ("br_features_pdf", convert_bool),
    ("br_features_flash", convert_bool),
    ("br_features_java", convert_bool),
    ("br_features_director", convert_bool),
    ("br_features_quicktime", convert_bool),
    ("br_features_realplayer", convert_bool),
    ("br_features_windowsmedia", convert_bool),
    ("br_features_gears", convert_bool),
    ("br_features_silverlight", convert_bool),
    ("br_cookies", convert_bool),
    ("br_colordepth", convert_string),
    ("br_viewwidth", convert_int),
    ("br_viewheight", convert_int),
    ("os_name", convert_string),
    ("os_family", convert_string),
    ("os_manufacturer", convert_string),
    ("os_timezone", convert_string),
    ("dvce_type", convert_string),
    ("dvce_ismobile", convert_bool),
    ("dvce_screenwidth", convert_int),
    ("dvce_screenheight", convert_int),
    ("doc_charset", convert_string),
    ("doc_width", convert_int),
    ("doc_height", convert_int),
    ("tr_currency", convert_string),
    ("tr_total_base", convert_double),
    ("tr_tax_base", convert_double),
    ("tr_shipping_base", convert_double),
    ("ti_currency", convert_string),
    ("ti_price_base", convert_double),
    ("base_currency", convert_string),
    ("geo_timezone", convert_string),
    ("mkt_clickid", convert_string),
    ("mkt_network", convert_string),
    ("etl_tags", convert_string),
    ("dvce_sent_tstamp", convert_tstamp),
    ("refr_domain_userid", convert_string),
    ("refr_device_tstamp", convert_tstamp),
    ("derived_contexts", convert_contexts),
    ("domain_sessionid", convert_string),
    ("derived_tstamp", convert_tstamp),
    ("event_vendor", convert_string),
    ("event_name", convert_string),
    ("event_format", convert_string),
    ("event_version", convert_string),
    ("event_fingerprint", convert_string),
    ("true_tstamp", convert_tstamp)
)

def transform(line, known_fields=ENRICHED_EVENT_FIELD_TYPES, add_geolocation_data=True):
    """
    Convert a Snowplow enriched event TSV into a JSON
    """
    return jsonify_good_event(line.split('\t'), known_fields, add_geolocation_data)

def jsonify_good_event(event, known_fields=ENRICHED_EVENT_FIELD_TYPES, add_geolocation_data=True):
    """
    Convert a Snowplow enriched event in the form of an array of fields into a JSON
    """
    if len(event) != len(known_fields):
        raise SnowplowEventTransformationException(
            ["Expected {} fields, received {} fields.".format(len(known_fields), len(event))]
        )
    else:
        output = {}
        errors = []
        if add_geolocation_data and event[LATITUDE_INDEX] != '' and event[LONGITUDE_INDEX] != '':
            output['geo_location'] = event[LATITUDE_INDEX] + ',' + event[LONGITUDE_INDEX]
        for i in range(len(event)):
            key = known_fields[i][0]
            if event[i] != '':
                try:
                    kvpairs = known_fields[i][1](key, event[i])
                    for kvpair in kvpairs:
                        output[kvpair[0]] = kvpair[1]
                except SnowplowEventTransformationException as sete:
                    errors += sete.error_messages
                except Exception as e:
                    errors += ["Unexpected exception parsing field with key {} and value {}: {}".format(
                        known_fields[i][0],
                        event[i],
                        repr(e)
                    )]
        if errors:
            raise SnowplowEventTransformationException(errors)
        else:
            return output


from __future__ import absolute_import, unicode_literals

from django.db.models import Model
from django.forms import BaseForm
from django.forms.models import BaseFormSet
from django.utils.translation import ugettext_lazy as _
from django.views.generic import View
from debug_toolbar.panels import Panel
from debugtools.utils.xview import get_view_name, get_used_template

class ViewPanel(Panel):
    """
    Shows versions of Python, Django, and installed apps if possible.
    """
    template = 'debugtools/view_panel.html'
    nav_title = _("View")

    def __init__(self, *args, **kwargs):
        super(ViewPanel, self).__init__(*args, **kwargs)
        self.view_module = None
        self.view_name = None

    def process_view(self, request, view_func, view_args, view_kwargs):
        # Store the information about the view being called.
        self.view_module = view_func.__module__
        self.view_name = get_view_name(view_func)

    def process_response(self, request, response):
        # Find out what template was used.
        template, choices = get_used_template(response)

        # See if more information can be read from the TemplateResponse object.
        if template and getattr(response, 'context_data', None):
            context_data = response.context_data
        else:
            context_data = None

        self.record_stats({
            'view_module': self.view_module,
            'view_name': self.view_name,
            'view_data': self._get_view_data(context_data) if context_data else None,
            'template': template,
            'template_choices': choices,
        })

    def _get_view_data(self, context_data):
        """
        Extract the used view from the TemplateResponse context (ContextMixin)
        """
        view = context_data.get('view')
        if not isinstance(view, View):
            view = None

        # Denote interesting objects in the template context
        template_context = []
        for key, obj in context_data.items():
            if isinstance(obj, (BaseForm, BaseFormSet, Model)):
                template_context.append((key, _format_path(obj.__class__)))

        return {
            'model': _get_view_model(view),
            'form': _get_form_class(view),
            'template_context': template_context,
        }

    @property
    def title(self):
        if self.view_name:
            return "%s (%s)" % (self.nav_title, self.view_name)
        else:
            return self.nav_subtitle

    @property
    def nav_subtitle(self):
        if self.view_name:
            return self.view_name
        else:
            return ''

def _get_form_class(view):
    if hasattr(view, 'get_form_class'):
        form = view.get_form_class()
    elif hasattr(view, 'form_class'):
        form = view.form_class
    else:
        return None

    if form is None:
        return None
    else:
        return _format_path(form)

def _get_view_model(view):
    if hasattr(view, 'model'):
        model = view.model
    elif hasattr(view, 'get_queryset'):
        model = view.get_queryset().model
    else:
        return None

    if model is None:
        return None
    else:
        return _format_path(model)

def _format_path(cls):
    return "{0}.{1}".format(cls.__module__, cls.__name__)


"""
INTERNAL FUNCTIONS FOR XViewMiddleware and ViewPanel
"""
from django.conf import settings
from django.template import TemplateDoesNotExist
from django.template.loader import get_template
from django.utils import six

def track_view_name(request, view_func):
    if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS or (request.user.is_active and request.user.is_staff):
        view_name = "{0}.{1}".format(view_func.__module__, get_view_name(view_func))
        request._xview = view_name
        return view_name

def get_view_name(view_func):
    if not hasattr(view_func, '__name__'):
        # e.g. django.contrib.formtools.views.FormWizard object with __call__() method
        return view_func.__class__.__name__
    else:
        return view_func.__name__

def get_used_view_name(request):
    return getattr(request, '_xview', None)

def get_used_template(response):
    """
    Get the template used in a TemplateResponse.
    This returns a tuple of "active choice, all choices"
    """
    if not hasattr(response, 'template_name'):
        return None, None

    template = response.template_name
    if template is None:
        return None, None

    if isinstance(template, (list, tuple)):
        # See which template name was really used.
        if len(template) == 1:
            return template[0], None
        else:
            used_name = _get_used_template_name(template)
            return used_name, template
    elif isinstance(template, six.string_types):
        # Single string
        return template, None
    else:
        # Template object.
        filename = _get_template_filename(template)
        template_name = '<template object from {0}>'.format(filename) if filename else '<template object>'
        return template_name, None

def _get_used_template_name(template_name_list):
    """
    Find which template of the template_names is selected by the Django loader.
    """
    for template_name in template_name_list:
        try:
            get_template(template_name)
            return template_name
        except TemplateDoesNotExist:
            continue

def _get_template_filename(template):
    # With TEMPLATE_DEBUG = True, each node tracks it's origin.
    try:
        return template.nodelist[0].origin[0].name
    except (AttributeError, IndexError):
        return None


"""
Debugging features in in the template.
"""
from django.template import Library, Node, Variable, VariableDoesNotExist
from django.template.defaultfilters import linebreaksbr
from django.utils import six
from django.utils.functional import Promise
from django.utils.html import escape, mark_safe

from debugtools.formatter import pformat_sql_html, pformat_django_context_html, pformat_dict_summary_html

try:
    from django.template import context_processors  # Django 1.8+
except ImportError:
    from django.core import context_processors

SHORT_NAME_TYPES = (bool,int,float,Promise) + six.string_types

DEBUG_WRAPPER_BLOCK = u'
{0}
'

# Twitter Bootstrap
 style:
PRE_STYLE = u"""clear: both; font-family: Menlo,Monaco,"Courier new",monospace; color: #333; background-color: #f5f5f5; border: 1px solid rgba(0, 0, 0, 0.15); border-radius: 4px 4px 4px 4px; font-size: 12.025px; text-align: left; line-height: 18px; margin: 9px; padding: 8px;"""

PRE_ALERT_STYLE = u"""clear: both; font-family: Menlo,Monaco,"Courier new",monospace; color: #C09853; background-color: #FCF8E3; border: 1px solid #FBEED5; border-radius: 4px 4px 4px 4px; font-size: 12.025px; text-align: left; line-height: 18px; margin-bottom: 18px; padding: 8px 35px 8px 14px; text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); white-space: pre-wrap; word-break: normal; word-wrap: normal;"""  # different word-wrap then Twitter Bootstrap

CONTEXT_TITLE = u'
TEMPLATE CONTEXT SCOPE:
\n'

CONTEXT_BLOCK = \
    u"
" \
    u"{num}:" \
    u"{dump1}{dump2}
"

BASIC_TYPE_BLOCK = u"
{name} = {value}
"

ERROR_TYPE_BLOCK = u"
{error}
"

OBJECT_TYPE_BLOCK = u"
{name} = {type}:\n{value}
"

register = Library()

class PrintNode(Node):
    @classmethod
    def parse(cls, parser, token):
        varnames = token.contents.split()[1:]
        return cls(
            variables=[(name, parser.compile_filter(name)) for name in varnames]
        )

    def __init__(self, variables):
        # Thread safety OK: the list of varnames won't change for this node.
        # Data is read only inside the render() function.
        self.variables = list(variables)

    def render(self, context):
        if self.variables:
            text = self.print_variables(context)
        else:
            text = self.print_context(context)
        return mark_safe(DEBUG_WRAPPER_BLOCK.format(text))

    def print_context(self, context):
        """
        Print the entire template context
        """
        text = [CONTEXT_TITLE]
        for i, context_scope in enumerate(context):
            dump1 = linebreaksbr(pformat_django_context_html(context_scope))
            dump2 = pformat_dict_summary_html(context_scope)

            # Collapse long objects by default (e.g. request, LANGUAGES and sql_queries)
            if len(context_scope) <= 3 and dump1.count('
') > 20:
                (dump1, dump2) = (dump2, dump1)

            text.append(CONTEXT_BLOCK.format(
                style=PRE_STYLE,
                num=i,
                dump1=dump1,
                dump2=dump2
            ))
        return u''.join(text)

    def print_variables(self, context):
        """
        Print a set of variables
        """
        text = []
        for name, expr in self.variables:
            # Some extended resolving, to handle unknown variables
            data = ''
            try:
                if isinstance(expr.var, Variable):
                    data = expr.var.resolve(context)
                else:
                    data = expr.resolve(context)  # could return TEMPLATE_STRING_IF_INVALID
            except VariableDoesNotExist as e:
                # Failed to resolve, display exception inline
                keys = []
                for scope in context:
                    keys += scope.keys()
                keys = sorted(set(keys))  # Remove duplicates, e.g. csrf_token
                return ERROR_TYPE_BLOCK.format(style=PRE_ALERT_STYLE, error=escape(u"Variable '{0}' not found!  Available context variables are:\n\n{1}".format(expr, u', '.join(keys))))
            else:
                # Regular format
                textdata = linebreaksbr(pformat_django_context_html(data))

            # At top level, prefix class name if it's a longer result
            if isinstance(data, SHORT_NAME_TYPES):
                text.append(BASIC_TYPE_BLOCK.format(style=PRE_STYLE, name=name, value=textdata))
            else:
                text.append(OBJECT_TYPE_BLOCK.format(style=PRE_STYLE, name=name, type=data.__class__.__name__, value=textdata))
        return u''.join(text)

@register.tag('print')
def _print(parser, token):
    """
    A template tag which prints dumps the contents of objects.
    """
    return PrintNode.parse(parser, token)

@register.inclusion_tag('debugtools/sql_queries.html', takes_context=True)
def print_queries(context):
    return context_processors.debug(context['request'])

@register.filter
def format_sql(sql):
    return mark_safe(pformat_sql_html(sql))

def _format_exception(exception):
    return u'{0}'.format(escape('<{0}>'.format(exception)))


"""
An enhanced ``pprint.pformat`` that prints data structures in a readable HTML style.
"""
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db import IntegrityError
from django.db.models.base import Model
from django.db.models.manager import Manager
from django.db.models.query import QuerySet
from django.forms.forms import BaseForm
import django.template.loader  # avoid recursive import issues with loader_tags
from django.template.loader_tags import BlockNode
from django.utils import six
from django.utils.encoding import smart_str
from django.utils.functional import Promise
from django.utils.html import escape
from itertools import chain
from pprint import PrettyPrinter, _StringIO
import re
import inspect
import types
import sys
from django.utils.safestring import mark_safe

try:
    from django.urls import NoReverseMatch  # Django 1.10+
except ImportError:
    from django.core.urlresolvers import NoReverseMatch

if sys.version_info[0] >= 3:
    PY3 = True
    py3_str = str
else:
    PY3 = False
    py3_str = unicode

DICT_EXPANDED_TYPES = (bool, int,) + six.string_types
HANDLED_EXCEPTIONS = (
    TypeError, IndexError, KeyError, AttributeError, ValueError,
    ObjectDoesNotExist, MultipleObjectsReturned, IntegrityError, NoReverseMatch,
    AssertionError, NotImplementedError, RuntimeError,
)

RE_SQL_NL = re.compile(r'\b(FROM|LEFT OUTER|RIGHT|LEFT|INNER|OUTER|WHERE|ORDER BY|GROUP BY)\b')
RE_SQL = re.compile(r'\b(SELECT|UPDATE|DELETE'
                    r'|COUNT|AVG|MAX|MIN|CASE'
                    r'|FROM|SET'
                    r'|ORDER|GROUP|BY|ASC|DESC|LIMIT'
                    r'|WHERE|AND|OR|IN|LIKE|BETWEEN|IS|NULL'
                    r'|LEFT|RIGHT|INNER|OUTER|JOIN|HAVING)\b')

def pformat_sql_html(sql):
    """
    Highlight common SQL words in a string.
    """
    sql = escape(sql)
    sql = RE_SQL_NL.sub(u'
\n\\1', sql)
    sql = RE_SQL.sub(u'\\1', sql)
    return sql

def pformat_django_context_html(object):
    """
    Dump a variable to a HTML string with sensible output for template context fields.
    It filters out all fields which are not usable in a template context.
    """
    if isinstance(object, QuerySet):
        text = ''
        lineno = 0
        for item in object.all()[:21]:
            lineno += 1
            if lineno >= 21:
                text += u'   (remaining items truncated...)'
                break
            text += u'   {0}\n'.format(escape(repr(item)))
        return text
    elif isinstance(object, Manager):
        return mark_safe(u'    (use .all to read it)')
    elif isinstance(object, six.string_types):
        return escape(repr(object))
    elif isinstance(object, Promise):
        # lazy() object
        return escape(_format_lazy(object))
    elif isinstance(object, dict):
        # This can also be a ContextDict
        return _format_dict(object)
    elif isinstance(object, list):
        return _format_list(object)
    elif hasattr(object, '__dict__'):
        return _format_object(object)
    else:
        # Use regular pprint as fallback.
        text = DebugPrettyPrinter(width=200).pformat(object)
        return _style_text(text)

def pformat_dict_summary_html(dict):
    """
    Briefly print the dictionary keys.
    """
    if not dict:
        return '   {}'

    html = []
    for key, value in sorted(six.iteritems(dict)):
        if not isinstance(value, DICT_EXPANDED_TYPES):
            value = '...'

        html.append(_format_dict_item(key, value))

    return mark_safe(u'
'.join(html))

# The start marker helps to detect the beginning of a new element.
_start = r'([{\[ ])'  # Allow to start with {, [ or space.
RE_PROXY = re.compile(escape(_start + r'<django.utils.functional.__proxy__ object at 0x[0-9a-f]+>'))
RE_FUNCTION = re.compile(escape(_start + r'<function [^ ]+ at 0x[0-9a-f]+>'))
RE_GENERATOR = re.compile(escape(_start + r'<generator object [^ ]+ at 0x[0-9a-f]+>'))
RE_OBJECT_ADDRESS = re.compile(escape(_start + r'<([^ ]+) object at 0x[0-9a-f]+>'))
RE_MANAGER = re.compile(escape(_start + r'<([^ ]+) manager>'))
RE_CLASS_REPR = re.compile(escape(_start + r"<class '([^']+)'>"))
RE_FIELD_END = re.compile(escape(r",([\r\n] ')"))
RE_FIELDNAME = re.compile(escape(r"^ u?'([^']+)': "), re.MULTILINE)
RE_REQUEST_FIELDNAME = re.compile(escape(r"^(\w+):\{'([^']+)': "), re.MULTILINE)
RE_REQUEST_CLEANUP1 = re.compile(escape(r"\},([\r\n]META:)"))
RE_REQUEST_CLEANUP2 = re.compile(escape(r"\)\}>$"))

def _style_text(text):
    """
    Apply some HTML highlighting to the contents.
    This can't be done in the
    """
    # Escape text and apply some formatting.
    # To have really good highlighting, pprint would have to be re-implemented.
    text = escape(text)
    text = text.replace(' <iterator object>', " <<var>this object can be used in a 'for' loop</var>>")
    text = text.replace(' <dynamic item>', ' <<var>this object may have extra field names</var>>')
    text = text.replace(' <dynamic attribute>', ' <<var>this object may have extra field names</var>>')
    text = RE_PROXY.sub('\g<1><<var>proxy object</var>>', text)
    text = RE_FUNCTION.sub('\g<1><<var>object method</var>>', text)
    text = RE_GENERATOR.sub("\g<1><<var>generator, use 'for' to traverse it</var>>", text)
    text = RE_OBJECT_ADDRESS.sub('\g<1><<var>\g<2> object</var>>', text)
    text = RE_MANAGER.sub('\g<1><<var>manager, use .all to traverse it</var>>', text)
    text = RE_CLASS_REPR.sub('\g<1><<var>\g<2> class</var>>', text)

    # Since Django's WSGIRequest does a pprint like format for it's __repr__, make that styling consistent
    text = RE_REQUEST_FIELDNAME.sub('\g<1>:\n   \g<2>: ', text)
    text = RE_REQUEST_CLEANUP1.sub('\g<1>', text)
    text = RE_REQUEST_CLEANUP2.sub(')', text)

    return mark_safe(text)

def _format_object(object):
    """
    # Instead of just printing <SomeType at 0xfoobar>, expand the fields.
    """

    attrs = iter(object.__dict__.items())
    if object.__class__:
        # Add class members too.
        attrs = chain(attrs, iter(object.__class__.__dict__.items()))

    # Remove private and protected variables
    # Filter needless exception classes which are added to each model.
    # Filter unremoved form.Meta (unline model.Meta) which makes no sense either
    is_model = isinstance(object, Model)
    is_form = isinstance(object, BaseForm)
    attrs = dict(
        (k, v)
        for k, v in attrs
            if not k.startswith('_')
            and not getattr(v, 'alters_data', False)
            and not (is_model and k in ('DoesNotExist', 'MultipleObjectsReturned'))
            and not (is_form and k in ('Meta',))
    )

    # Add members which are not found in __dict__.
    # This includes values such as auto_id, c, errors in a form.
    for member in dir(object):
        try:
            if member.startswith('_') or not hasattr(object, member):
                continue
        except HANDLED_EXCEPTIONS as e:
            attrs[member] = _format_exception(e)
            continue

        value = getattr(object, member)
        if callable(value) or member in attrs or getattr(value, 'alters_data', False):
            continue

        attrs[member] = value

    # Format property objects
    for name, value in list(attrs.items()):  # not iteritems(), so can delete.
        if isinstance(value, property):
            attrs[name] = _try_call(lambda: getattr(object, name))
        elif isinstance(value, types.FunctionType):
            if PY3:
                spec = inspect.getfullargspec(value)
            else:
                spec = inspect.getargspec(value)
            if len(spec.args) == 1 or len(spec.args) == len(spec.defaults or ()) + 1:
                if _is_unsafe_name(name):
                    # The delete and save methods should have an alters_data = True set.
                    # however, when delete or save methods are overridden, this is often missed.
                    attrs[name] = LiteralStr('<Skipped for safety reasons (could alter the database)>')
                else:
                    # should be simple method(self) signature to be callable in the template
                    # function may have args (e.g. BoundField.as_textarea) as long as they have defaults.
                    attrs[name] = _try_call(lambda: value(object))
            else:
                del attrs[name]
        elif hasattr(value, '__get__'):
            # fetched the descriptor, e.g. django.db.models.fields.related.ForeignRelatedObjectsDescriptor
            attrs[name] = value = _try_call(lambda: getattr(object, name), return_exceptions=True)
            if isinstance(value, Manager):
                attrs[name] = LiteralStr('<{0} manager>'.format(value.__class__.__name__))
            elif isinstance(value, AttributeError):
                del attrs[name]  # e.g. Manager isn't accessible via Model instances.
            elif isinstance(value, HANDLED_EXCEPTIONS):
                attrs[name] = _format_exception(value)

    # Include representations which are relevant in template context.
    if getattr(object, '__str__', None) is not object.__str__:
        attrs['__str__'] = _try_call(lambda: smart_str(object))
    elif getattr(object, '__unicode__', None) is not object.__unicode__:
        attrs['__unicode__'] = _try_call(lambda: smart_str(object))

    if hasattr(object, '__getattr__'):
        attrs['__getattr__'] = LiteralStr('<dynamic attribute>')
    if hasattr(object, '__getitem__'):
        attrs['__getitem__'] = LiteralStr('<dynamic item>')
    if hasattr(object, '__iter__'):
        attrs['__iter__'] = LiteralStr('<iterator object>')
    if hasattr(object, '__len__'):
        attrs['__len__'] = len(object)

    # Add known __getattr__ members which are useful for template designers.
    if isinstance(object, BaseForm):
        for field_name in list(object.fields.keys()):
            attrs[field_name] = object[field_name]
        del attrs['__getitem__']

    return _format_dict(attrs)

def _format_list(list):
    list = list[:]
    for i, value in enumerate(list):
        list[i] = _format_value(value)

    text = DebugPrettyPrinter(width=200).pformat(list)
    return _style_text(text)

def _format_dict(dict):
    if not dict:
        return mark_safe('   (<var>empty dict</var>)')
    else:
        html = []
        for key, value in sorted(dict.items()):
            html.append(_format_dict_item(key, value))
        return mark_safe(u'
'.join(html))

def _format_dict_item(key, value):
    if isinstance(key, six.string_types):
        key_html = key
        key_len = len(key)
    else:
        key_html = DebugPrettyPrinter().pformat(_format_value(key))
        key_len = len(key_html)
        key_html = _style_text(key_html)

    if not isinstance(value, DICT_EXPANDED_TYPES):
        value = _style_text(DebugPrettyPrinter(width=200).pformat_sub(_format_value(value), indent=key_len + 5, level=1))
    else:
        value = escape(repr(value))

    return u'   {0}: {1}'.format(key_html, value)

def _format_value(value):
    if isinstance(value, BlockNode):
        # The Block node is very verbose, making debugging hard.
        return LiteralStr(u"<BlockNode: {0}>".format(value.name))
    elif isinstance(value, Promise):
        # lazy() object
        return _format_lazy(value)
    else:
        return value

def _format_lazy(value):
    """
    Expand a _("TEST") call to something meaningful.
    """
    args = value._proxy____args
    kw = value._proxy____kw
    if not kw and len(args) == 1 and isinstance(args[0], six.string_types):
        # Found one of the Xgettext_lazy() calls.
        return LiteralStr(u'ugettext_lazy({0})'.format(repr(value._proxy____args[0])))

    # Prints <django.functional.utils.__proxy__ object at ..>
    return value

def _format_exception(e):
    return LiteralStr(u"<caught exception: {0}>".format(repr(e)))

def _is_unsafe_name(name):
    # Sometimes the `alters_data` is forgotten, or lost when a method is overwritten.
    # This extra safeguard makes sure those methods aren't called.
    if 'delete' in name or 'save' in name:
        return True
    elif name in ('full_clean',):
        return True
    elif name.startswith('clean') or name.startswith('copy') or name.startswith('move'):
        return True
    return False

def _try_call(func, extra_exceptions=(), return_exceptions=False):
    """
    Call a method, but
    :param func:
    :type func:
    :param extra_exceptions:
    :type extra_exceptions:
    :return:
    :rtype:
    """
    try:
        return func()
    except HANDLED_EXCEPTIONS as e:
        if return_exceptions:
            return e
        else:
            return _format_exception(e)
    except extra_exceptions as e:
        if return_exceptions:
            return e
        else:
            return _format_exception(e)

class LiteralStr(object):
    """
    A trick to make pformat() print a custom string without quotes.
    """
    def __init__(self, rawvalue):
        self.rawvalue = rawvalue

    def __repr__(self):
        if isinstance(self.rawvalue, six.string_types):
            return self.rawvalue
        else:
            return repr(self.rawvalue)

class DebugPrettyPrinter(PrettyPrinter):
    """
    Extended pformat() handling to trap exceptions
    This is an old-style class, hence no super() usage here
    """
    def pformat_sub(self, object, indent=0, level=0):
        sio = _StringIO()
        self._format(object, sio, indent, 0, {}, level)
        return sio.getvalue()

    def format(self, object, context, maxlevels, level):
        """
        Format an item in the result.
        Could be a dictionary key, value, etc..
        """
        try:
            return PrettyPrinter.format(self, object, context, maxlevels, level)
        except HANDLED_EXCEPTIONS as e:
            return _format_exception(e), True, False

    def _format(self, object, stream, indent, allowance, context, level):
        """
        Recursive part of the formatting
        """
        try:
            PrettyPrinter._format(self, object, stream, indent, allowance, context, level)
        except Exception as e:
            stream.write(_format_exception(e))