"""Hotkey-related routines."""

import wx, sys, logging
try:
 import win32con
except ImportError:
 win32con = None # Not available on this system.
from collections import OrderedDict

logger = logging.getLogger(__name__)

_tables = {} # Control:table pairs.
_hotkeys = {} # control: [hotkey, id] pairs.

mods = OrderedDict() # List of modifiers.
global_mods = OrderedDict()
converts = OrderedDict() # Give command keys their proper names.

logger.debug('Creating command keys.')
if sys.platform == 'darwin':
 logger.debug('OS X detected.')
 mods[wx.MOD_CONTROL] = 'CMD'
 mods[wx.MOD_ALT] = 'OPT'
 # Don't bother with global hotkeys... They don't work on OS X anyways.
 converts['WXK_CONTROL'] = 'CMD'
 converts['WXK_RAW_CONTROL'] = 'CTRL'
 converts['WXK_ALT'] = 'OPT'
else:
 logger.debug('%s detected.', sys.platform.title())
 mods[wx.MOD_CONTROL] = 'CTRL'
 mods[wx.MOD_ALT] = 'ALT'
 if win32con:
  global_mods[win32con.MOD_CONTROL] = 'CTRL'
  global_mods[win32con.MOD_ALT] = 'ALT'
 converts['WXK_CONTROL'] = 'CTRL'
 converts['WXK_RAW_CONTROL'] = 'CTRL'
converts['VK_CONTROL'] = 'CTRL'
converts['VK_SHIFT'] = 'SHIFT'
converts['VK_ALT'] = 'ALT'
mods[wx.MOD_SHIFT] = 'SHIFT'
if win32con:
 global_mods[win32con.MOD_SHIFT] = 'SHIFT'

def key_to_str(modifiers, key, mods_table = mods, key_table = wx, key_prefix = 'WXK_'):
 """
 Returns a human-readable version of numerical modifiers and key.

 To make the key suitable for global hotkey usage, supply:
 mods_table = global_mods, key_table = win32con, key_prefix = 'VK_'
 """
 logger.debug('Converting (%s, %s) to string.', modifiers, key)
 if not key:
  key_str = 'NONE'
 else:
  key_str = None
 res = ''
 for value, name in mods_table.items():
  if (modifiers & value):
   res += name + '+'
 for x in dir(key_table):
  if x.startswith(key_prefix):
   if getattr(key_table, x) == key:
    key_str = converts.get(x, x[len(key_prefix):])
 if not key_str:
  key_str = chr(key)
 res += key_str
 logger.debug('Final result: %s.', res)
 return res

def str_to_key(value, key_table = wx, accel_format = 'ACCEL_%s', key_format = 'WXK_%s', key_transpositions = {}):
 """
 Turns a string like "CTRL_ALT+K" into (3, 75).

 To get a global hotkey, try passing:
 key_table = win32con, accel_format = 'MOD_%s', key_format = 'VK_%s', key_transpositions = {'CTRL': 'CONTROL'}
 """
 logger.debug('Converting "%s" to integers.', value)
 modifiers = 0
 key = 0
 split = value.split('+')
 for v in split:
  v = v.upper()
  a = accel_format % key_transpositions.get(v, v)
  logger.debug('Accelerator format = %s.', a)
  k = key_format % key_transpositions.get(v, v)
  logger.debug('Key format = %s.', k)
  if hasattr(key_table, a):
   logger.debug('Found accelerator on %r.', key_table)
   modifiers = modifiers | getattr(key_table, a)
  elif hasattr(key_table, k):
   logger.debug('Found key on %r.', key_table)
   if key:
    raise ValueError('Multiple keys specified.')
   else:
    key = getattr(key_table, k)
 if not key:
  logger.debug('No key yet, falling back to ord.')
  key = ord(split[-1])
 logger.debug('modifiers = %d, key = %d.', modifiers, key)
 return (modifiers, key)

def get_id(id):
 """Get a new id if the provided one is None."""
 if id == None:
  id = wx.NewId()
  logger.debug('Generated new ID %s.', id)
 else:
  logger.debug('Using provided id %s.', id)
 return id

def add_accelerator(control, key, func, id = None):
 """
 Adds a key to the control.

 control: The control that the accelerator should be added to.
 key: A string like "CTRL+F", or "CMD+T" that specifies the key to use.
 func: The function that should be called when key is pressed.
 id: The id to Bind the event to. Defaults to wx.NewId().
 """
 logger.debug('Adding key "%s" to control %s to call %s.', key, control, func)
 id = get_id(id)
 control.Bind(wx.EVT_MENU, func, id = id)
 t = _tables.get(control, [])
 modifiers, key_int = str_to_key(key)
 t.append((modifiers, key_int, id))
 _tables[control] = t
 update_accelerators(control)
 return id

def remove_accelerator(control, key):
 """
 Removes an accelerator from control.

 control: The control to affect.
 key: The key to remove.
 """
 key = str_to_key(key)
 t = _tables.get(control, [])
 for a in t:
  if a[:2] == key:
   t.remove(a)
   if t:
    _tables[control] = t
   else:
    del _tables[control]
   update_accelerators(control)
   return True
 return False

def add_hotkey(control, key, func, id = None):
 """
 Add a global hotkey bound to control via id that should call func.

 control: The control to bind to.
 key: The hotkey to use.
 func: The func to call.
 id: The new ID to use (defaults to creating a new ID.
 """
 if win32con is None:
  raise RuntimeError('win32con is not available.')
 logger.debug('Adding hotkey "%s" to control %s to call %s.', key, control, func)
 modifiers, keycode = str_to_key(key, key_table = win32con, accel_format = 'MOD_%s', key_format = 'VK_%s', key_transpositions = {'CTRL': 'CONTROL'})
 id = get_id(id)
 control.Bind(wx.EVT_HOTKEY, func, id = id)
 l = _hotkeys.get(control, [])
 l.append([key, id])
 _hotkeys[control] = l
 return control.RegisterHotKey(id, modifiers, keycode)

def remove_hotkey(control, key):
 """
 Remove a global hotkey.

 control - The control to affect
 key - The key to remove.
 """
 l = _hotkeys.get(control, [])
 for a in l:
  key_str, id = a
  if key_str == key:
   control.Unbind(wx.EVT_HOTKEY, id = id)
   control.UnregisterHotKey(id)
   l.remove(a)
   if l:
    _hotkeys[control] = l
   else:
    del _hotkeys[control]

def update_accelerators(control):
 """Update the accelerator table for control."""
 control.SetAcceleratorTable(wx.AcceleratorTable(_tables.get(control, [])))


#
# Copyright 2015 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from boundary import ApiCall
import argparse
import logging
import sys
from pygments import highlight, lexers, formatters
import pygments.lexers.html as html

"""
Base class for all the Boundary CLI commands
"""

class ApiCli(ApiCall):
    def __init__(self):
        ApiCall.__init__(self)
        self.product_name = 'TrueSight Pulse'
        # Construct a dictionary with each of the HTTP methods that we support

        self.levels = {"debug": logging.DEBUG,
                       "info": logging.INFO,
                       "warn": logging.WARN,
                       "error": logging.ERROR,
                       "critical": logging.CRITICAL}

        # Properties
        self._message = None

        self.args = None

        self.parser = argparse.ArgumentParser(description=self.get_description())
        self._aggregate_times = ['1 second',
                                 '15 seconds',
                                 '1 minute',
                                 '5 minutes',
                                 '1 hour',
                                 '1.5 hours',
                                 '3 hours',
                                 '6 hours',
                                 '12 hours']

    def get_aggregate_grains(self):
        """
        Returns the standard cube aggregates in the alarm/measurement APIs
        """
        return self._aggregate_times

    def get_description(self):
        """
        Returns a description of the CLI
        """
        return "General API CLI"

    def add_logging_argument(self):
        self.parser.add_argument('-l', '--log-level', dest='logLevel', action='store',
                                 choices=['debug', 'info', 'warning', 'error', 'critical'],
                                 help='Sets logging level to one of debug,info,warning,error,critical. ' +
                                      'Default is logging is disabled')

    def add_arguments(self):
        """
        Configure handling of command line arguments.
        """
        self.add_logging_argument()
        self.parser.add_argument('-a', '--api-host', dest='api_host', action='store', metavar="api_host",
                                 help='{0} API host endpoint'.format(self.product_name))
        self.parser.add_argument('-e', '--email', dest='email', action='store', metavar="e_mail",
                                 help='e-mail that has access to the {0} account'.format(self.product_name))
        self.parser.add_argument('-t', '--api-token', dest='api_token', required=False, action='store',
                                 metavar="api_token",
                                 help='API token for given e-mail that has access to the {0} account'.format(
                                     self.product_name))
        self.parser.add_argument('-z', '--curl', dest='curl', required=False, action='store_true', default=False,
                                 help='Output the corresponding curl command line and exit')

    def _parse_args(self):
        """
        Handles the parse of the command line arguments
        """
        # Perform the actual parsing of the command line arguments
        self.args = self.parser.parse_args()

    def _configure_logging(self):
        """
        Configure logging based on command line options
        """
        if self.args.logLevel is not None:
            logging.basicConfig(level=self.levels[self.args.logLevel])
        logging.info("Set logging level to {0}".format(self.args.logLevel))

    def get_arguments(self):
        """
        CLIs get called back so that they can process any command line arguments
        that are given. This method handles the standard command line arguments for:
        API Host, user, password, etc.
        """

        # We call this first so that logging is enabled as soon as possible
        self._configure_logging()

        # Extract the common command line arguments
        if self.args.api_host is not None:
            self._api_host = self.args.api_host
        if self.args.email is not None:
            self._email = self.args.email
        if self.args.api_token is not None:
            self._api_token = self.args.api_token
        self._curl = self.args.curl

        logging.debug("apihost: {0}".format(self._api_host))
        logging.debug("email: {0}".format(self._email))
        logging.debug("apitoken: {0}".format(self._api_token))

    def set_error_message(self, message):
        """
        Sets the error message to be displayed if an error occurs
        """
        self._message = message

    def _validate_arguments(self):
        """
        Validates the command line arguments passed to the CLI
        Derived classes that override need to call this method before
        validating their arguments
        """
        if self._email is None:
            self.set_error_message("E-mail for the account not provided")
            return False
        if self._api_token is None:
            self.set_error_message("API Token for the account not provided")
            return False
        return True

    def colorize_json(self, json):
        if sys.stdout.isatty():
            return highlight(json, lexers.JsonLexer(), formatters.TerminalFormatter())
        else:
            return json

    def colorize_xml(self, xml):
        if sys.stdout.isatty():
            return highlight(xml, html.XmlLexer(), formatters.TerminalFormatter())
        else:
            return xml

    def _handle_results(self):
        """
        Call back function to be implemented by the CLI.
        Default is to just print the results to standard out
        """
        print(self.colorize_json(self._api_result.text))

    def execute(self):
        """
        Run the steps to execute the CLI
        """

        # Set default arguments from environment variables
        self._get_environment()

        # Call our member function to add command line arguments, child classes that override need
        # to call the ApiCli version first to add standard arguments
        self.add_arguments()

        # Parse the command line arguments
        self._parse_args()

        # Arguments are parsed call back to the instance so that it can extract the command line
        # arguments for its use
        self.get_arguments()

        self.get_api_parameters()
        if self._validate_arguments():
            if self._curl:
                self._curl_output()
            else:
                self._call_api()
                self._handle_results()
        else:
            print(self._message)

    if __name__ == "__main__":
        import doctest
        doctest.testmod()


#
# Copyright 2015 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from boundary import ApiCli

class PluginBase(ApiCli):
    def __init__(self):
        ApiCli.__init__(self)
        self.method = "GET"
        self.path = "v1/plugins"
        self.pluginName = None

    def add_arguments(self):
        ApiCli.add_arguments(self)
        self.parser.add_argument('-n', '--plugin-name', dest='pluginName', action='store', required=True,
                                 metavar="plugin_name", help='Plugin name')

    def get_arguments(self):
        """
        Extracts the specific arguments of this CLI
        """
        ApiCli.get_arguments(self)
        if self.args.pluginName is not None:
            self.pluginName = self.args.pluginName


#
# Copyright 2016 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

class PropertyHandler(object):
    def __init__(self):
        self._properties = None

    def _process_properties(self, properties):
        """
        Transforms the command line properties into python dictionary
        :return:
        """
        if properties is not None:
            self._properties = {}
            for p in properties:
                d = p.split('=')
                self._properties[d[0]] = d[1]

    def _add_property_argument(self, parser, help_text):

        parser.add_argument('-p', '--property', dest='properties', action='append',
                            required=False, metavar='property=value', help=help_text)


#
# Copyright 2015 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import json

from boundary import MetricCommon

"""
Common Base class for defining and update metric definitions
"""

class MetricModify (MetricCommon):

    def __init__(self, update):
        """
        """
        MetricCommon.__init__(self)
        self.update = update
        self.metricName = None
        self.displayName = None
        self.displayNameShort = None
        self.description = None
        self.aggregate = None
        self.unit = None
        self.resolution = None
        self.isDisabled = None
        self.type = None

    def add_arguments(self):
        """
        Add the specific arguments of this CLI
        """
        MetricCommon.add_arguments(self)
        self.parser.add_argument('-n', '--metric-name', dest='metricName', action='store',
                                 required=True, metavar='metric_name', help='Metric identifier')
        self.parser.add_argument('-d', '--display-name', dest='displayName', action='store',
                                 required=True, metavar='display_name', help='Metric display name')
        self.parser.add_argument('-s', '--display-name-short', dest='displayNameShort', action='store',
                                 required=True, metavar='display_short_name', help='Metric short display name')
        self.parser.add_argument('-i', '--description', dest='description', action='store',
                                 required=not self.update, metavar='description', help='Metric description')
        self.parser.add_argument('-g', '--aggregate', dest='aggregate', action='store',
                                 required=True, choices=['avg', 'max', 'min', 'sum'],
                                 help='Metric default aggregate')
        self.parser.add_argument('-u', '--unit', dest='unit', action='store',
                                 required=False, choices=['percent', 'number', 'bytecount', 'duration'],
                                 help='Metric unit')
        self.parser.add_argument('-r', '--resolution', dest='resolution', action='store', metavar='resolution',
                                 required=False, help='Metric default resolution')
        self.parser.add_argument('-y', '--type', dest='type', action='store', default=None,
                                 required=False, metavar='type', help='Sets the type metadata field')
        self.parser.add_argument('-x', '--is-disabled', dest='isDisabled', action='store', default=None,
                                 required=False,
                                 choices=['true', 'false'], help='Enable or disable the metric definition')

    def get_arguments(self):
        """
        Extracts the specific arguments of this CLI
        """
        MetricCommon.get_arguments(self)

        if self.args.metricName is not None:
            self.metricName = self.args.metricName

        if self.args.displayName is not None:
            self.displayName = self.args.displayName

        if self.args.displayNameShort is not None:
            self.displayNameShort = self.args.displayNameShort

        if self.args.description is not None:
            self.description = self.args.description

        if self.args.aggregate is not None:
            self.aggregate = self.args.aggregate

        if self.args.unit is not None:
            self.unit = self.args.unit

        if self.args.resolution is not None:
            self.resolution = self.args.resolution

        if self.args.isDisabled is not None:
            self.isDisabled = self.args.isDisabled

        if self.args.type is not None:
            self.type = self.args.type

        data = {}
        if self.metricName is not None:
            data['name'] = self.metricName
        if self.displayName is not None:
            data['displayName'] = self.displayName
        if self.displayNameShort is not None:
            data['displayNameShort'] = self.displayNameShort
        if self.description is not None:
            data['description'] = self.description
        if self.aggregate is not None:
            data['defaultAggregate'] = self.aggregate
        if self.unit is not None:
            data['unit'] = self.unit
        if self.resolution is not None:
            data['defaultResolutionMS'] = self.resolution
        if self.isDisabled is not None:
            data['isDisabled'] = True if self.isDisabled == 'yes' else False
        if self.type is not None:
            data['type'] =  self.type

        self.path = "v1/metrics/{0}".format(self.metricName)
        self.data = json.dumps(data, sort_keys=True)
        self.headers = {'Content-Type': 'application/json', "Accept": "application/json"}


#
# Copyright 2015 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Implements a command to get the information about an alarm
"""

from boundary import ApiCli
from boundary.alarm_common import result_to_alarm
import json
import requests

"""
Uses the following TrueSight Pulse API:

https://premium-api.boundary.com/v1/alarms/search?name=
"""

class AlarmSearch(ApiCli):
    def __init__(self):
        """
        """
        ApiCli.__init__(self)
        self._alarm_name = None

    def add_arguments(self):
        """
        """
        ApiCli.add_arguments(self)

        self.parser.add_argument('-n', '--alarm-name', dest='alarm_name', action='store', required=True,
                                 metavar='alarm-name', help='Alarm name')

    def get_arguments(self):
        """
        Extracts the specific arguments of this CLI
        """
        ApiCli.get_arguments(self)
        self._alarm_name = self.args.alarm_name if self.args.alarm_name is not None else None

    def handle_key_word_args(self):
        self._alarm_name = self._kwargs['name'] if 'name' in self._kwargs else None

    def get_api_parameters(self):
        self.method = "GET"
        self.path = "v2/alarms/search"
        self.url_parameters = {'name': self._alarm_name}

    def get_description(self):
        """
        """
        return "Searches for an alarm definition by name from a {0} account".format(self.product_name)

    def _handle_api_results(self):
        # Only process if we get HTTP result of 200
        if self._api_result.status_code == requests.codes.ok:
            alarm_result = json.loads(self._api_result.text)
            return result_to_alarm(alarm_result['result'])
        else:
            return None


#
# Copyright 2015 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import json
import logging

"""
Reads and provides access to a plugin.json file the manifest of plugins.
"""

class PluginManifest(object):
    def __init__(self, path="plugin.json"):
        """
        Initialize the PluginManifest instance
        """
        self.path = path
        self.manifest_json = None
        self.manifest = None

    def get_metric_names(self):
        """
        Returns the list of metrics associated with the plugin manifest
        """
        return self.manifest['metrics']

    def read(self):
        """
        Load the metrics file from the given path
        """
        f = open(self.path, "r")
        self.manifest_json = f.read()

    def parse(self):
        """
        Parses the manifest JSON into a dictionary
        """
        self.manifest = json.loads(self.manifest_json)

    def load(self):
        """
        Read the JSON file and parse into a dictionary
        """
        self.read()
        self.parse()

    @property
    def command(self):
        return self.manifest['command']

    @property
    def command_lua(self):
        return self.manifest['command_lua']

    @property
    def description(self):
        return self.manifest['description']

    @property
    def icon(self):
        return self.manifest['icon']

    @property
    def ignore(self):
        return self.manifest['ignore']

    @property
    def metrics(self):
        return self.manifest['metrics']

    @property
    def name(self):
        logging.debug(self.manifest)
        return self.manifest['name']

    @property
    def param_array(self):
        return self.manifest['paramArray']

    @property
    def post_extract(self):
        return self.manifest['postExtract']

    @property
    def post_extract_lua(self):
        return self.manifest['postExtract_lua']

    @property
    def tags(self):
        return self.manifest['tags']

    @property
    def version(self):
        return self.manifest['version']

    def get_manifest(self):
        """
        Returns the dictionary from the parse JSON plugin manifest
        """
        return self.manifest


#
# Copyright 2015 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import json
import logging
import requests

from boundary import ApiCli
from boundary import PluginManifest

"""
Reads the plugin.json manifest file looks up the definition and then outputs a markdown table

Format of output table:

|Metric Name|Metric Identifier|Description
|:-----------|:--------|
"""

class MetricMarkdown(ApiCli):
    def __init__(self):
        ApiCli.__init__(self)
        self.metric_definitions = None
        self.dashboards = None
        self.metrics = None
        self.manifest = None
        self.file_path = None
        self.fields = None

    def add_arguments(self):
        ApiCli.add_arguments(self)
        self.parser.add_argument('-f', '--file', dest='file_path', action='store', required=False, metavar='path',
                                 help='Path to plugin.json manifest. Defaults to plugin.json')

    def get_arguments(self):
        ApiCli.get_arguments(self)

        if self.args.file_path is None:
            self.file_path = 'plugin.json'
        else:
            self.file_path = self.args.file_path

        self.path = "v1/metrics"

        self.load()

    def get_description(self):
        return 'Generates a README file'

    def handle_sesults(self, result):
        if result.status_code == requests.codes.ok:
            result = json.loads(result.text)
            self.metric_definitions = result['result']

            self.generateMarkdown()
        else:
            pass

    def load(self):
        """
        Read the file and parse JSON into dictionary
        """
        manifest = PluginManifest(self.file_path)
        manifest.get()
        self.manifest = manifest.get_manifest()

    def getMetricDefinition(self, name):
        """
        Looks up the metric definition from the definitions from the API call
        """
        metric = None
        for m in self.metric_definitions:
            if m['name'] == name:
                metric = m
                break
        return metric

    def printMetricsHeader(self, m, d):
        """
        Prints out table header based on the size of the data in columns
        """
        mstr = "Metric Name"
        dstr = "Description"

        print('|{0}{1}|{2}{3}|'.format(mstr, ' ' * (m - len(mstr)), dstr, ' ' * (d - len(dstr))))
        print('|:{0}|:{1}|'.format('-' * (m - 1), '-' * (d - 1)))

    def getDashboardColumnLengths(self):
        return (0, 0)

    def getFieldsColumnLengths(self):
        """
        Gets the maximum length of each column in the field table
        """
        nameLen = 0
        descLen = 0
        for f in self.fields:
            nameLen = max(nameLen, len(f['title']))
            descLen = max(descLen, len(f['description']))
        return (nameLen, descLen)

    def getMetricsColumnLengths(self):
        """
        Gets the maximum length of each column
        """
        displayLen = 0
        descLen = 0
        for m in self.metrics:
            displayLen = max(displayLen, len(m['displayName']))
            descLen = max(descLen, len(m['description']))
        return (displayLen, descLen)

    def escapeUnderscores(self):
        """
        Escape underscores so that the markdown is correct
        """
        new_metrics = []
        for m in self.metrics:
            m['name'] = m['name'].replace("_", "\_")
            new_metrics.append(m)
        self.metrics = new_metrics

    def printFieldsHeader(self, f, d):
        """
        Prints out table header based on the size of the data in columns
        """
        fstr = "Field Name"
        dstr = "Description"
        f = max(f, len(fstr))
        d = max(d, len(dstr))

        print('|{0}{1}|{2}{3}|'.format(fstr, ' ' * (f - len(fstr)), dstr, ' ' * (d - len(dstr))))
        print('|:{0}|:{1}|'.format('-' * (f - 1), '-' * (d - 1)))
        return (f, d)

    def printMetrics(self, m, d):
        """
        Prints out table rows based on the size of the data in columns
        """
        for metric in self.metrics:
            mstr = metric['displayName']
            dstr = metric['description']
            mlen = m - len(mstr)
            dlen = d - len(dstr)
            print("|{0}{1}|{2}{3}|".format(mstr, ' ' * mlen, dstr, ' ' * dlen))

    def printFields(self, f, d):
        """
        Prints out table rows based on the size of the data in columns
        """
        for field in self.fields:
            fstr = field["title"]
            dstr = field["description"]
            flen = f - len(fstr)
            dlen = d - len(dstr)
            print("|{0}{1}|{2}{3}|".format(fstr, ' ' * flen, dstr, ' ' * dlen))

    def generateFieldDefinitions(self):
        manifest_fields = self.manifest['paramSchema']
        fields = []
        for f in manifest_fields:
            fields.append(f)
        self.fields = fields

    def generateMetricDefinitions(self):
        manifest_metrics = self.manifest['metrics']
        metrics = []
        for m in manifest_metrics:
            metric = self.getMetricDefinition(m)
            if metric is None:
                logging.error("metric name: {0} not found\n".format(m))
            else:
                metrics.append(metric)

        self.metrics = metrics

    def generateDashboardDefinitions(self):

        if 'dashboards' in self.manifest:
            dashboards = self.manifest['dashboards']

            dashes = []
            for dashboard in dashboards:
                dashes.append(name)
            self.dashboards = dashes

    def outputFieldMarkdown(self):
        """
        Sends the field definitions ot standard out
        """
        f, d = self.getFieldsColumnLengths()

        fc, dc = self.printFieldsHeader(f, d)
        f = max(fc, f)
        d = max(dc, d)
        self.printFields(f, d)

    def outputMetricMarkdown(self):
        """
        Sends the markdown of the metric definitions to standard out
        """
        self.escapeUnderscores()
        m, d = self.getMetricsColumnLengths()

        self.printMetricsHeader(m, d)
        self.printMetrics(m, d)

    def outputDashboardMarkdown(self):
        f,d = self.getDashboardColumnLengths()

    def outputMarkdown(self):
        """
        Sends the markdown to standard out
        """
        self.outputFieldMarkdown()
        self.outputMetricMarkdown()
        self.outputDashboardMarkdown()

    def generateMarkdown(self):
        """
        Look up each of the metrics and then output in Markdown
        """
        self.generateMetricDefinitions()
        self.generateFieldDefinitions()
        self.generateDashboardDefinitions()
        self.outputMarkdown()


#
# Copyright 2015 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import json
import string
from datetime import datetime
from xml.dom import minidom
from xml.etree import ElementTree
from xml.etree.ElementTree import (Element, SubElement, Comment)

import requests
from boundary import ApiCli
from dateutil import parser

"""
Gets measurements from a Boundary account
"""

class MeasurementGet(ApiCli):
    def __init__(self):
        ApiCli.__init__(self)
        self._indent = 4
        self.method = "GET"
        self._metric_name = None
        self.format = None
        self.date_format = None
        # Default to 1 second sample period
        self.sample = 1
        self.source = None
        self.aggregate = None
        self.startTime = None
        self.stopTime = None
        self.now = datetime.now()

    def add_arguments(self):
        """
        Add specific command line arguments for this command
        """

        # Call our parent to add the default arguments
        ApiCli.add_arguments(self)

        # Command specific arguments
        self.parser.add_argument('-f', '--format', dest='format', action='store', required=False,
                                 choices=['csv', 'json', 'raw', 'xml'], help='Output format. Default is raw')
        self.parser.add_argument('-n', '--name', dest='metric_name', action='store', required=True,
                                 metavar="metric_name", help='Metric identifier')
        self.parser.add_argument('-g', '--aggregate', dest='aggregate', action='store', required=False,
                                 choices=['sum', 'avg', 'max', 'min'], help='Metric default aggregate')
        self.parser.add_argument('-r', '--sample', dest='sample', action='store', type=int, metavar="sample",
                                 help='Down sample rate sample in seconds')
        self.parser.add_argument('-s', '--source', dest='source', action='store', metavar="source", required=True,
                                 help='Source of measurement')
        self.parser.add_argument('-b', '--start', dest='start', action='store', required=True, metavar="start",
                                 help='Start of time range as ISO 8601 string or epoch seconds')
        self.parser.add_argument('-d', '--end', dest='end', action='store', metavar="end", required=False,
                                 help='End of time range as ISO 8601 string or epoch seconds')

        self.parser.add_argument('-o', '--date-format', dest='date_format', action='store', metavar="format",
                                 required=False,
                                 help='For CSV, JSON, and XML output formats dates (see Python date.strftime). ' +
                                      'Default format is %%s')

    def get_arguments(self):
        """
        Extracts the specific arguments of this CLI
        """
        ApiCli.get_arguments(self)
        if self.args.metric_name is not None:
            self._metric_name = self.args.metric_name

        if self.args.sample is not None:
            self.sample = self.args.sample

        if self.args.source is not None:
            self.source = self.args.source
        else:
            self.source = None

        if self.args.aggregate is not None:
            self.aggregate = self.args.aggregate
        else:
            self.aggregate = "avg"

        if self.args.format is not None:
            self.format = self.args.format
        else:
            self.format = "json"

        if self.args.date_format is not None:
            self.date_format = self.args.date_format

        start_time = int(self.parse_time_date(self.args.start).strftime("%s"))

        # If the end time is not specified then
        # default to the current time
        if self.args.end is None:
            stop_time = int(self.now.strftime("%s"))
        else:
            stop_time = int(self.parse_time_date(self.args.end).strftime("%s"))

        # Convert to epoch time in milli-seconds
        start_time *= 1000
        stop_time *= 1000

        self.path = "v1/measurements/{0}".format(self._metric_name)
        url_parameters = {"start": str(start_time),
                          "end": str(stop_time),
                          "sample": str(self.sample),
                          "agg": self.aggregate}
        if self.source is not None:
            url_parameters['source'] = self.source
        self.url_parameters = url_parameters

    def parse_time_date(self, s):
        """
        Attempt to parse the passed in string into a valid datetime.
        If we get a parse error then assume the string is an epoch time
        and convert to a datetime.
        """
        try:
            ret = parser.parse(str(s))
        except ValueError:
            try:
                ret = datetime.fromtimestamp(int(s))
            except TypeError:
                ret = None
        return ret

    def _format_timestamp(self, ts):
        if self.date_format is not None:
            fts = datetime.fromtimestamp(int(float(ts)/1000.0)).strftime(self.date_format)
        else:
            fts = datetime.fromtimestamp(int(float(ts)/1000.0)).strftime('%s')

        return str(fts)

    def get_description(self):
        """
        Returns the description of this command
        """
        return 'Retrieves measurement values from a metric in a {0} account'.format(self.product_name)

    def output_csv(self, text):
        """
        Output results in CSV format
        """
        payload = json.loads(text)
        # Print CSV header
        print("{0},{1},{2},{3},{4}".format('timestamp', 'metric', 'aggregate', 'source', 'value'))
        metric_name = self._metric_name
        # Loop through the aggregates one row per timestamp, and 1 or more source/value pairs
        for r in payload['result']['aggregates']['key']:
            timestamp = self._format_timestamp(r[0][0])
            # timestamp = string.strip(timestamp, ' ')
            # timestamp = string.strip(timestamp, "'")
            for s in r[1]:
                print('{0},"{1}","{2}","{3}",{4}'.format(timestamp, metric_name, self.aggregate, s[0], s[1]))

    def output_json(self, text):
        """
        Output results in structured JSON format
        """
        payload = json.loads(text)
        data = []
        metric_name = self._metric_name
        for r in payload['result']['aggregates']['key']:
            timestamp = self._format_timestamp(r[0][0])
            for s in r[1]:
                data.append({
                    "timestamp": timestamp,
                    "metric": metric_name,
                    "aggregate": self.aggregate,
                    "source": s[0],
                    "value": s[1],
                })
        payload = {"data": data}
        out = json.dumps(payload, indent=self._indent, separators=(',', ': '))
        print(self.colorize_json(out))

    def output_raw(self, text):
        """
        Output results in raw JSON format
        """
        payload = json.loads(text)
        out = json.dumps(payload, sort_keys=True, indent=self._indent, separators=(',', ': '))
        print(self.colorize_json(out))

    def output_xml(self, text):
        """
        Output results in JSON format
        """

        # Create the main document nodes
        document = Element('results')
        comment = Comment('Generated by TrueSight Pulse measurement-get CLI')
        document.append(comment)
        aggregates = SubElement(document, 'aggregates')
        aggregate = SubElement(aggregates, 'aggregate')
        measurements = SubElement(aggregate, 'measurements')

        # Parse the JSON result so we can translate to XML
        payload = json.loads(text)

        # Current only support a single metric, if we move to the batch API then
        # we can handle multiple
        metric_name = self._metric_name

        # Loop through the aggregates one row per timestamp, and 1 or more source/value pairs
        for r in payload['result']['aggregates']['key']:
            timestamp = self._format_timestamp(r[0][0])
            for s in r[1]:
                # Each timestamp, metric, source, values is placed in a measure tag
                measure_node = SubElement(measurements, 'measure')
                source = s[0]
                value = str(s[1])
                ts_node = SubElement(measure_node, 'timestamp')
                ts_node.text = str(timestamp)
                metric_node = SubElement(measure_node, 'metric')
                metric_node.text = metric_name
                metric_node = SubElement(measure_node, 'aggregate')
                metric_node.text = self.aggregate
                source_node = SubElement(measure_node, 'source')
                source_node.text = source
                value_node = SubElement(measure_node, 'value')
                value_node.text = value

        rough_string = ElementTree.tostring(document, 'utf-8')
        reparse = minidom.parseString(rough_string)
        output = reparse.toprettyxml(indent=" ")
        print(self.colorize_xml(output))

    def _handle_results(self):
        """
        Call back function to be implemented by the CLI.
        """

        # Only process if we get HTTP result of 200
        if self._api_result.status_code == requests.codes.ok:
            if self.format == "json":
                self.output_json(self._api_result.text)
            elif self.format == "csv":
                self.output_csv(self._api_result.text)
            elif self.format == "raw":
                self.output_raw(self._api_result.text)
            elif self.format == "xml":
                self.output_xml(self._api_result.text)
        else:
            pass


#
# Copyright 2015 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from boundary import ApiCli
import json

class HostgroupModify(ApiCli):
    def __init__(self, update):
        ApiCli.__init__(self)
        self.update = update
        if self.update:
            self.method = "PUT"
        else:
            self.method = "POST"
        self.path = "v1/hostgroups"
        self.sources = None
        self.host_group_name = None

    def get_arguments(self):
        """
        Extracts the specific arguments of this CLI
        """
        ApiCli.get_arguments(self)

        # Get the host group name
        if self.args.host_group_name is not None:
            self.host_group_name = self.args.host_group_name

        # Get the list of sources separated by commas
        if self.args.sources is not None:
            self.sources = self.args.sources

        payload = {}
        if self.host_group_name is not None:
            payload['name'] = self.host_group_name

        if self.sources is not None:
            source_list = str.split(self.sources, ',')
            if 'hostnames' not in payload:
                payload['hostnames'] = []

            for s in source_list:
                payload['hostnames'].append(s)
        self.data = json.dumps(payload, sort_keys=True)
        self.headers = {'Content-Type': 'application/json', "Accept": "application/json"}


# symbol for type checking
import weakref

class Symbol:
    """
    Symbol are semantic name used in a language.
    Could be map to binary symbol but conceptually are more related
    to language symbol table.
    """

    def __init__(self, name, parent=None):
        self.name = name
        self.set_parent(parent)

    def __getstate__(self):
        """
        For pickle don't handle weakrefs...
        """
        state = self.__dict__.copy()
        del state['parent']
        return state

    def set_parent(self, parent) -> object:
        if parent is not None:
            self.parent = weakref.ref(parent)
        else:
            self.parent = None
        return self

    def get_parent(self) -> object:
        """
        Auto deref parent and return the instance.
        """
        if hasattr(self, 'parent') and self.parent is not None:
            return self.parent()
        return None

    def get_scope_list(self) -> list:
        """
        Return the list of all contained scope from global to local
        """
        # by default only return scoped name
        lstparent = [self]
        p = self.get_parent()
        while p is not None:
            lstparent.append(p)
            p = p.get_parent()
        return lstparent

    def get_scope_names(self) -> list:
        """
        Return the list of all contained scope from global to local
        """
        # allow global scope to have an None string instance
        lscope = []
        for scope in reversed(self.get_scope_list()):
            if scope.name is not None:
                # handle fun/block scope decoration
                lscope.append(scope.name)
        return lscope

    # to overload for new language
    def show_name(self) -> str:
        """
        Return a convenient name for type checking
        """
        return ".".join(self.get_scope_names())

    # to overload for new language
    def internal_name(self) -> str:
        """
        Returns the namespace's internal_name.
        """
        unq = "_".join(self.get_scope_names())
        return unq

    def __str__(self) -> str:
        return self.show_name()

    def __repr__(self) -> str:
        return self.internal_name()


import collections

"""An immutable position in a Stream.

index is the index in the Stream.
lineno is the line number of the position.
col_offset is the column offset of the position on the line.
"""
Position = collections.namedtuple('Position', 'index lineno col_offset')

class Cursor:
    """A mutable position in a Stream.

    It can be initialized or set from an immutable Position.
    """
    def __init__(self, position: Position=Position(0, 1, 1)):
        self._maxindex = self._index = position.index
        self._maxcol = self._col_offset = position.col_offset
        self._maxline = self._lineno = position.lineno
        self._eol = []

    @property
    def index(self) -> int:
        """The current index of the cursor."""
        return self._index

    @property
    def lineno(self) -> int:
        """The current line number of the cursor."""
        return self._lineno

    @property
    def col_offset(self) -> int:
        """The current column offset of the cursor."""
        return self._col_offset

    @property
    def position(self) -> Position:
        """The current position of the cursor."""
        return Position(self._index, self._lineno, self._col_offset)

    @position.setter
    def position(self, position: Position):
        self._index = position.index
        self._lineno = position.lineno
        self._col_offset = position.col_offset

    @property
    def max_readed_position(self) -> Position:
        """The index of the deepest character readed."""
        return Position(self._maxindex, self._maxline, self._maxcol)

    def step_next_char(self):
        """Puts the cursor on the next character."""
        self._index += 1
        self._col_offset += 1
        if self._index > self._maxindex:
            self._maxindex = self._index
            self._maxcol = self._col_offset
            self._maxline = self._lineno

    def step_prev_char(self):
        """Puts the cursor on the previous character."""
        self._col_offset -= 1
        self._index -= 1

    def step_next_line(self):
        """Sets cursor as beginning of next line."""
        self._eol.append(self.position)
        self._lineno += 1
        self._col_offset = 0

    def step_prev_line(self):
        """Sets cursor as end of previous line."""
        #TODO(bps): raise explicit error for unregistered eol
        #assert self._eol[-1].index == self._index
        if len(self._eol) > 0:
            self.position = self._eol.pop()

class Tag:
    """Provide capture facilities"""
    def __init__(self, stream: str, begin: int, end=0):
        self._stream = stream
        self._begin = begin
        if end == 0:
            self._end = begin
        else:
            self._end = end

    def set_begin(self, begin: int):
        self._begin = begin

    def set_end(self, end: int):
        self._end = end

    def __str__(self) -> str:
        if self._begin == self._end:
            return ""
        return self._stream[self._begin:self._end]

    def __repr__(self) -> str:
        return "strid:%d %s:%s" % (id(self._stream), self._begin, self._end)

class Stream:
    """Helps keep track of stream processing progress."""
    def __init__(self, content: str=None, name: str=None):
        self._content = content
        self._len = len(content)
        self._name = name
        self._contexts = []
        self._cursor = Cursor()
        # use to store begin:end => value
        self.value_cache = dict()

    def __len__(self) -> int:
        return self._len

    def __getitem__(self, key: int or slice) -> str:
        return self._content.__getitem__(key)

    @property
    def name(self) -> int:
        """Name of the stream."""
        return self._name

    @property
    def eos_index(self) -> int:
        """End Of Stream index."""
        return self._len

    @property
    def index(self) -> int:
        """The current position index."""
        return self._cursor.position.index

    @property
    def lineno(self) -> int:
        """The current position line number."""
        return self._cursor.lineno

    @property
    def col_offset(self) -> int:
        """The current position column offset."""
        return self._cursor.col_offset

    @property
    def peek_char(self) -> str:
        """The current position character value."""
        return self._content[self._cursor.index]

    @property
    def last_readed_line(self) -> str:
        """Usefull string to compute error message."""
        mpos = self._cursor.max_readed_position
        mindex = mpos.index
        # search last \n
        prevline = mindex - 1 if mindex == self.eos_index else mindex
        while prevline >= 0 and self._content[prevline] != '\n':
            prevline -= 1
        # search next \n
        nextline = mindex
        while nextline < self.eos_index and self._content[nextline] != '\n':
            nextline += 1
        last_line = self._content[prevline + 1:nextline]
        return last_line

    def incpos(self, length: int=1) -> int:
        """Increment the cursor to the next character."""
        if length < 0:
            raise ValueError("length must be positive")
        i = 0
        while (i < length):
            if self._cursor.index < self._len:
                if self.peek_char == '\n':
                    self._cursor.step_next_line()
                self._cursor.step_next_char()
            i += 1
        return self._cursor.index

    def decpos(self, length: int=1) -> int:
        if length < 0:
            raise ValueError("length must be positive")
        if (self._cursor.index - length) < 0:
            raise ValueError("can't go before first byte")
        i = 0
        while (i < length):
            if self.peek_char == '\n':
                self._cursor.step_prev_line()
                i += 1
            else:
                self._cursor.step_prev_char()
            i += 1
        return self._cursor.index

    def save_context(self) -> bool:
        """Save current position."""
        self._contexts.append(self._cursor.position)
        return True

    def restore_context(self) -> bool:
        """Rollback to previous saved position."""
        self._cursor.position = self._contexts.pop()
        return False

    def validate_context(self) -> bool:
        """Discard previous saved position."""
        del self._contexts[-1]
        return True


# fmt classes
from pyrser import fmt, meta

from pyrser.type_system.scope import Scope
from pyrser.type_system.type import Type
from pyrser.type_system.evalctx import EvalCtx
from pyrser.type_system.translator import Translator
from pyrser.type_system.translator import MapTargetTranslate
from pyrser.type_system.translator import MapSourceTranslate
from pyrser.type_system.signature import Signature
from pyrser.type_system.val import Val
from pyrser.type_system.var import Var
from pyrser.type_system.fun import Fun

# ======== PRETTY PRINTING ========

@meta.add_method(Scope)
def to_fmt(self) -> fmt.indentable:
    """
    Return an Fmt representation for pretty-printing
    """
    qual = "scope"
    txt = fmt.sep(" ", [qual])
    name = self.show_name()
    if name != "":
        txt.lsdata.append(name)
    if len(self._hsig) > 0 or len(self.mapTypeTranslate) > 0:
        lsb = []
        if len(self.mapTypeTranslate) > 0:
            lsb.append("translate:\n")
            lsb.append(fmt.end("\n", self.mapTypeTranslate.to_fmt()))
        for k in sorted(self._hsig.keys()):
            s = self._hsig[k]
            lsb.append(fmt.end("\n", [s.to_fmt()]))
        block = fmt.block(":\n", "", fmt.tab(lsb))
        txt.lsdata.append(block)
    return txt

@meta.add_method(Type)
def to_fmt(self) -> fmt.indentable:
    """
    Return an Fmt representation for pretty-printing
    """
    qual = "type"
    txt = fmt.sep(" ", [qual])
    txt.lsdata.append(self.show_name())
    if hasattr(self, '_hsig') and len(self._hsig) > 0:
        lsb = []
        for k in sorted(self._hsig.keys()):
            s = self._hsig[k]
            lsb.append(fmt.end("\n", [s.to_fmt()]))
        block = fmt.block(":\n", "", fmt.tab(lsb))
        txt.lsdata.append(block)
    return txt

@meta.add_method(EvalCtx)
def to_fmt(self):
    """
    Return an Fmt representation for pretty-printing
    """
    qual = "evalctx"
    lseval = []
    block = fmt.block(":\n", "", fmt.tab(lseval))
    txt = fmt.sep(" ", [qual, block])
    lseval.append(self._sig.to_fmt())
    if len(self.resolution) > 0:
        lsb = []
        for k in sorted(self.resolution.keys()):
            s = self.resolution[k]
            if s is not None:
                lsb.append(
                    fmt.end(
                        "\n",
                        ["'%s': %s (%s)" % (k, s, s().show_name())]
                    )
                )
            else:
                lsb.append(fmt.end("\n", ["'%s': Unresolved" % (k)]))
        if self._translate_to is not None:
            lsb.append("use translator:")
            lsb.append(self._translate_to.to_fmt())
        if self._variadic_types is not None:
            lsb.append("variadic types:\n")
            arity = self._sig.arity
            for t in self._variadic_types:
                lsb.append("[%d] : %s\n" % (arity, t))
                arity += 1
        lseval.append(fmt.block("\nresolution :\n", "", fmt.tab(lsb)))
    return txt

@meta.add_method(Translator)
def to_fmt(self, with_from=False) -> fmt.indentable:
    """
    Return a Fmt representation of Translator for pretty-printing
    """
    txt = fmt.sep("\n", [
        fmt.sep(
            " ",
            [
                self._type_source,
                "to",
                self._type_target,
                '=',
                self._fun.to_fmt()
            ]
        ),
        self._notify.get_content(with_from)
    ])
    return txt

@meta.add_method(MapTargetTranslate)
def to_fmt(self, with_from=False) -> fmt.indentable:
    txt = fmt.block('{\n', '\n}', [])
    items = fmt.sep('\n---\n', [])
    for k in sorted(self._internal.keys()):
        items.lsdata.append(
            fmt.sep(': ', [
                k,
                fmt.tab([self._internal[k].to_fmt(with_from)])
                ])
        )
    txt.lsdata.append(fmt.tab([items]))
    return txt

@meta.add_method(MapSourceTranslate)
def to_fmt(self, with_from=False) -> fmt.indentable:
    txt = fmt.block('{\n', '\n}', [])
    items = fmt.sep('\n---\n', [])
    for k in sorted(self._internal.keys()):
        items.lsdata.append(
            fmt.sep(': ', [
                k,
                fmt.tab([self._internal[k].to_fmt(with_from)])
                ])
        )
    txt.lsdata.append(fmt.tab([items]))
    return txt

@meta.add_method(Signature)
def to_fmt(self):
    raise TypeError("Signature.to_fmt must be redefined")

@meta.add_method(Val)
def to_fmt(self):
    """
    Return an Fmt representation for pretty-printing
    """
    params = ""
    txt = fmt.sep(" ", ['val'])
    name = self.show_name()
    if name != "":
        txt.lsdata.append(name)
    txt.lsdata.append('(%s)' % self.value)
    txt.lsdata.append(': ' + self.tret)
    return txt

@meta.add_method(Var)
def to_fmt(self):
    """
    Return an Fmt representation for pretty-printing
    """
    params = ""
    txt = fmt.sep(" ", ['var'])
    name = self.show_name()
    if name != "":
        txt.lsdata.append(name)
    txt.lsdata.append(': ' + self.tret)
    return txt

@meta.add_method(Fun)
def to_fmt(self):
    """
    Return an Fmt representation for pretty-printing
    """
    params = ""
    txt = fmt.sep(" ", ['fun'])
    name = self.show_name()
    if name != "":
        txt.lsdata.append(name)
    tparams = []
    if self.tparams is not None:
        tparams = list(self.tparams)
    if self.variadic:
        tparams.append('...')
    params = '(' + ", ".join(tparams) + ')'
    txt.lsdata.append(': ' + params)
    txt.lsdata.append('-> ' + self.tret)
    return txt


"""
This module implements a generic walking method for all pyrser.parsing.node.
"""

from pyrser import meta
from pyrser.parsing import node
from pyrser.ast import state

@meta.add_method(node.Node)
def walk(self, lc: state.LivingContext, user_data=None, parent=None):
    """
    TODO: should_test_type??
    """
    global _cacheid
    # root node autorefence
    if parent is None:
        parent = self
    ## walk attributes
    if hasattr(self, '__dict__') and not isinstance(self, node.ListNode):
        for k in sorted(vars(self).keys()):
            print("RECURS key %s ID %d" % (k, id(getattr(self, k))))
            walk(getattr(self, k), lc, user_data, self)
            # k == ?
            #print('test attr .%s' % k)
            lc.checkAttr(k, self)
            # check precond
            lc.checkEventExpr()
            # do sub Event (for unstrict mode)
            lc.doSubEvent()
    # ...as dict, walk values, match keys
    if hasattr(self, 'keys'):
        for k in sorted(self.keys()):
            #print("RECURS ID %d" % id(self[k]))
            walk(self[k], lc, user_data, self)
            # k == ?
            #print('test key [%s]' % repr(k))
            lc.checkKey(k, self)
            # check precond
            lc.checkEventExpr()
            # do sub Event (for unstrict mode)
            lc.doSubEvent()
    # ...as list, walk values, match indices
    elif not isinstance(self, str) and hasattr(self, '__iter__'):
        idx = 0
        for i in self:
            #print("RECURS ID %d" % id(i))
            walk(i, lc, user_data, self)
            # idx == ?
            #print('test indice [%s]' % str(idx))
            lc.checkIndice(idx, self)
            idx += 1
            # check precond
            lc.checkEventExpr()
            # do sub Event (for unstrict mode)
            lc.doSubEvent()
    # ...type or value
    # type(self) == ?
    #print("test type %s" % type(self))
    lc.checkType(type(self), self, parent)
    # self == ?
    #print("test value %s" % str(self))
    lc.checkValue(self)
    ## Check EVENTS
    # TODO: what if the event do something
    # but don't change current state and default change it!!!
    lc.checkEventExpr()
    #print("RESULT")
    # check Event
    lc.doResultEvent()
    # check Hook
    lc.doResultHook(self, user_data, parent)
    # no transition, fallback to default
    lc.doDefault()
    # maintain the pool of LivingState
    lc.resetLivingState()


""" Module for scoping manipulation.
"""

# scope for type checking
import weakref
from collections import *
from pyrser.type_system.symbol import *
from pyrser.type_system.signature import *
from pyrser.type_system.evalctx import *
from pyrser.type_system.translator import *
from pyrser.parsing.node import *
from pyrser.passes.to_yml import *

StateScope = meta.enum('FREE', 'LINKED', 'EMBEDDED')

# forward just for annotation (not the same id that final type)
class Scope:
    pass

class Scope(Symbol):
    """ Scope of Signature for a Scope/namespace/type etc...
    Basic abstraction of type checking.
    Scope is not a 'pure' python set but something between a set and a dict...
    """

    def __str__(self) -> str:
        import pyrser.type_system.to_fmt
        return str(self.to_fmt())

    def __init__(self, name: str=None, sig: [Signature]=None,
                 state=StateScope.FREE, is_namespace=True):
        """ Unnamed scope for global scope

        A Scope have basically 3 possibles states:

            FREE: it's a standalone Scope, generally the global Scope
            LINKED: the Scope is related to another Scope for type resolution,
                like compound statement
            EMBEDDED: the Scope was added into another Scope,
                it forwards all 'in' calls...like namespacing

        A Scope could or couldn't act like a namespace.
        All Signature added into a 'namespaced' Scope was prefixed
        by the Scope name.
        """
        if name is not None and not isinstance(name, str):
            raise TypeError("name must be a str object.")
        super().__init__(name)
        # during typing, this scope need or not feedback pass
        self.need_feedback = False
        # state of the scope
        self.state = state
        self.is_namespace = is_namespace
        # internal mapping for Type Conversion
        self.mapTypeTranslate = MapSourceTranslate()
        # AST Translator Injector use to add Translator in AST
        # def astTranslatorInjector(old: Node, trans: Translator,
        #               d: Diagnostic, li: LocationInfo) -> Node
        self.astTranslatorInjector = None
        # TODO: ...could be unusable
        self._ntypes = 0
        self._nvars = 0
        self._nfuns = 0
        # internal store of Signature
        self._hsig = {}
        if sig is not None:
            if isinstance(sig, Signature) or isinstance(sig, Scope):
                self.add(sig)
            elif len(sig) > 0:
                self.update(sig)

    def set_parent(self, parent: Scope) -> object:
        Symbol.set_parent(self, parent)
        if parent is not None:
            self.mapTypeTranslate.set_parent(parent.mapTypeTranslate)
        if hasattr(self, 'state') and self.state == StateScope.FREE:
            self.state = StateScope.LINKED
        return self

    def set_name(self, name: str):
        """ You could set the name after construction """
        self.name = name
        # update internal names
        lsig = self._hsig.values()
        self._hsig = {}
        for s in lsig:
            self._hsig[s.internal_name()] = s

    def count_types(self) -> int:
        """ Count subtypes """
        n = 0
        for s in self._hsig.values():
            if type(s).__name__ == 'Type':
                n += 1
        return n

    def count_vars(self) -> int:
        """ Count var define by this scope """
        n = 0
        for s in self._hsig.values():
            if hasattr(s, 'is_var') and s.is_var:
                n += 1
        return n

    def count_funs(self) -> int:
        """ Count function define by this scope """
        n = 0
        for s in self._hsig.values():
            if hasattr(s, 'is_fun') and s.is_fun:
                n += 1
        return n

    def __update_count(self):
        """ Update internal counters """
        self._ntypes = self.count_types()
        self._nvars = self.count_vars()
        self._nfuns = self.count_funs()

    def __repr__(self) -> str:
        """ Internal representation """
        return repr(self._hsig)

    def __getstate__(self):
        """ For pickle, we don't handle weakrefs... """
        state = self.__dict__.copy()
        del state['parent']
        return state

    ## TODO: __getitem__, __setitem__, __delete__??

    # ======== SET OPERATORS OVERLOADING ========
    # in
    def __contains__(self, s: Signature) -> bool:
        """ check if a Signature or a Type is declared in a Scope """
        # fail if in global
        from pyrser.type_system.type import Type
        found = False
        txt = ""
        if isinstance(s, Signature):
            txt = s.internal_name()
        elif isinstance(s, Type):
            txt = s.type_name
        elif isinstance(s, str):
            txt = s
        found = (txt in self._hsig)
        if not found and self.parent is not None:
            if self.state != StateScope.FREE and isinstance(s, Type):
                found = (s.type_name in self.parent())
            if self.state == StateScope.EMBEDDED:
                found = (txt in self.parent())
        return found

    # |=
    def __ior__(self, sig: list or Scope) -> Scope:
        """ \|= operator """
        return self.update(sig)

    def update(self, sig: list or Scope) -> Scope:
        """ Update the Set with values of another Set """
        values = sig
        if hasattr(sig, 'values'):
            values = sig.values()
        for s in values:
            if self.is_namespace:
                s.set_parent(self)
            if isinstance(s, Scope):
                s.state = StateScope.EMBEDDED
            self._hsig[s.internal_name()] = s
        self.__update_count()
        return self

    # |
    def __or__(self, sig: Scope) -> Scope:
        """ | operator """
        return self.union(sig)

    def union(self, sig: Scope) -> Scope:
        """ Create a new Set produce by the union of 2 Set """
        new = Scope(sig=self._hsig.values(), state=self.state)
        new |= sig
        return new

    # &=
    def __iand__(self, sig: Scope) -> Scope:
        """ &= operator """
        return self.intersection_update(sig)

    def intersection_update(self, oset: Scope) -> Scope:
        """ Update Set with common values of another Set """
        keys = list(self._hsig.keys())
        for k in keys:
            if k not in oset:
                del self._hsig[k]
            else:
                self._hsig[k] = oset.get(k)
        return self

    # &
    def __and__(self, sig: Scope) -> Scope:
        """ & operator """
        return self.intersection(sig)

    def intersection(self, sig: Scope) -> Scope:
        """ Create a new Set produce by the intersection of 2 Set """
        new = Scope(sig=self._hsig.values(), state=self.state)
        new &= sig
        return new

    # -=
    def __isub__(self, sig: Scope) -> Scope:
        """ -= operator """
        return self.difference_update(sig)

    def difference_update(self, oset: Scope) -> Scope:
        """ Remove values common with another Set """
        keys = list(self._hsig.keys())
        for k in keys:
            if k in oset:
                del self._hsig[k]
        return self

    # -
    def __sub__(self, sig: Scope) -> Scope:
        """ - operator """
        return self.difference(sig)

    def difference(self, sig: Scope) -> Scope:
        """ Create a new Set produce by a Set subtracted by another Set """
        new = Scope(sig=self._hsig.values(), state=self.state)
        new -= sig
        return new

    # ^=
    def __ixor__(self, sig: Scope) -> Scope:
        """ ^= operator """
        return self.symmetric_difference_update(sig)

    def symmetric_difference_update(self, oset: Scope) -> Scope:
        """ Remove common values
            and Update specific values from another Set
        """
        skey = set()
        keys = list(self._hsig.keys())
        for k in keys:
            if k in oset:
                skey.add(k)
        for k in oset._hsig.keys():
            if k not in skey:
                self._hsig[k] = oset.get(k)
        for k in skey:
            del self._hsig[k]
        return self

    # ^
    def __xor__(self, sig: Scope) -> Scope:
        """ ^ operator """
        return self.symmetric_difference(sig)

    def symmetric_difference(self, sig: Scope) -> Scope:
        """ Create a new Set with values present in only one Set """
        new = Scope(sig=self._hsig.values(), state=self.state)
        new ^= sig
        return new

    # ======== SCOPE MANIPULATION ========
    def add(self, it: Signature) -> bool:
        """ Add it to the Set """
        if isinstance(it, Scope):
            it.state = StateScope.EMBEDDED
        txt = it.internal_name()
        it.set_parent(self)
        if self.is_namespace:
            txt = it.internal_name()
        if txt == "":
            txt = '_' + str(len(self._hsig))
        if txt in self._hsig:
            raise KeyError("Already exists %s" % txt)
        self._hsig[txt] = it
        self.__update_count()
        return True

    def remove(self, it: Signature) -> bool:
        """ Remove it but raise KeyError if not found """
        txt = it.internal_name()
        if txt not in self._hsig:
            raise KeyError(it.show_name() + ' not in Set')
        sig = self._hsig[txt]
        if isinstance(sig, Scope):
            sig.state = StateScope.LINKED
        del self._hsig[txt]
        return True

    def discard(self, it: Signature) -> bool:
        """ Remove it only if present """
        txt = it.internal_name()
        if txt in self._hsig:
            sig = self._hsig[txt]
            if isinstance(sig, Scope):
                sig.state = StateScope.LINKED
            del self._hsig[txt]
            return True
        return False

    def clear(self) -> bool:
        """ Clear all signatures in the Set """
        self._hsig.clear()
        return True

    def pop(self) -> Signature:
        """ Pop a random Signature """
        return self._hsig.popitem()

    # ======== SCOPE ITERATION ========
    def __len__(self) -> int:
        """ Len of the Set """
        return len(self._hsig)

    def values(self) -> [Signature]:
        """ Retrieve all values """
        if self.state == StateScope.EMBEDDED and self.parent is not None:
            return list(self._hsig.values()) + list(self.parent().values())
        else:
            return self._hsig.values()

    def keys(self) -> [str]:
        """ Retrieve all keys """
        return self._hsig.keys()

    # ======== SCOPE QUERY ========
    def first(self) -> Signature:
        """ Retrieve the first Signature ordered by mangling descendant """
        k = sorted(self._hsig.keys())
        return self._hsig[k[0]]

    def last(self) -> Signature:
        """ Retrieve the last Signature ordered by mangling descendant """
        k = sorted(self._hsig.keys())
        return self._hsig[k[-1]]

    def get(self, key: str, default=None) -> Signature:
        """ Get a signature instance by its internal_name """
        item = default
        if key in self._hsig:
            item = self._hsig[key]
        return item

    def get_by_symbol_name(self, name: str) -> Scope:
        """ Retrieve a Set of all signature by symbol name """
        lst = []
        for s in self.values():
            if s.name == name:
                # create an EvalCtx only when necessary
                lst.append(EvalCtx.from_sig(s))
        # include parent
        # TODO: see all case of local redefinition for
        #       global overloads
        # possible algos... take all with different internal_name
        if len(lst) == 0:
            p = self.get_parent()
            if p is not None:
                return p.get_by_symbol_name(name)
        rscope = Scope(sig=lst, state=StateScope.LINKED, is_namespace=False)
        # inherit type/translation from parent
        rscope.set_parent(self)
        return rscope

    def getsig_by_symbol_name(self, name: str) -> Signature:
        """ Retrieve the unique Signature of a symbol.
        Fail if the Signature is not unique
        """
        subscope = self.get_by_symbol_name(name)
        if len(subscope) != 1:
            raise KeyError("%s have multiple candidates in scope" % name)
        v = list(subscope.values())
        return v[0]

    def get_by_return_type(self, tname: str) -> Scope:
        """ Retrieve a Set of all signature by (return) type """
        lst = []
        for s in self.values():
            if hasattr(s, 'tret') and s.tret == tname:
                lst.append(EvalCtx.from_sig(s))
        rscope = Scope(sig=lst, state=StateScope.LINKED, is_namespace=False)
        # inherit type/translation from parent
        rscope.set_parent(self)
        return rscope

    def get_all_polymorphic_return(self) -> bool:
        """ For now, polymorphic return type are handle by symbol artefact.

        --> possible multi-polymorphic but with different constraint attached!
        """
        lst = []
        for s in self.values():
            if hasattr(s, 'tret') and s.tret.is_polymorphic:
                # encapsulate s into a EvalCtx for meta-var resolution
                lst.append(EvalCtx.from_sig(s))
        rscope = Scope(sig=lst, state=StateScope.LINKED, is_namespace=False)
        # inherit type/translation from parent
        rscope.set_parent(self)
        return rscope

    def get_by_params(self, params: [Scope]) -> (Scope, [[Scope]]):
        """ Retrieve a Set of all signature that match the parameter list.
        Return a pair:

            pair[0] the overloads for the functions
            pair[1] the overloads for the parameters
                (a list of candidate list of parameters)
        """
        lst = []
        scopep = []
        # for each of our signatures
        for s in self.values():
            # for each params of this signature
            if hasattr(s, 'tparams'):
                # number of matched params
                mcnt = 0
                # temporary collect
                nbparam_sig = (0 if s.tparams is None else len(s.tparams))
                nbparam_candidates = len(params)
                # don't treat signature too short
                if nbparam_sig > nbparam_candidates:
                    continue
                # don't treat call signature too long if not variadic
                if nbparam_candidates > nbparam_sig and not s.variadic:
                    continue
                tmp = [None] * nbparam_candidates
                variadic_types = []
                for i in range(nbparam_candidates):
                    tmp[i] = Scope(state=StateScope.LINKED)
                    tmp[i].set_parent(self)
                    # match param of the expr
                    if i < nbparam_sig:
                        if params[i].state == StateScope.EMBEDDED:
                            raise ValueError(
                                ("params[%d] of get_by_params is a StateScope."
                                 + "EMBEDDED scope... "
                                 + "read the doc and try a StateScope.FREE"
                                 + " or StateScope.LINKED.") % i
                            )
                        m = params[i].get_by_return_type(s.tparams[i])
                        if len(m) > 0:
                            mcnt += 1
                            tmp[i].update(m)
                        else:
                            # co/contra-variance
                            # we just need to search a t1->t2
                            # and add it into the tree (with/without warnings)
                            t1 = params[i]
                            t2 = s.tparams[i]
                            # if exist a fun (t1) -> t2
                            (is_convertible,
                             signature,
                             translator
                             ) = t1.findTranslationTo(t2)
                            if is_convertible:
                                # add a translator in the EvalCtx
                                signature.use_translator(translator)
                                mcnt += 1
                                nscope = Scope(
                                    sig=[signature],
                                    state=StateScope.LINKED,
                                    is_namespace=False
                                    )
                                nscope.set_parent(self)
                                tmp[i].update(nscope)
                            elif s.tparams[i].is_polymorphic:
                                # handle polymorphic parameter
                                mcnt += 1
                                if not isinstance(params[i], Scope):
                                    raise Exception(
                                        "params[%d] must be a Scope" % i
                                    )
                                tmp[i].update(params[i])
                            else:
                                # handle polymorphic return type
                                m = params[i].get_all_polymorphic_return()
                                if len(m) > 0:
                                    mcnt += 1
                                    tmp[i].update(m)
                    # for variadic extra parameters
                    else:
                        mcnt += 1
                        if not isinstance(params[i], Scope):
                            raise Exception("params[%d] must be a Scope" % i)
                        variadic_types.append(params[i].first().tret)
                        tmp[i].update(params[i])
                # we have match all candidates
                if mcnt == len(params):
                    # select this signature but
                    # box it (with EvalCtx) for type resolution
                    lst.append(EvalCtx.from_sig(s))
                    lastentry = lst[-1]
                    if lastentry.variadic:
                        lastentry.use_variadic_types(variadic_types)
                    scopep.append(tmp)
        rscope = Scope(sig=lst, state=StateScope.LINKED, is_namespace=False)
        # inherit type/translation from parent
        rscope.set_parent(self)
        return (rscope, scopep)

    def addTranslator(self, val: Translator, as_global=False):
        return self.mapTypeTranslate.addTranslator(val, as_global=as_global)

    def addTranslatorInjector(self, ast_method):
        """ Could be redefine in a subscope """
        if self.astTranslatorInjector is not None:
            raise TypeError("Already define a translator injector")
        self.astTranslatorInjector = ast_method

    def callInjector(self, old: Node, trans: Translator) -> Node:
        """ If don't have injector call from parent """
        if self.astTranslatorInjector is None:
            if self.parent is not None:
                # TODO: think if we forward for all StateScope
                # forward to parent scope
                return self.parent().callInjector(old, trans)
            else:
                raise TypeError("Must define an Translator Injector")
        return self.astTranslatorInjector(old, trans)

    def findTranslationTo(self, t2: str) -> (bool, Signature, Translator):
        """ Find an arrow (->)
        aka a function able to translate something to t2
        """
        if not t2.is_polymorphic:
            collect = []
            for s in self.values():
                t1 = s.tret
                if t1.is_polymorphic:
                    continue
                if (s.tret in self.mapTypeTranslate):
                    if (t2 in self.mapTypeTranslate[t1]):
                        collect.append((
                            True,
                            s,
                            self.mapTypeTranslate[t1][t2]
                        ))
            # if len > 1 too many candidates
            if len(collect) == 1:
                return collect[0]
        return (False, None, None)


""" Basic classes for AST manipulation """

import weakref

class Node(dict):
    """Base class for node manipulation."""

    def __bool__(self):
        return True

    def __repr__(self):
        #TODO: use to_yml
        items = []
        if len(self) > 0:
            items.append(repr(self.items()))
        for k, v in vars(self).items():
            items.append("{} = {}".format(k, repr(v)))
        return "{}({})".format(self.__class__.__name__, ', '.join(items))

    def check(self, ndict: dict, info="") -> bool:
        """
        Debug method, help detect cycle and/or
        other incoherence in a tree of Node
        """
        def iscycle(thing, ndict: dict, info: str) -> bool:
            # check if not already here
            idthing = id(thing)
            ndict[info] = idthing
            if idthing not in ndict:
                # add myself
                ndict[idthing] = "%s:%s no cycle" % (type(thing), info)
                return False
            else:
                ndict[idthing] += "\n%s:%s cycle" % (type(thing), info)
                return True

        def recurs(thing, ndict: dict, info: str) -> bool:
            if not iscycle(thing, ndict, info):
                res = False
                if isinstance(thing, list):
                    idx = 0
                    for i in thing:
                        res |= recurs(i, ndict, "%s[%d]" % (info, idx))
                        idx += 1
                elif isinstance(thing, Node):
                    res |= thing.check(ndict, info)
                elif isinstance(thing, dict):
                    for k, v in thing.items():
                        res |= recurs(v, ndict, "%s[%s]" % (info, k))
                return res
            return True
        # add ME FIRST
        if len(ndict) == 0:
            ndict['self'] = id(self)
            info = 'self'
        if not iscycle(self, ndict, info):
            res = False
            if len(self) > 0:
                if hasattr(self, 'keys'):
                    keys = list(self.keys())
                    for k in keys:
                        ndict["[" + repr(k) + "]"] = id(self[k])
                        res |= recurs(self[k], ndict, "%s[%s]" % (info, k))
            keys = list(vars(self).keys())
            for k in keys:
                ndict["." + k] = id(getattr(self, k))
                res |= recurs(getattr(self, k), ndict, "%s.%s" % (info, k))
            return res
        return True

    def clean(self):
        if len(self) > 0:
            keys = list(self.keys())
            for k in keys:
                del self[k]
        keys = list(vars(self).keys())
        for k in keys:
            delattr(self, k)

    def set(self, othernode):
        """allow to completly mutate the node into any subclasses of Node"""
        self.__class__ = othernode.__class__
        self.clean()
        if len(othernode) > 0:
            for k, v in othernode.items():
                self[k] = v
        for k, v in vars(othernode).items():
            setattr(self, k, v)

class ListNodeItem:
    pass

class ListNodeItemIterator:
    pass

class DictNode(dict):
    pass

class TupleNode(list):
    pass

class ListNode:
    allinst = []

    def __init__(self, it: []=None):
        self.cache = None
        self.must_update = False
        self.begin = None
        self.end = None
        if it is not None:
            head = None
            for data in it:
                self.append(data)

    def append(self, d):
        if self.end is None:
            self.begin = ListNodeItem(d)
            self.end = self.begin
            self.must_update = True
        else:
            self.end = self.end.append(d)

    def prepend(self, d):
        if self.begin is None:
            self.begin = ListNodeItem(d)
            self.end = self.begin
            self.must_update = True
        else:
            self.begin = self.begin.prepend(d)

    def __eq__(self, oth) -> bool:
        s1 = len(self)
        s2 = len(oth)
        if s1 != s2:
            return False
        for i1, i2 in zip(self, oth):
            if i1 != i2:
                return False
        return True

    def __str__(self) -> str:
        return repr(self)

    def __repr__(self) -> str:
        txt = []
        for it in self.begin._fwd():
            txt.append(it)
        return repr(txt)

    def dump(self) -> str:
        txt = ""
        for it in self.begin._fwd():
            txt += dump(it)
        return txt

    def __len__(self) -> int:
        self._update()
        return len(self.cache)

    def __iter__(self) -> ListNodeItemIterator:
        return ListNodeItemIterator(self.begin)

    def __reversed__(self) -> ListNodeItemIterator:
        return ListNodeItemIterator(self.end, True)

    def _trueindex(self, k) -> int:
        if k >= 0:
            return k
        if k < 0:
            return len(self.cache) + k

    def _cache(self) -> str:
        txt = "{\n"
        txt += "begin= %d\n" % id(self.begin)
        txt += "end= %d\n" % id(self.end)
        txt += "}\n"
        return txt

    def _update(self):
        if self.cache is None or self.must_update:
            self.cache = {}
            idx = 0
            for it in self.begin._fwd():
                it.thelist = weakref.ref(self)
                self.cache[idx] = it
                idx += 1
            self.must_update = False

    def index(self, data) -> int:
        self._update()
        for k, v in self.cache.items():
            if v.data == data:
                return k
        raise ValueError("%d is not in list" % v)

    def count(self, data) -> int:
        self._update()
        cnt = 0
        for v in self.cache.values():
            if v.data == data:
                cnt += 1
        return cnt

    def get(self, k) -> ListNodeItem:
        if type(k) is not int:
            raise ValueError('Key must be an int')
        self._update()
        k = self._trueindex(k)
        if k not in self.cache:
            raise IndexError("list index out of range")
        return self.cache[k]

    # []
    def __getitem__(self, k) -> object:
        if type(k) is not int and type(k) is not slice:
            raise ValueError('Key must be an int or a slice receive %s' % type(k))
        self._update()
        if type(k) is int:
            k = self._trueindex(k)
            if k not in self.cache:
                raise IndexError("list index out of range")
            return self.cache[k].data
        if type(k) is slice:
            start = k.start
            if start is None:
                start = 0
            stop = k.stop
            if stop is None:
                stop = len(self.cache)
            step = k.step
            if step is None:
                step = 1
            lstmp = []
            for it in range(start, stop, step):
                lstmp.append(self.cache[it].data)
            return ListNode(lstmp)

    # [] =
    def __setitem__(self, k, d):
        if type(k) is not int:
            raise ValueError('Key must be an int')
        self._update()
        k = self._trueindex(k)
        if k not in self.cache:
            raise IndexError("list index out of range")
        self.cache[k].data = d

    # del []
    def __delitem__(self, k):
        if type(k) is not int:
            raise ValueError('Key must be an int')
        self._update()
        k = self._trueindex(k)
        if k not in self.cache:
            raise IndexError("list index out of range")
        self.cache[k].popitem()
        self.must_update = True

class ListNodeItem:
    def __init__(self, data):
        self.data = data
        self.prev = None
        self.next = None
        self.thelist = None

    def __str__(self) -> str:
        return repr(self.data)

    def __repr__(self) -> str:
        return repr(self.data)

    def dump(self) -> str:
        txt = "{\n"
        txt += "prev= %d\n" % id(self.prev)
        txt += "self= %d\n" % id(self)
        txt += "data= %d - %s\n" % (id(self.data), repr(self.data))
        txt += "thelist= %d\n" % id(self.thelist)
        txt += "next= %d\n" % id(self.next)
        txt += "}\n"
        return txt

    def index(self, v) -> int:
        idx = 0
        for it in self._fwd():
            if it.data == v:
                return idx
            idx += 1
        raise ValueError("%d is not in list" % v)

    def count(self, v) -> int:
        cnt = 0
        for it in self._fwd():
            if it.data == v:
                cnt += 1
        return cnt

    def _get_slice(self, sl) -> [object]:
        # TODO: slice getter
        begin, end, step = sl

    # []
    def __getitem__(self, k) -> object:
        if self.thelist is None:
            raise ValueError("Bad initialised ListNode")
        return self.thelist()[k]

    # [] =
    def __setitem__(self, k, d):
        if self.thelist is None:
            raise ValueError("Bad initialised ListNode")
        self.thelist()[k] = d

    # del []
    def __delitem__(self, k):
        if type(k) is not int:
            raise ValueError('Key must be an int')
        gen = self._fwd
        if k < 0:
            gen = self._bwd
        idx = 0
        for it in gen():
            if idx == k:
                it.popitem()
                return
            idx += 1
        raise IndexError("list index out of range")

    def popitem(self) -> object:
        if self.prev is not None:
            self.prev.next = self.next
        if self.next is not None:
            self.next.prev = self.prev
        thelist = self.thelist()
        if thelist is not None:
            if thelist.begin is self:
                thelist.begin = self.next
            if thelist.end is self:
                thelist.end = self.prev
            thelist.must_update = True
        return self.data

    def append(self, data) -> ListNodeItem:
        if self.thelist is None:
            ListNode.allinst.append(ListNode())
            self.thelist = weakref.ref(ListNode.allinst[-1])
        thelist = self.thelist()
        thelist.must_update = True
        new = ListNodeItem(data)
        new.thelist = weakref.ref(thelist)
        new.next = self.next
        self.next = new
        new.prev = self
        if new.next is not None:
            new.next.prev = new
        # update thelist
        if thelist.end is self or thelist.end is None:
            thelist.end = new
        if thelist.begin is None:
            thelist.begin = self
        return new

    def prepend(self, data) -> ListNodeItem:
        if self.thelist is None:
            ListNode.allinst.append(ListNode())
            self.thelist = weakref.ref(ListNode.allinst[-1])
        thelist = self.thelist()
        thelist.must_update = True
        new = ListNodeItem(data)
        new.thelist = weakref.ref(thelist)
        new.prev = self.prev
        self.prev = new
        new.next = self
        if new.prev is not None:
            new.prev.next = new
        # update thelist
        if thelist.begin is self or thelist.begin is None:
            thelist.begin = new
        if thelist.end is None:
            thelist.end = self
        return new

    def values(self):
        """
        in order
        """
        tmp = self
        while tmp is not None:
            yield tmp.data
            tmp = tmp.next

    def rvalues(self):
        """
        in reversed order
        """
        tmp = self
        while tmp is not None:
            yield tmp.data
            tmp = tmp.prev

    def _fwd(self):
        tmp = self
        while tmp is not None:
            yield tmp
            tmp = tmp.next

    def _bwd(self):
        tmp = self
        while tmp is not None:
            yield tmp
            tmp = tmp.prev

class ListNodeItemIterator:
    def __init__(self, current: ListNodeItem, reverse=False):
        self.current = current
        self.attr = 'next'
        if reverse:
            self.attr = 'prev'

    def __iter__(self):
        return self

    def __next__(self):
        if self.current is None:
            raise StopIteration
        data = self.current.data
        self.current = getattr(self.current, self.attr)
        return data

def normalize(ast: Node) -> Node:
    """
    Normalize an AST nodes.

    all builtins containers are replace by referencable subclasses
    """
    res = ast
    typemap = {DictNode, ListNode, TupleNode}
    if type(ast) is dict:
        res = DictNode(ast)
    elif type(ast) is list:
        res = ListNode(ast)
    elif type(ast) is tuple:
        res = TupleNode(ast)
    # in-depth change
    if hasattr(res, 'items'):
        for k, v in res.items():
            res[k] = normalize(v)
    elif hasattr(res, '__getitem__'):
        for idx, v in zip(range(len(res)), res):
            res[idx] = normalize(v)
    if type(res) not in typemap and hasattr(res, '__dict__'):
        subattr = vars(res)
        for k, v in subattr.items():
            setattr(res, k, normalize(v))
    return res