# coding=utf-8
from __future__ import absolute_import, division, print_function, \
  unicode_literals

from abc import ABCMeta, abstractmethod as abstract_method
from importlib import import_module
from inspect import getmembers as get_members, isabstract as is_abstract, \
  isclass as is_class
from pkgutil import walk_packages
from types import ModuleType
from typing import Any, Dict, Mapping, Optional, Text, Union

import filters as f
from six import string_types, with_metaclass

from iota.adapter import BaseAdapter
from iota.exceptions import with_context

__all__ = [
  'BaseCommand',
  'command_registry',
  'discover_commands',
  'CustomCommand',
  'FilterCommand',
  'RequestFilter',
  'ResponseFilter',
]

command_registry = {} # type: Dict[Text, CommandMeta]
"""
Registry of commands, indexed by command name.
"""

def discover_commands(package, recursively=True):
  # type: (Union[ModuleType, Text], bool) -> Dict[Text, 'CommandMeta']
  """
  Automatically discover commands in the specified package.

  :param package:
    Package path or reference.

  :param recursively:
    If True, will descend recursively into sub-packages.

  :return:
    All commands discovered in the specified package, indexed by
    command name (note: not class name).
  """
  # http://stackoverflow.com/a/25562415/
  if isinstance(package, string_types):
    package = import_module(package) # type: ModuleType

  commands = {}

  for _, name, is_package in walk_packages(package.__path__, package.__name__ + '.'):
    # Loading the module is good enough; the CommandMeta metaclass will
    # ensure that any commands in the module get registered.

    # Prefix in name module move to function "walk_packages" for fix
    # conflict with names importing packages
    # Bug https://github.com/iotaledger/iota.lib.py/issues/63
    sub_package = import_module(name)

    # Index any command classes that we find.
    for (_, obj) in get_members(sub_package):
      if is_class(obj) and isinstance(obj, CommandMeta):
        command_name = getattr(obj, 'command')
        if command_name:
          commands[command_name] = obj

    if recursively and is_package:
      commands.update(discover_commands(sub_package))

  return commands

class CommandMeta(ABCMeta):
  """
  Automatically register new commands.
  """
  # noinspection PyShadowingBuiltins
  def __init__(cls, what, bases=None, dict=None):
    super(CommandMeta, cls).__init__(what, bases, dict)

    if not is_abstract(cls):
      command = getattr(cls, 'command')
      if command:
        command_registry[command] = cls

class BaseCommand(with_metaclass(CommandMeta)):
  """
  An API command ready to send to the node.
  """
  command = None # Text

  def __init__(self, adapter):
    # type: (BaseAdapter) -> None
    """
    :param adapter:
      Adapter that will send request payloads to the node.
    """
    self.adapter  = adapter

    self.called   = False
    self.request  = None # type: dict
    self.response = None # type: dict

  def __call__(self, **kwargs):
    # type: (**Any) -> dict
    """
    Sends the command to the node.
    """
    if self.called:
      raise with_context(
        exc = RuntimeError('Command has already been called.'),

        context = {
          'last_request':   self.request,
          'last_response':  self.response,
        },
      )

    self.request = kwargs

    replacement = self._prepare_request(self.request)
    if replacement is not None:
      self.request = replacement

    self.response = self._execute(self.request)

    replacement = self._prepare_response(self.response)
    if replacement is not None:
      self.response = replacement

    self.called = True

    return self.response

  def reset(self):
    # type: () -> None
    """
    Resets the command, allowing it to be called again.
    """
    self.called   = False
    self.request  = None # type: dict
    self.response = None # type: dict

  def _execute(self, request):
    # type: (dict) -> dict
    """
    Sends the request object to the adapter and returns the response.

    The command name will be automatically injected into the request
    before it is sent (note: this will modify the request object).
    """
    request['command'] = self.command
    return self.adapter.send_request(request)

  @abstract_method
  def _prepare_request(self, request):
    # type: (dict) -> Optional[dict]
    """
    Modifies the request before sending it to the node.

    If this method returns a dict, it will replace the request
    entirely.

    Note:  the `command` parameter will be injected later; it is
    not necessary for this method to include it.

    :param request:
      Guaranteed to be a dict, but it might be empty.
    """
    raise NotImplementedError(
      'Not implemented in {cls}.'.format(cls=type(self).__name__),
    )

  @abstract_method
  def _prepare_response(self, response):
    # type: (dict) -> Optional[dict]
    """
    Modifies the response from the node.

    If this method returns a dict, it will replace the response
    entirely.

    :param response:
      Guaranteed to be a dict, but it might be empty.
    """
    raise NotImplementedError(
      'Not implemented in {cls}.'.format(cls=type(self).__name__),
    )

class CustomCommand(BaseCommand):
  """
  Sends an arbitrary command to the node, with no request/response
  validation.

  Useful for executing experimental/undocumented commands.
  """
  def __init__(self, adapter, command):
    # type: (BaseAdapter, Text) -> None
    super(CustomCommand, self).__init__(adapter)

    self.command = command

  def _prepare_request(self, request):
    pass

  def _prepare_response(self, response):
    pass

class RequestFilter(f.FilterChain):
  """
  Template for filter applied to API requests.
  """
  # Be more strict about missing/extra keys for requests, since they
  # tend to come from code that the developer has control over.
  def __init__(
      self,
      filter_map,
      allow_missing_keys  = False,
      allow_extra_keys    = False,
  ):
    super(RequestFilter, self).__init__(
        f.Type(Mapping)
      | f.FilterMapper(filter_map, allow_missing_keys, allow_extra_keys)
    )

  def _apply_none(self):
    # Some commands do accept/require empty requests, but in those
    # cases, the request must be an empty object, not ``None``.
    return self._filter(None, f.Required)

class ResponseFilter(f.FilterChain):
  """
  Template for filter applied to API responses.
  """
  # Be a little looser about missing/extra keys for responses, since we
  # can't control what the node sends us back.
  def __init__(
      self,
      filter_map,
      allow_missing_keys  = True,
      allow_extra_keys    = True,
  ):
    super(ResponseFilter, self).__init__(
        f.Type(Mapping)
      | f.FilterMapper(filter_map, allow_missing_keys, allow_extra_keys)
    )

  def _apply_none(self):
    # If for some reason we don't get a response from the node, and the
    # adapter didn't complain, pretend like the response was an empty
    # object.
    return self._apply({})

class FilterCommand(with_metaclass(ABCMeta, BaseCommand)):
  """
  Uses filters to manipulate request/response values.
  """
  @abstract_method
  def get_request_filter(self):
    # type: () -> Optional[RequestFilter]
    """
    Returns the filter that should be applied to the request (if any).

    Generally, this filter should be strict about validating/converting
    the values in the request, to minimize the chance of an error
    response from the node.
    """
    raise NotImplementedError(
      'Not implemented in {cls}.'.format(cls=type(self).__name__),
    )

  @abstract_method
  def get_response_filter(self):
    # type: () -> Optional[ResponseFilter]
    """
    Returns the filter that should be applied to the response (if any).

    Generally, this filter should be less concerned with validation and
    more concerned with ensuring the response values have the correct
    types, since we can't control what the node sends us.
    """
    raise NotImplementedError(
      'Not implemented in {cls}.'.format(cls=type(self).__name__),
    )

  def _prepare_request(self, request):
    return self._apply_filter(
      value           = request,
      filter_         = self.get_request_filter(),
      failure_message = 'Request failed validation',
    )

  def _prepare_response(self, response):
    return self._apply_filter(
      value           = response,
      filter_         = self.get_response_filter(),
      failure_message = 'Response failed validation',
    )

  @staticmethod
  def _apply_filter(value, filter_, failure_message):
    # type: (dict, Optional[f.BaseFilter], Text) -> dict
    """
    Applies a filter to a value.  If the value does not pass the
    filter, an exception will be raised with lots of contextual info
    attached to it.
    """
    if filter_:
      runner = f.FilterRunner(filter_, value)

      if runner.is_valid():
        return runner.cleaned_data
      else:
        raise with_context(
          exc = ValueError(
            '{message} ({error_codes}) '
            '(`exc.context["filter_errors"]` '
            'contains more information).'.format(
              message     = failure_message,
              error_codes = runner.error_codes,
            ),
          ),

          context = {
            'filter_errors': runner.get_errors(with_context=True),
          },
        )

    return value

# Autodiscover commands in this package.
discover_commands(__name__)


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

from time import sleep
from typing import Container, Optional, Text, Union

from requests import Response, codes
from six import moves as compat, text_type

from iota.adapter import BadApiResponse, HttpAdapter, SplitResult
from iota.exceptions import with_context

__all__ = [
    'SandboxAdapter',
]

STATUS_ABORTED = 'ABORTED'
STATUS_FAILED = 'FAILED'
STATUS_FINISHED = 'FINISHED'
STATUS_QUEUED = 'QUEUED'
STATUS_RUNNING = 'RUNNING'

class SandboxAdapter(HttpAdapter):
    """
    HTTP adapter that sends requests to remote nodes operating in
    "sandbox" mode.

    In sandbox mode, the node will only accept authenticated requests
    from clients, and certain jobs are completed asynchronously.

    Note: for compatibility with Iota APIs, SandboxAdapter still
    operates synchronously; it blocks until it determines that a job has
    completed successfully.

    References:
        - https://github.com/iotaledger/iota.lib.py/issues/19
        - https://github.com/iotaledger/documentation/blob/sandbox/source/index.html.md
    """
    DEFAULT_POLL_INTERVAL = 15
    """
    Number of seconds to wait between requests to check job status.
    """

    DEFAULT_MAX_POLLS = 8
    """
    Maximum number of times to poll for job status before giving up.
    """

    def __init__(
            self,
            uri,
            auth_token,
            poll_interval=DEFAULT_POLL_INTERVAL,
            max_polls=DEFAULT_MAX_POLLS,
    ):
        # type: (Union[Text, SplitResult], Optional[Text], int, int) -> None
        """
        :param uri:
            URI of the node to connect to.
            ``https://` URIs are recommended!

            Note: Make sure the URI specifies the correct path!

            Example:

            - Incorrect: ``https://sandbox.iota:14265``
            - Correct:   ``https://sandbox.iota:14265/api/v1/``

        :param auth_token:
            Authorization token used to authenticate requests.

            Contact the node's maintainer to obtain a token.

            If ``None``, the adapter will not include authorization
            metadata with requests.

        :param poll_interval:
            Number of seconds to wait between requests to check job
            status.  Must be a positive integer.

            Smaller values will cause the adapter to return a result
            sooner (once the node completes the job), but it increases
            traffic to the node (which may trip a rate limiter and/or
            incur additional costs).

        :param max_polls:
            Max number of times to poll for job status before giving up.
            Must be a positive integer.

            This is effectively a timeout setting for asynchronous jobs;
            multiply by ``poll_interval`` to get the timeout duration.
        """
        super(SandboxAdapter, self).__init__(uri)

        if not (isinstance(auth_token, text_type) or (auth_token is None)):
            raise with_context(
                exc=TypeError(
                    '``auth_token`` must be a unicode string or ``None`` '
                    '(``exc.context`` has more info).'
                ),

                context={
                    'auth_token': auth_token,
                },
            )

        if auth_token == '':
            raise with_context(
                exc=ValueError(
                    'Set ``auth_token=None`` if requests do not require '
                    'authorization (``exc.context`` has more info).',
                ),

                context={
                    'auth_token': auth_token,
                },
            )

        if not isinstance(poll_interval, int):
            raise with_context(
                exc=TypeError(
                    '``poll_interval`` must be an int '
                    '(``exc.context`` has more info).',
                ),

                context={
                    'poll_interval': poll_interval,
                },
            )

        if poll_interval < 1:
            raise with_context(
                exc=ValueError(
                    '``poll_interval`` must be > 0 '
                    '(``exc.context`` has more info).',
                ),

                context={
                    'poll_interval': poll_interval,
                },
            )

        if not isinstance(max_polls, int):
            raise with_context(
                exc=TypeError(
                    '``max_polls`` must be an int '
                    '(``exc.context`` has more info).',
                ),

                context={
                    'max_polls': max_polls,
                },
            )

        if max_polls < 1:
            raise with_context(
                exc=ValueError(
                    '``max_polls`` must be > 0 '
                    '(``exc.context`` has more info).',
                ),

                context={
                    'max_polls': max_polls,
                },
            )

        self.auth_token = auth_token  # type: Optional[Text]
        self.poll_interval = poll_interval  # type: int
        self.max_polls = max_polls  # type: int

    @property
    def node_url(self):
        return compat.urllib_parse.urlunsplit((
            self.uri.scheme,
            self.uri.netloc,
            self.uri.path.rstrip('/') + '/commands',
            self.uri.query,
            self.uri.fragment,
        ))

    @property
    def authorization_header(self):
        # type: () -> Text
        """
        Returns the value to use for the ``Authorization`` header.
        """
        return 'token {auth_token}'.format(auth_token=self.auth_token)

    def get_jobs_url(self, job_id):
        # type: (Text) -> Text
        """
        Returns the URL to check job status.

        :param job_id:
            The ID of the job to check.
        """
        return compat.urllib_parse.urlunsplit((
            self.uri.scheme,
            self.uri.netloc,
            self.uri.path.rstrip('/') + '/jobs/' + job_id,
            self.uri.query,
            self.uri.fragment,
        ))

    def send_request(self, payload, **kwargs):
        # type: (dict, dict) -> dict
        if self.auth_token:
            kwargs.setdefault('headers', {})
            kwargs['headers']['Authorization'] = self.authorization_header

        return super(SandboxAdapter, self).send_request(payload, **kwargs)

    def _interpret_response(self, response, payload, expected_status):
        # type: (Response, dict, Container[int]) -> dict
        decoded = super(SandboxAdapter, self)._interpret_response(
            response=response,
            payload=payload,
            expected_status={codes['ok'], codes['accepted']},
        )

        # Check to see if the request was queued for asynchronous
        # execution.
        if response.status_code == codes['accepted']:
            poll_count = 0
            while decoded['status'] in (STATUS_QUEUED, STATUS_RUNNING):
                if poll_count >= self.max_polls:
                    raise with_context(
                        exc=BadApiResponse(
                            '``{command}`` job timed out after '
                            '{duration} seconds '
                            '(``exc.context`` has more info).'.format(
                                command=decoded['command'],
                                duration=self.poll_interval * self.max_polls,
                            ),
                        ),

                        context={
                            'request': payload,
                            'response': decoded,
                        },
                    )

                self._wait_to_poll()
                poll_count += 1

                poll_response = self._send_http_request(
                    headers={'Authorization': self.authorization_header},
                    method='get',
                    payload=None,
                    url=self.get_jobs_url(decoded['id']),
                )

                decoded = super(SandboxAdapter, self)._interpret_response(
                    response=poll_response,
                    payload=payload,
                    expected_status={codes['ok']},
                )

            if decoded['status'] == STATUS_FINISHED:
                return decoded[
                    '{command}Response'.format(command=decoded['command'])]

            raise with_context(
                exc=BadApiResponse(
                    decoded.get('error', {}).get('message')
                    or 'Command {status}: {decoded}'.format(
                        decoded=decoded,
                        status=decoded['status'].lower(),
                    ),
                ),

                context={
                    'request': payload,
                    'response': decoded,
                },
            )

        return decoded

    def _wait_to_poll(self):
        """
        Waits for 1 interval (according to :py:attr:`poll_interval`).

        Implemented as a separate method so that it can be mocked during
        unit tests ("Do you bite your thumb at us, sir?").
        """
        sleep(self.poll_interval)


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

from typing import Generator, List, Optional, Text

from iota.crypto.kerl import Kerl
from iota.crypto.signing import validate_signature_fragments
from iota.transaction.base import Bundle, Transaction

__all__ = [
    'BundleValidator',
]

# In very rare cases, the IOTA protocol may switch hash algorithms.
# When this happens, the IOTA Foundation will create a snapshot, so
# that all new objects on the Tangle use the new hash algorithm.
#
# However, the snapshot will still contain references to addresses
# created using the legacy hash algorithm, so the bundle validator has
# to be able to use that as a fallback when validation fails.
SUPPORTED_SPONGE = Kerl
LEGACY_SPONGE = None  # Curl

class BundleValidator(object):
    """
    Checks a bundle and its transactions for problems.
    """

    def __init__(self, bundle):
        # type: (Bundle) -> None
        super(BundleValidator, self).__init__()

        self.bundle = bundle

        self._errors = []  # type: Optional[List[Text]]
        self._validator = self._create_validator()

    @property
    def errors(self):
        # type: () -> List[Text]
        """
        Returns all errors found with the bundle.
        """
        try:
            self._errors.extend(self._validator)  # type: List[Text]
        except StopIteration:
            pass

        return self._errors

    def is_valid(self):
        # type: () -> bool
        """
        Returns whether the bundle is valid.
        """
        if not self._errors:
            try:
                # We only have to check for a single error to determine
                # if the bundle is valid or not.
                self._errors.append(next(self._validator))
            except StopIteration:
                pass

        return not self._errors

    def _create_validator(self):
        # type: () -> Generator[Text, None, None]
        """
        Creates a generator that does all the work.
        """
        # Group transactions by address to make it easier to iterate
        # over inputs.
        grouped_transactions = self.bundle.group_transactions()

        # Define a few expected values.
        bundle_hash = self.bundle.hash
        last_index = len(self.bundle) - 1

        # Track a few others as we go along.
        balance = 0

        # Check indices and balance first.
        # Note that we use a counter to keep track of the current index,
        # since at this point we can't trust that the transactions have
        # correct ``current_index`` values.
        counter = 0
        for group in grouped_transactions:
            for txn in group:
                balance += txn.value

                if txn.bundle_hash != bundle_hash:
                    yield 'Transaction {i} has invalid bundle hash.'.format(
                        i=counter,
                    )

                if txn.current_index != counter:
                    yield (
                        'Transaction {i} has invalid current index value '
                        '(expected {i}, actual {actual}).'.format(
                            actual=txn.current_index,
                            i=counter,
                        )
                    )

                if txn.last_index != last_index:
                    yield (
                        'Transaction {i} has invalid last index value '
                        '(expected {expected}, actual {actual}).'.format(
                            actual=txn.last_index,
                            expected=last_index,
                            i=counter,
                        )
                    )

                counter += 1

        # Bundle must be balanced (spends must match inputs).
        if balance != 0:
            yield (
                'Bundle has invalid balance '
                '(expected 0, actual {actual}).'.format(
                    actual=balance,
                )
            )

        # Signature validation is only meaningful if the transactions
        # are otherwise valid.
        if not self._errors:
            signature_validation_queue = []  # type: List[List[Transaction]]

            for group in grouped_transactions:
                # Signature validation only applies to inputs.
                if group[0].value >= 0:
                    continue

                validate_group_signature = True
                for j, txn in enumerate(group):
                    if (j > 0) and (txn.value != 0):
                        # Input is malformed; signature fragments after
                        # the first should have zero value.
                        yield (
                            'Transaction {i} has invalid value '
                            '(expected 0, actual {actual}).'.format(
                                actual=txn.value,

                                # If we get to this point, we know that
                                # the ``current_index`` value for each
                                # transaction can be trusted.
                                i=txn.current_index,
                            )
                        )

                        # We won't be able to validate the signature,
                        # but continue anyway, so that we can check that
                        # the other transactions in the group have the
                        # correct ``value``.
                        validate_group_signature = False
                        continue

                # After collecting the signature fragment from each
                # transaction in the group, queue them up to run through
                # the validator.
                #
                # We have to perform signature validation separately so
                # that we can try different algorithms (for
                # backwards-compatibility).
                #
                # References:
                #
                # - https://github.com/iotaledger/kerl#kerl-integration-in-iota
                if validate_group_signature:
                    signature_validation_queue.append(group)

            # Once we've finished checking the attributes from each
            # transaction in the bundle, go back and validate
            # signatures.
            if signature_validation_queue:
                # ``yield from`` is an option here, but for
                # compatibility with Python 2 clients, we will do it the
                # old-fashioned way.
                for error in self._get_bundle_signature_errors(
                        signature_validation_queue
                ):
                    yield error

    def _get_bundle_signature_errors(self, groups):
        # type: (List[List[Transaction]]) -> List[Text]
        """
        Validates the signature fragments in the bundle.

        :return:
            List of error messages.
            If empty, signature fragments are valid.
        """
        # Start with the currently-supported hash algo.
        current_pos = None
        current_errors = []
        for current_pos, group in enumerate(groups):
            error = self._get_group_signature_error(group, SUPPORTED_SPONGE)
            if error:
                current_errors.append(error)

                # Pause and retry with the legacy algo.
                break

        # If validation failed, then go back and try with the legacy
        # algo (only applies if we are currently transitioning to a new
        # algo).
        if current_errors and LEGACY_SPONGE:
            for group in groups:
                # noinspection PyTypeChecker
                if self._get_group_signature_error(group, LEGACY_SPONGE):
                    # Legacy algo doesn't work, either; no point in
                    # continuing.
                    break
            else:
                # If we get here, then we were able to validate the
                # signature fragments successfully using the legacy
                # algorithm.
                return []

        # If we get here, then validation also failed when using the
        # legacy algorithm.

        # At this point, we know that the bundle is invalid, but we will
        # continue validating with the supported algorithm anyway, so
        # that we can return an error message for every invalid input.
        current_errors.extend(filter(None, (
            self._get_group_signature_error(group, SUPPORTED_SPONGE)
            for group in groups[current_pos + 1:]
        )))

        return current_errors

    @staticmethod
    def _get_group_signature_error(group, sponge_type):
        # type: (List[Transaction], type) -> Optional[Text]
        """
        Validates the signature fragments for a group of transactions
        using the specified sponge type.

        Note: this method assumes that the transactions in the group
        have already passed basic validation (see
        :py:meth:`_create_validator`).

        :return:
          - ``None``:  Indicates that the signature fragments are valid.
          - ``Text``:  Error message indicating the fragments are invalid.
        """
        validate_group_signature = validate_signature_fragments(
            fragments=[txn.signature_message_fragment for txn in group],
            hash_=group[0].bundle_hash,
            public_key=group[0].address,
            sponge_type=sponge_type,
        )

        if validate_group_signature:
            return None

        return (
            'Transaction {i} has invalid signature '
            '(using {fragments} fragments).'.format(
                fragments=len(group),
                i=group[0].current_index,
            )
        )


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

from typing import List, Optional

import filters as f

from iota import BadApiResponse, Bundle, BundleHash, Transaction, \
    TransactionHash, TryteString
from iota.commands import FilterCommand, RequestFilter
from iota.commands.core.get_trytes import GetTrytesCommand
from iota.exceptions import with_context
from iota.filters import Trytes
from iota.transaction.validator import BundleValidator

__all__ = [
    'GetBundlesCommand',
]

class GetBundlesCommand(FilterCommand):
    """
    Executes ``getBundles`` extended API command.

    See :py:meth:`iota.api.Iota.get_bundles` for more info.
    """
    command = 'getBundles'

    def get_request_filter(self):
        return GetBundlesRequestFilter()

    def get_response_filter(self):
        pass

    def _execute(self, request):
        transaction_hash = request['transaction']  # type: TransactionHash

        bundle = Bundle(self._traverse_bundle(transaction_hash))
        validator = BundleValidator(bundle)

        if not validator.is_valid():
            raise with_context(
                exc=BadApiResponse(
                    'Bundle failed validation (``exc.context`` has more info).',
                ),

                context={
                    'bundle': bundle,
                    'errors': validator.errors,
                },
            )

        return {
            # Always return a list, so that we have the necessary
            # structure to return multiple bundles in a future
            # iteration.
            'bundles': [bundle],
        }

    def _traverse_bundle(self, txn_hash, target_bundle_hash=None):
        # type: (TransactionHash, Optional[BundleHash]) -> List[Transaction]
        """
        Recursively traverse the Tangle, collecting transactions until
        we hit a new bundle.

        This method is (usually) faster than ``findTransactions``, and
        it ensures we don't collect transactions from replayed bundles.
        """
        trytes = (
            GetTrytesCommand(self.adapter)(hashes=[txn_hash])['trytes']
        )  # type: List[TryteString]

        if not trytes:
            raise with_context(
                exc=BadApiResponse(
                    'Bundle transactions not visible '
                    '(``exc.context`` has more info).',
                ),

                context={
                    'transaction_hash': txn_hash,
                    'target_bundle_hash': target_bundle_hash,
                },
            )

        transaction = Transaction.from_tryte_string(trytes[0])

        if (not target_bundle_hash) and transaction.current_index:
            raise with_context(
                exc=BadApiResponse(
                    '``_traverse_bundle`` started with a non-tail transaction '
                    '(``exc.context`` has more info).',
                ),

                context={
                    'transaction_object': transaction,
                    'target_bundle_hash': target_bundle_hash,
                },
            )

        if target_bundle_hash:
            if target_bundle_hash != transaction.bundle_hash:
                # We've hit a different bundle; we can stop now.
                return []
        else:
            target_bundle_hash = transaction.bundle_hash

        if transaction.current_index == transaction.last_index == 0:
            # Bundle only has one transaction.
            return [transaction]

        # Recursively follow the trunk transaction, to fetch the next
        # transaction in the bundle.
        return [transaction] + self._traverse_bundle(
            txn_hash=transaction.trunk_transaction_hash,
            target_bundle_hash=target_bundle_hash
        )

class GetBundlesRequestFilter(RequestFilter):
    def __init__(self):
        super(GetBundlesRequestFilter, self).__init__({
            'transaction': f.Required | Trytes(TransactionHash),
        })


#!/usr/bin/env python
# coding=utf-8
"""
Launches a Python shell with a configured API client ready to go.
"""
from __future__ import absolute_import, division, print_function, \
    unicode_literals

from argparse import ArgumentParser
from logging import basicConfig, getLogger, DEBUG
from sys import stderr

from six import text_type
from six.moves import http_client

# Import all IOTA symbols into module scope, so that it's more
# convenient for the user.
from iota import *
from iota.adapter import resolve_adapter
from iota.adapter.wrappers import RoutingWrapper
from iota.bin import IotaCommandLineApp

class IotaReplCommandLineApp(IotaCommandLineApp):
    """
    Creates an IOTA API instance and drops the user into a REPL.
    """

    def execute(self, api, **arguments):
        # type: (Iota, ...) -> int
        debug_requests = arguments['debug_requests']
        pow_uri = arguments['pow_uri']

        # If ``pow_uri`` is specified, route POW requests to a separate
        # node.
        if pow_uri:
            pow_adapter = resolve_adapter(pow_uri)

            api.adapter = RoutingWrapper(api.adapter)
            api.adapter.add_route('attachToTangle', pow_adapter)
            api.adapter.add_route('interruptAttachingToTangle', pow_adapter)

        # If ``debug_requests`` is specified, log HTTP requests/responses.
        if debug_requests:
            # Inject a logger into the IOTA HTTP adapter.
            basicConfig(level=DEBUG, stream=stderr)

            logger = getLogger(__name__)
            logger.setLevel(DEBUG)

            api.adapter.set_logger(logger)

            # Turn on debugging for the underlying HTTP library.
            http_client.HTTPConnection.debuglevel = 1

        try:
            self._start_repl(api)
        except KeyboardInterrupt:
            pass

        return 0

    def create_argument_parser(self):
        # type: () -> ArgumentParser
        parser = super(IotaReplCommandLineApp, self).create_argument_parser()

        parser.add_argument(
            '--pow-uri',
            type=text_type,
            default=None,
            dest='pow_uri',
            help='URI of node to send POW requests to.'
        )

        parser.add_argument(
            '--debug',
            action='store_true',
            default=False,
            dest='debug_requests',
            help='If set, log HTTP requests to stderr.'
        )

        return parser

    @staticmethod
    def _start_repl(api):
        # type: (Iota) -> None
        """
        Starts the REPL.
        """
        banner = (
            'IOTA API client for {uri} ({testnet}) '
            'initialized as variable `api`.\n'
            'Type `help(api)` for list of API commands.'.format(
                testnet='testnet' if api.testnet else 'mainnet',
                uri=api.adapter.get_uri(),
            )
        )

        scope_vars = {'api': api}

        try:
            # noinspection PyUnresolvedReferences
            import IPython
        except ImportError:
            # IPython not available; use regular Python REPL.
            from code import InteractiveConsole
            InteractiveConsole(locals=scope_vars).interact(banner, '')
        else:
            print(banner)
            IPython.start_ipython(argv=[], user_ns=scope_vars)

def main():
    """
    Entry point for ``setup.py``.
    """
    IotaReplCommandLineApp().main()

if __name__ == '__main__':
    main()


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

import warnings
from typing import Optional

from iota.crypto import FRAGMENT_LENGTH, HASH_LENGTH, SeedWarning
from iota.crypto.kerl import Kerl
from iota.exceptions import with_context
from iota.transaction.base import Bundle
from iota.types import Hash, TryteString, TrytesCompatible

__all__ = [
    'Digest',
    'PrivateKey',
    'Seed',
]

class Digest(TryteString):
    """
    A private key digest.  Basically the same thing as a regular
    :py:class:`TryteString`, except that it (usually) has a key index
    associated with it.

    Note: in a few cases (e.g., generating multisig addresses), a key
    index is not necessary/available.
    """

    def __init__(self, trytes, key_index=None):
        # type: (TrytesCompatible, Optional[int]) -> None
        super(Digest, self).__init__(trytes)

        # A digest is a series of hashes; its length should reflect
        # that.
        if len(self) % Hash.LEN:
            raise with_context(
                exc=ValueError(
                    'Length of {cls} values '
                    'must be a multiple of {len} trytes.'.format(
                        cls=type(self).__name__,
                        len=Hash.LEN,
                    ),
                ),

                context={
                    'trytes': trytes,
                },
            )

        self.key_index = key_index

    @property
    def security_level(self):
        # type: () -> int
        """
        Returns the number of iterations that were used to generate this
        digest (also known as "security level").
        """
        return len(self) // Hash.LEN

    def as_json_compatible(self):
        # type: () -> dict
        return {
            'trytes': self._trytes.decode('ascii'),
            'key_index': self.key_index,
        }

class Seed(TryteString):
    """
    A TryteString that acts as a seed for crypto functions.

    Note: This class is identical to :py:class:`TryteString`, but it has
    a distinct type so that seeds can be identified in Python code.

    IMPORTANT: For maximum security, a seed must be EXACTLY 81 trytes!

    References:

    - https://iota.stackexchange.com/q/249
    """

    def __init__(self, trytes=None):
        # type: (Optional[TrytesCompatible]) -> None
        if trytes and len(trytes) > Hash.LEN:
            warnings.warn(
                message=(
                    "Seed has inappropriate length! "
                    "(see https://iota.stackexchange.com/q/249 for more info)."
                ),

                category=SeedWarning,
            )

        super(Seed, self).__init__(trytes)

    @classmethod
    def random(cls, length=Hash.LEN):
        """
        Generates a random seed using a CSPRNG.

        :param length:
            Length of seed, in trytes.

            For maximum security, this should always be set to 81, but
            you can change it if you're 110% sure you know what you're
            doing.

            See https://iota.stackexchange.com/q/249 for more info.
        """
        return super(Seed, cls).random(length)

class PrivateKey(TryteString):
    """
    A TryteString that acts as a private key, e.g., for generating
    message signatures, new addresses, etc.
    """

    def __init__(self, trytes, key_index=None, security_level=None):
        # type: (TrytesCompatible, Optional[int], Optional[int]) -> None
        super(PrivateKey, self).__init__(trytes)

        if len(self._trytes) % FRAGMENT_LENGTH:
            raise with_context(
                exc=ValueError(
                    'Length of {cls} values '
                    'must be a multiple of {len} trytes.'.format(
                        cls=type(self).__name__,
                        len=FRAGMENT_LENGTH,
                    ),
                ),

                context={
                    'trytes': self._trytes,
                },
            )

        self.key_index = key_index
        self.security_level = security_level

    def as_json_compatible(self):
        # type: () -> dict
        return {
            'trytes': self._trytes.decode('ascii'),
            'key_index': self.key_index,
            'security_level': self.security_level,
        }

    def get_digest(self):
        # type: () -> Digest
        """
        Generates the digest used to do the actual signing.

        Signing keys can have variable length and tend to be quite long,
        which makes them not-well-suited for use in crypto algorithms.

        The digest is essentially the result of running the signing key
        through a PBKDF, yielding a constant-length hash that can be
        used for crypto.
        """
        hashes_per_fragment = FRAGMENT_LENGTH // Hash.LEN

        key_fragments = self.iter_chunks(FRAGMENT_LENGTH)

        # The digest will contain one hash per key fragment.
        digest = [0] * HASH_LENGTH * len(key_fragments)

        # Iterate over each fragment in the key.
        for i, fragment in enumerate(key_fragments):
            fragment_trits = fragment.as_trits()

            key_fragment = [0] * FRAGMENT_LENGTH
            hash_trits = []

            # Within each fragment, iterate over one hash at a time.
            for j in range(hashes_per_fragment):
                hash_start = j * HASH_LENGTH
                hash_end = hash_start + HASH_LENGTH
                hash_trits = fragment_trits[hash_start:hash_end]

                for k in range(26):
                    sponge = Kerl()
                    sponge.absorb(hash_trits)
                    sponge.squeeze(hash_trits)

                key_fragment[hash_start:hash_end] = hash_trits

            # After processing all of the hashes in the fragment,
            # generate a final hash and append it to the digest.
            #
            # Note that we will do this once per fragment in the key, so
            # the longer the key is, the longer the digest will be.
            sponge = Kerl()
            sponge.absorb(key_fragment)
            sponge.squeeze(hash_trits)

            fragment_hash_start = i * HASH_LENGTH
            fragment_hash_end = fragment_hash_start + HASH_LENGTH

            digest[fragment_hash_start:fragment_hash_end] = hash_trits

        return Digest(TryteString.from_trits(digest), self.key_index)

    def sign_input_transactions(self, bundle, start_index):
        # type: (Bundle, int) -> None
        """
        Signs the inputs starting at the specified index.

        :param bundle:
            The bundle that contains the input transactions to sign.

        :param start_index:
            The index of the first input transaction.

            If necessary, the resulting signature will be split across
            subsequent transactions automatically.
        """

        if not bundle.hash:
            raise with_context(
                exc=ValueError('Cannot sign inputs without a bundle hash!'),

                context={
                    'bundle': bundle,
                    'key_index': self.key_index,
                    'start_index': start_index,
                },
            )

        from iota.crypto.signing import SignatureFragmentGenerator
        signature_fragment_generator = (
            SignatureFragmentGenerator(self, bundle.hash)
        )

        # We can only fit one signature fragment into each transaction,
        # so we have to split the entire signature.
        for j in range(self.security_level):
            # Do lots of validation before we attempt to sign the
            # transaction, and attach lots of context info to any
            # exception.
            #
            # This method is likely to be invoked at a very low level in
            # the application, so if anything goes wrong, we want to
            # make sure it's as easy to troubleshoot as possible!
            try:
                txn = bundle[start_index + j]
            except IndexError as e:
                raise with_context(
                    exc=e,

                    context={
                        'bundle': bundle,
                        'key_index': self.key_index,
                        'current_index': start_index + j,
                    },
                )

            # Only inputs can be signed.
            if txn.value > 0:
                raise with_context(
                    exc=ValueError(
                        'Attempting to sign non-input transaction #{i} '
                        '(value={value}).'.format(
                            i=txn.current_index,
                            value=txn.value,
                        ),
                    ),

                    context={
                        'bundle': bundle,
                        'key_index': self.key_index,
                        'start_index': start_index,
                    },
                )

            if txn.signature_message_fragment:
                raise with_context(
                    exc=ValueError(
                        'Attempting to sign input transaction #{i}, '
                        'but it has a non-empty fragment '
                        '(is it already signed?).'.format(
                            i=txn.current_index,
                        ),
                    ),

                    context={
                        'bundle': bundle,
                        'key_index': self.key_index,
                        'start_index': start_index,
                    },
                )

            txn.signature_message_fragment = next(signature_fragment_generator)


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

from abc import ABCMeta, abstractmethod as abstract_method
from json.encoder import JSONEncoder as BaseJsonEncoder
from typing import Iterable, Mapping

from six import with_metaclass

class JsonSerializable(with_metaclass(ABCMeta)):
    """
    Interface for classes that can be safely converted to JSON.
    """

    @abstract_method
    def as_json_compatible(self):
        """
        Returns a JSON-compatible representation of the object.

        References:

        - :py:class:`iota.json.JsonEncoder`.
        """
        raise NotImplementedError(
            'Not implemented in {cls}.'.format(cls=type(self).__name__),
        )

    def _repr_pretty_(self, p, cycle):
        """
        Makes JSON-serializable objects play nice with IPython's default
        pretty-printer.

        Sadly, :py:func:`pprint.pprint` does not have a similar
        mechanism.

        References:

        - http://ipython.readthedocs.io/en/stable/api/generated/IPython.lib.pretty.html
        - :py:meth:`IPython.lib.pretty.RepresentationPrinter.pretty`
        - :py:func:`pprint._safe_repr`
        """
        class_name = type(self).__name__

        if cycle:
            p.text('{cls}(...)'.format(
                cls=class_name,
            ))
        else:
            with p.group(
                    len(class_name) + 1,
                    '{cls}('.format(cls=class_name),
                    ')',
            ):
                prepared = self.as_json_compatible()

                if isinstance(prepared, Mapping):
                    p.text('**')
                elif isinstance(prepared, Iterable):
                    p.text('*')

                p.pretty(prepared)

class JsonEncoder(BaseJsonEncoder):
    """
    JSON encoder with support for :py:class:`JsonSerializable`.
    """

    def default(self, o):
        if isinstance(o, JsonSerializable):
            return o.as_json_compatible()

        return super(JsonEncoder, self).default(o)


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

from typing import MutableSequence, Optional

from sha3 import keccak_384
from six import PY2

from iota.crypto.kerl import conv
from iota.exceptions import with_context

__all__ = [
    'Kerl',
]

BYTE_HASH_LENGTH = 48
TRIT_HASH_LENGTH = 243

class Kerl(object):
    k = None  # type: keccak_384

    def __init__(self):
        self.reset()

    def absorb(self, trits, offset=0, length=None):
        # type: (MutableSequence[int], int, Optional[int]) -> None
        """
        Absorb trits into the sponge from a buffer.

        :param trits:
            Buffer that contains the trits to absorb.

        :param offset:
            Starting offset in ``trits``.

        :param length:
            Number of trits to absorb.  Defaults to ``len(trits)``.
        """
        # Pad input if necessary, so that it can be divided evenly into
        # hashes.
        # Note that this operation creates a COPY of ``trits``; the
        # incoming buffer is not modified!
        pad = ((len(trits) % TRIT_HASH_LENGTH) or TRIT_HASH_LENGTH)
        trits += [0] * (TRIT_HASH_LENGTH - pad)

        if length is None:
            length = len(trits)

        if length < 1:
            raise with_context(
                exc=ValueError('Invalid length passed to ``absorb``.'),

                context={
                    'trits': trits,
                    'offset': offset,
                    'length': length,
                },
            )

        while offset < length:
            stop = min(offset + TRIT_HASH_LENGTH, length)

            # If we're copying over a full chunk, zero last trit.
            if stop - offset == TRIT_HASH_LENGTH:
                trits[stop - 1] = 0

            signed_nums = conv.convertToBytes(trits[offset:stop])

            # Convert signed bytes into their equivalent unsigned
            # representation, in order to use Python's built-in bytes
            # type.
            unsigned_bytes = bytearray(
                conv.convert_sign(b) for b in signed_nums
            )

            self.k.update(unsigned_bytes)

            offset += TRIT_HASH_LENGTH

    def squeeze(self, trits, offset=0, length=None):
        # type: (MutableSequence[int], int, Optional[int]) -> None
        """
        Squeeze trits from the sponge into a buffer.

        :param trits:
            Buffer that will hold the squeezed trits.

            IMPORTANT:  If ``trits`` is too small, it will be extended!

        :param offset:
            Starting offset in ``trits``.

        :param length:
            Number of trits to squeeze from the sponge.

            If not specified, defaults to :py:data:`TRIT_HASH_LENGTH`
            (i.e., by default, we will try to squeeze exactly 1 hash).
        """
        # Pad input if necessary, so that it can be divided evenly into
        # hashes.
        pad = ((len(trits) % TRIT_HASH_LENGTH) or TRIT_HASH_LENGTH)
        trits += [0] * (TRIT_HASH_LENGTH - pad)

        if length is None:
            # By default, we will try to squeeze one hash.
            # Note that this is different than ``absorb``.
            length = len(trits) or TRIT_HASH_LENGTH

        if length < 1:
            raise with_context(
                exc=ValueError('Invalid length passed to ``squeeze``.'),

                context={
                    'trits': trits,
                    'offset': offset,
                    'length': length,
                },
            )

        while offset < length:
            unsigned_hash = self.k.digest()

            if PY2:
                unsigned_hash = map(ord, unsigned_hash)  # type: ignore

            signed_hash = [conv.convert_sign(b) for b in unsigned_hash]

            trits_from_hash = conv.convertToTrits(signed_hash)
            trits_from_hash[TRIT_HASH_LENGTH - 1] = 0

            stop = min(TRIT_HASH_LENGTH, length - offset)
            trits[offset:offset + stop] = trits_from_hash[0:stop]

            flipped_bytes = bytearray(
                conv.convert_sign(~b) for b in unsigned_hash
            )

            # Reset internal state before feeding back in.
            self.reset()
            self.k.update(flipped_bytes)

            offset += TRIT_HASH_LENGTH

    def reset(self):
        self.k = keccak_384()


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

__all__ = [
    'with_context',
]

def with_context(exc, context):
    # type: (Exception, dict) -> Exception
    """
    Attaches a ``context`` value to an Exception.

    Before:

    .. code-block:: python

        exc = Exception('Frog blast the vent core!')
        exc.context = { ... }
        raise exc

    After:

    .. code-block:: python

        raise with_context(
            exc=Exception('Frog blast the vent core!'),
            context={ ... },
        )
    """
    if not hasattr(exc, 'context'):
        exc.context = {}

    exc.context.update(context)
    return exc


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

from typing import Text

import filters as f
from filters.macros import filter_macro
from six import binary_type, moves as compat, text_type

from iota import Address, TryteString, TrytesCompatible
from iota.crypto.addresses import AddressGenerator

class GeneratedAddress(f.BaseFilter):
    """
    Validates an incoming value as a generated :py:class:`Address` (must
    have ``key_index`` set).
    """
    CODE_NO_KEY_INDEX = 'no_key_index'
    CODE_NO_SECURITY_LEVEL = 'no_security_level'

    templates = {
        CODE_NO_KEY_INDEX:
            'Address must have ``key_index`` attribute set.',

        CODE_NO_SECURITY_LEVEL:
            'Address must have ``security_level`` attribute set.',
    }

    def _apply(self, value):
        value = self._filter(value, f.Type(Address))  # type: Address

        if self._has_errors:
            return None

        if value.key_index is None:
            return self._invalid_value(value, self.CODE_NO_KEY_INDEX)

        if value.security_level is None:
            return self._invalid_value(value, self.CODE_NO_SECURITY_LEVEL)

        return value

class NodeUri(f.BaseFilter):
    """
    Validates a string as a node URI.
    """
    SCHEMES = {'tcp', 'udp'}
    """
    Allowed schemes for node URIs.
    """

    CODE_NOT_NODE_URI = 'not_node_uri'

    templates = {
        CODE_NOT_NODE_URI:
            'This value does not appear to be a valid node URI.',
    }

    def _apply(self, value):
        value = self._filter(value, f.Type(text_type))  # type: Text

        if self._has_errors:
            return None

        parsed = compat.urllib_parse.urlparse(value)

        if parsed.scheme not in self.SCHEMES:
            return self._invalid_value(value, self.CODE_NOT_NODE_URI)

        return value

# noinspection PyPep8Naming
@filter_macro
def SecurityLevel():
    """
    Generates a filter chain for validating a security level.
    """
    return (
            f.Type(int) |
            f.Min(1) |
            f.Max(3) |
            f.Optional(default=AddressGenerator.DEFAULT_SECURITY_LEVEL)
    )

class Trytes(f.BaseFilter):
    """
    Validates a sequence as a sequence of trytes.
    """
    CODE_NOT_TRYTES = 'not_trytes'
    CODE_WRONG_FORMAT = 'wrong_format'

    templates = {
        CODE_NOT_TRYTES: 'This value is not a valid tryte sequence.',
        CODE_WRONG_FORMAT: 'This value is not a valid {result_type}.',
    }

    def __init__(self, result_type=TryteString):
        # type: (type) -> None
        super(Trytes, self).__init__()

        if not isinstance(result_type, type):
            raise TypeError(
                'Invalid result_type for {filter_type} '
                '(expected subclass of TryteString, '
                'actual instance of {result_type}).'.format(
                    filter_type=type(self).__name__,
                    result_type=type(result_type).__name__,
                ),
            )

        if not issubclass(result_type, TryteString):
            raise ValueError(
                'Invalid result_type for {filter_type} '
                '(expected TryteString, actual {result_type}).'.format(
                    filter_type=type(self).__name__,
                    result_type=result_type.__name__,
                ),
            )

        self.result_type = result_type

    def _apply(self, value):
        # noinspection PyTypeChecker
        value = self._filter(
            filter_chain=f.Type(
                (binary_type, bytearray, text_type, TryteString)
            ),

            value=value,
        )  # type: TrytesCompatible

        if self._has_errors:
            return None

        # If the incoming value already has the correct type, then we're
        # done.
        if isinstance(value, self.result_type):
            return value

        # First convert to a generic TryteString, to make sure that the
        # sequence doesn't contain any invalid characters.
        try:
            value = TryteString(value)
        except ValueError:
            return self._invalid_value(
                value=value,
                reason=self.CODE_NOT_TRYTES,
                exc_info=True,
            )

        if self.result_type is TryteString:
            return value

        # Now coerce to the expected type and verify that there are no
        # type-specific errors.
        try:
            return self.result_type(value)
        except ValueError:
            return self._invalid_value(
                value=value,
                reason=self.CODE_WRONG_FORMAT,
                exc_info=True,

                template_vars={
                    'result_type': self.result_type.__name__,
                },
            )

class AddressNoChecksum(Trytes):
    """
    Validates a sequence as an Address, then chops off the checksum if
    present.
    """
    ADDRESS_BAD_CHECKSUM = 'address_bad_checksum'

    templates = {
        ADDRESS_BAD_CHECKSUM:
            'Checksum is {supplied_checksum}, should be {expected_checksum}?',
    }

    def __init__(self):
        super(AddressNoChecksum, self).__init__(result_type=Address)

    def _apply(self, value):
        super(AddressNoChecksum, self)._apply(value)

        if self._has_errors:
            return None

        # Possible it's still just a TryteString.
        if not isinstance(value, Address):
            value = Address(value)

        # Bail out if we have a bad checksum.
        if value.checksum and not value.is_checksum_valid():
            return self._invalid_value(
                value=value,
                reason=self.ADDRESS_BAD_CHECKSUM,
                exc_info=True,

                context={
                    'supplied_checksum': value.checksum,
                    'expected_checksum': value.with_valid_checksum().checksum,
                },
            )

        return Address(value.address)


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

from typing import Iterable, Iterator, List, Optional, Sequence

from six import PY2

from iota.crypto import HASH_LENGTH
from iota.crypto.kerl import Kerl
from iota.crypto.signing import KeyGenerator, normalize
from iota.crypto.types import PrivateKey
from iota.exceptions import with_context
from iota.transaction.base import Bundle, Transaction
from iota.transaction.types import BundleHash, Fragment, Nonce, TransactionHash
from iota.transaction.utils import get_current_timestamp
from iota.trits import add_trits
from iota.types import Address, Tag, TryteString

__all__ = [
    'ProposedBundle',
    'ProposedTransaction',
    'Transfer',
]

class ProposedTransaction(Transaction):
    """
    A transaction that has not yet been attached to the Tangle.

    Provide to :py:meth:`iota.api.Iota.send_transfer` to attach to
    tangle and publish/store.
    """

    def __init__(
            self,
            address,  # type: Address
            value,  # type: int
            tag=None,  # type: Optional[Tag]
            message=None,  # type: Optional[TryteString]
            timestamp=None,  # type: Optional[int]
    ):
        if not timestamp:
            timestamp = get_current_timestamp()

        super(ProposedTransaction, self).__init__(
            address=address,
            tag=Tag(b'') if tag is None else Tag(tag),
            timestamp=timestamp,
            value=value,

            # These values will be populated when the bundle is
            # finalized.
            bundle_hash=None,
            current_index=None,
            hash_=None,
            last_index=None,
            signature_message_fragment=None,
            attachment_timestamp=0,
            attachment_timestamp_lower_bound=0,
            attachment_timestamp_upper_bound=0,

            # These values start out empty; they will be populated when
            # the node does PoW.
            branch_transaction_hash=TransactionHash(b''),
            nonce=Nonce(b''),
            trunk_transaction_hash=TransactionHash(b''),
        )

        self.message = TryteString(b'') if message is None else message

    def as_tryte_string(self):
        # type: () -> TryteString
        """
        Returns a TryteString representation of the transaction.
        """
        if not self.bundle_hash:
            raise with_context(
                exc=RuntimeError(
                    'Cannot get TryteString representation of {cls} instance '
                    'without a bundle hash; call ``bundle.finalize()`` first '
                    '(``exc.context`` has more info).'.format(
                        cls=type(self).__name__,
                    ),
                ),

                context={
                    'transaction': self,
                },
            )

        return super(ProposedTransaction, self).as_tryte_string()

    def increment_legacy_tag(self):
        """
        Increments the transaction's legacy tag, used to fix insecure
        bundle hashes when finalizing a bundle.

        References:

        - https://github.com/iotaledger/iota.lib.py/issues/84
        """
        self._legacy_tag = (
            Tag.from_trits(add_trits(self.legacy_tag.as_trits(), [1]))
        )

Transfer = ProposedTransaction
"""
Follow naming convention of other libs.
https://github.com/iotaledger/iota.lib.py/issues/72
"""

class ProposedBundle(Bundle, Sequence[ProposedTransaction]):
    """
    A collection of proposed transactions, to be treated as an atomic
    unit when attached to the Tangle.
    """

    def __init__(
            self,
            transactions=None,  # type: Optional[Iterable[ProposedTransaction]]
            inputs=None,  # type: Optional[Iterable[Address]]
            change_address=None,  # type: Optional[Address]
    ):
        super(ProposedBundle, self).__init__()

        self._transactions = []  # type: List[ProposedTransaction]

        if transactions:
            for t in transactions:
                self.add_transaction(t)

        if inputs:
            self.add_inputs(inputs)

        self.change_address = change_address

    def __bool__(self):
        # type: () -> bool
        """
        Returns whether this bundle has any transactions.
        """
        return bool(self._transactions)

    # :bc: Magic methods have different names in Python 2.
    if PY2:
        __nonzero__ = __bool__

    def __contains__(self, transaction):
        # type: (ProposedTransaction) -> bool
        return transaction in self._transactions

    def __getitem__(self, index):
        # type: (int) -> ProposedTransaction
        """
        Returns the transaction at the specified index.
        """
        return self._transactions[index]

    def __iter__(self):
        # type: () -> Iterator[ProposedTransaction]
        """
        Iterates over transactions in the bundle.
        """
        return iter(self._transactions)

    def __len__(self):
        # type: () -> int
        """
        Returns te number of transactions in the bundle.
        """
        return len(self._transactions)

    @property
    def balance(self):
        # type: () -> int
        """
        Returns the bundle balance.
        In order for a bundle to be valid, its balance must be 0:

        - A positive balance means that there aren't enough inputs to
          cover the spent amount; add more inputs using
          :py:meth:`add_inputs`.

        - A negative balance means that there are unspent inputs; use
          :py:meth:`send_unspent_inputs_to` to send the unspent inputs
          to a "change" address.
        """
        return sum(t.value for t in self._transactions)

    @property
    def tag(self):
        # type: () -> Tag
        """
        Determines the most relevant tag for the bundle.
        """
        for txn in reversed(self):  # type: ProposedTransaction
            if txn.tag:
                return txn.tag

        return Tag(b'')

    def as_json_compatible(self):
        # type: () -> List[dict]
        """
        Returns a JSON-compatible representation of the object.

        References:

        - :py:class:`iota.json.JsonEncoder`.
        """
        return [txn.as_json_compatible() for txn in self]

    def add_transaction(self, transaction):
        # type: (ProposedTransaction) -> None
        """
        Adds a transaction to the bundle.

        If the transaction message is too long, it will be split
        automatically into multiple transactions.
        """
        if self.hash:
            raise RuntimeError('Bundle is already finalized.')

        if transaction.value < 0:
            raise ValueError('Use ``add_inputs`` to add inputs to the bundle.')

        self._transactions.append(ProposedTransaction(
            address=transaction.address,
            value=transaction.value,
            tag=transaction.tag,
            message=transaction.message[:Fragment.LEN],
            timestamp=transaction.timestamp,
        ))

        # If the message is too long to fit in a single transactions,
        # it must be split up into multiple transactions so that it will
        # fit.
        fragment = transaction.message[Fragment.LEN:]
        while fragment:
            self._transactions.append(ProposedTransaction(
                address=transaction.address,
                value=0,
                tag=transaction.tag,
                message=fragment[:Fragment.LEN],
                timestamp=transaction.timestamp,
            ))

            fragment = fragment[Fragment.LEN:]

    def add_inputs(self, inputs):
        # type: (Iterable[Address]) -> None
        """
        Adds inputs to spend in the bundle.

        Note that each input may require multiple transactions, in order
        to hold the entire signature.

        :param inputs:
            Addresses to use as the inputs for this bundle.

            .. important::
                Must have ``balance`` and ``key_index`` attributes!
                Use :py:meth:`iota.api.get_inputs` to prepare inputs.
        """
        if self.hash:
            raise RuntimeError('Bundle is already finalized.')

        for addy in inputs:
            if addy.balance is None:
                raise with_context(
                    exc=ValueError(
                        'Address {address} has null ``balance`` '
                        '(``exc.context`` has more info).'.format(
                            address=addy,
                        ),
                    ),

                    context={
                        'address': addy,
                    },
                )

            if addy.key_index is None:
                raise with_context(
                    exc=ValueError(
                        'Address {address} has null ``key_index`` '
                        '(``exc.context`` has more info).'.format(
                            address=addy,
                        ),
                    ),

                    context={
                        'address': addy,
                    },
                )

            self._create_input_transactions(addy)

    def send_unspent_inputs_to(self, address):
        # type: (Address) -> None
        """
        Adds a transaction to send "change" (unspent inputs) to the
        specified address.

        If the bundle has no unspent inputs, this method does nothing.
        """
        if self.hash:
            raise RuntimeError('Bundle is already finalized.')

        self.change_address = address

    def finalize(self):
        # type: () -> None
        """
        Finalizes the bundle, preparing it to be attached to the Tangle.
        """
        if self.hash:
            raise RuntimeError('Bundle is already finalized.')

        if not self:
            raise ValueError('Bundle has no transactions.')

        # Quick validation.
        balance = self.balance

        if balance < 0:
            if self.change_address:
                self.add_transaction(ProposedTransaction(
                    address=self.change_address,
                    value=-balance,
                    tag=self.tag,
                ))
            else:
                raise ValueError(
                    'Bundle has unspent inputs (balance: {balance}); '
                    'use ``send_unspent_inputs_to`` to create '
                    'change transaction.'.format(
                        balance=balance,
                    ),
                )
        elif balance > 0:
            raise ValueError(
                'Inputs are insufficient to cover bundle spend '
                '(balance: {balance}).'.format(
                    balance=balance,
                ),
            )

        # Generate bundle hash.
        while True:
            sponge = Kerl()
            last_index = len(self) - 1

            for i, txn in enumerate(self):
                txn.current_index = i
                txn.last_index = last_index

                sponge.absorb(txn.get_signature_validation_trytes().as_trits())

            bundle_hash_trits = [0] * HASH_LENGTH
            sponge.squeeze(bundle_hash_trits)

            bundle_hash = BundleHash.from_trits(bundle_hash_trits)

            # Check that we generated a secure bundle hash.
            # https://github.com/iotaledger/iota.lib.py/issues/84
            if any(13 in part for part in normalize(bundle_hash)):
                # Increment the legacy tag and try again.
                tail_transaction = (
                    self.tail_transaction
                )  # type: ProposedTransaction
                tail_transaction.increment_legacy_tag()
            else:
                break

        # Copy bundle hash to individual transactions.
        for txn in self:
            txn.bundle_hash = bundle_hash

            # Initialize signature/message fragment.
            txn.signature_message_fragment = Fragment(txn.message or b'')

    def sign_inputs(self, key_generator):
        # type: (KeyGenerator) -> None
        """
        Sign inputs in a finalized bundle.
        """
        if not self.hash:
            raise RuntimeError('Cannot sign inputs until bundle is finalized.')

        # Use a counter for the loop so that we can skip ahead as we go.
        i = 0
        while i < len(self):
            txn = self[i]

            if txn.value < 0:
                # In order to sign the input, we need to know the index
                # of the private key used to generate it.
                if txn.address.key_index is None:
                    raise with_context(
                        exc=ValueError(
                            'Unable to sign input {input}; '
                            '``key_index`` is None '
                            '(``exc.context`` has more info).'.format(
                                input=txn.address,
                            ),
                        ),

                        context={
                            'transaction': txn,
                        },
                    )

                if txn.address.security_level is None:
                    raise with_context(
                        exc=ValueError(
                            'Unable to sign input {input}; '
                            '``security_level`` is None '
                            '(``exc.context`` has more info).'.format(
                                input=txn.address,
                            ),
                        ),

                        context={
                            'transaction': txn,
                        },
                    )

                self.sign_input_at(i, key_generator.get_key_for(txn.address))

                i += txn.address.security_level
            else:
                # No signature needed (nor even possible, in some
                # cases); skip this transaction.
                i += 1

    def sign_input_at(self, start_index, private_key):
        # type: (int, PrivateKey) -> None
        """
        Signs the input at the specified index.

        :param start_index:
            The index of the first input transaction.

            If necessary, the resulting signature will be split across
            multiple transactions automatically (i.e., if an input has
            ``security_level=2``, you still only need to call
            :py:meth:`sign_input_at` once).

        :param private_key:
            The private key that will be used to generate the signature.

            .. important::
                Be sure that the private key was generated using the
                correct seed, or the resulting signature will be
                invalid!
        """
        if not self.hash:
            raise RuntimeError('Cannot sign inputs until bundle is finalized.')

        private_key.sign_input_transactions(self, start_index)

    def _create_input_transactions(self, addy):
        # type: (Address) -> None
        """
        Creates transactions for the specified input address.
        """
        self._transactions.append(ProposedTransaction(
            address=addy,
            tag=self.tag,

            # Spend the entire address balance; if necessary, we will
            # add a change transaction to the bundle.
            value=-addy.balance,
        ))

        # Signatures require additional transactions to store, due to
        # transaction length limit.
        # Subtract 1 to account for the transaction we just added.
        for _ in range(addy.security_level - 1):
            self._transactions.append(ProposedTransaction(
                address=addy,
                tag=self.tag,

                # Note zero value; this is a meta transaction.
                value=0,
            ))


# coding=utf-8
from __future__ import absolute_import, division, print_function, \
    unicode_literals

from calendar import timegm as unix_timestamp
from datetime import datetime
from typing import Text

from iota import STANDARD_UNITS
from iota.exceptions import with_context

__all__ = [
    'convert_value_to_standard_unit',
    'get_current_timestamp',
]

def convert_value_to_standard_unit(value, symbol='i'):
    # type: (Text, Text) -> float
    """
    Converts between any two standard units of iota.

    :param value:
        Value (affixed) to convert. For example: '1.618 Mi'.

    :param symbol:
        Unit symbol of iota to convert to. For example: 'Gi'.

    :return:
        Float as units of given symbol to convert to.
    """
    try:
        # Get input value
        value_tuple = value.split()
        amount = float(value_tuple[0])
    except (ValueError, IndexError, AttributeError):
        raise with_context(
            ValueError('Value to convert is not valid.'),

            context={
                'value': value,
            },
        )

    try:
        # Set unit symbols and find factor/multiplier.
        unit_symbol_from = value_tuple[1]
        unit_factor_from = float(STANDARD_UNITS[unit_symbol_from])
        unit_factor_to = float(STANDARD_UNITS[symbol])
    except (KeyError, IndexError):
        # Invalid symbol or no factor
        raise with_context(
            ValueError('Invalid IOTA unit.'),

            context={
                'value': value,
                'symbol': symbol,
            },
        )

    return amount * (unit_factor_from / unit_factor_to)

def get_current_timestamp():
    # type: () -> int
    """
    Returns the current timestamp, used to set ``timestamp`` for new
    :py:class:`ProposedTransaction` objects.

    Split out into a separate function so that it can be mocked during
    unit tests.
    """
    # Python 3.3 introduced a :py:meth:`datetime.timestamp` method, but
    # for compatibility with Python 2, we have to do it the
    # old-fashioned way.
    # http://stackoverflow.com/q/2775864/
    return unix_timestamp(datetime.utcnow().timetuple())


from eth_typing import (
    BLSPubkey,
    BLSSignature,
    Hash32,
)
from eth_utils import (
    big_endian_to_int,
)

from py_ecc.fields import (
    optimized_bls12_381_FQ as FQ,
    optimized_bls12_381_FQ2 as FQ2,
)
from py_ecc.optimized_bls12_381 import (
    Z1,
    Z2,
    b,
    b2,
    field_modulus as q,
    is_inf,
    is_on_curve,
    multiply,
    normalize,
)

from .constants import (
    POW_2_381,
    POW_2_382,
    POW_2_383,
    FQ2_order,
    G2_cofactor,
    eighth_roots_of_unity,
)
from .hash import (
    hash_eth2,
)
from .typing import (
    G1Compressed,
    G1Uncompressed,
    G2Compressed,
    G2Uncompressed,
)

#
# Helpers
#

def modular_squareroot_in_FQ2(value: FQ2) -> FQ2:
    """
    ``modular_squareroot_in_FQ2(x)`` returns the value ``y`` such that ``y**2 % q == x``,
    and None if this is not possible. In cases where there are two solutions,
    the value with higher imaginary component is favored;
    if both solutions have equal imaginary component the value with higher real
    component is favored.
    """
    candidate_squareroot = value ** ((FQ2_order + 8) // 16)
    check = candidate_squareroot ** 2 / value
    if check in eighth_roots_of_unity[::2]:
        x1 = candidate_squareroot / eighth_roots_of_unity[eighth_roots_of_unity.index(check) // 2]
        x2 = -x1
        x1_re, x1_im = x1.coeffs
        x2_re, x2_im = x2.coeffs
        return x1 if (x1_im > x2_im or (x1_im == x2_im and x1_re > x2_re)) else x2
    return None

def _get_x_coordinate(message_hash: Hash32, domain: int) -> FQ2:
    domain_in_bytes = domain.to_bytes(8, 'big')

    # Initial candidate x coordinate
    x_re = big_endian_to_int(hash_eth2(message_hash + domain_in_bytes + b'\x01'))
    x_im = big_endian_to_int(hash_eth2(message_hash + domain_in_bytes + b'\x02'))
    x_coordinate = FQ2([x_re, x_im])  # x_re + x_im * i

    return x_coordinate

def hash_to_G2(message_hash: Hash32, domain: int) -> G2Uncompressed:
    x_coordinate = _get_x_coordinate(message_hash, domain)

    # Test candidate y coordinates until a one is found
    while 1:
        y_coordinate_squared = x_coordinate ** 3 + FQ2([4, 4])  # The curve is y^2 = x^3 + 4(i + 1)
        y_coordinate = modular_squareroot_in_FQ2(y_coordinate_squared)
        if y_coordinate is not None:  # Check if quadratic residue found
            break
        x_coordinate += FQ2([1, 0])  # Add 1 and try again

    return multiply(
        (x_coordinate, y_coordinate, FQ2([1, 0])),
        G2_cofactor,
    )

#
# G1
#
def compress_G1(pt: G1Uncompressed) -> G1Compressed:
    """
    A compressed point is a 384-bit integer with the bit order (c_flag, b_flag, a_flag, x),
    where the c_flag bit is always set to 1,
    the b_flag bit indicates infinity when set to 1,
    the a_flag bit helps determine the y-coordinate when decompressing,
    and the 381-bit integer x is the x-coordinate of the point.
    """
    if is_inf(pt):
        # Set c_flag = 1 and b_flag = 1. leave a_flag = x = 0
        return G1Compressed(POW_2_383 + POW_2_382)
    else:
        x, y = normalize(pt)
        # Record y's leftmost bit to the a_flag
        a_flag = (y.n * 2) // q
        # Set c_flag = 1 and b_flag = 0
        return G1Compressed(x.n + a_flag * POW_2_381 + POW_2_383)

def decompress_G1(z: G1Compressed) -> G1Uncompressed:
    """
    Recovers x and y coordinates from the compressed point.
    """
    # b_flag == 1 indicates the infinity point
    b_flag = (z % POW_2_383) // POW_2_382
    if b_flag == 1:
        return Z1
    x = z % POW_2_381

    # Try solving y coordinate from the equation Y^2 = X^3 + b
    # using quadratic residue
    y = pow((x**3 + b.n) % q, (q + 1) // 4, q)

    if pow(y, 2, q) != (x**3 + b.n) % q:
        raise ValueError(
            "The given point is not on G1: y**2 = x**3 + b"
        )
    # Choose the y whose leftmost bit is equal to the a_flag
    a_flag = (z % POW_2_382) // POW_2_381
    if (y * 2) // q != a_flag:
        y = q - y
    return (FQ(x), FQ(y), FQ(1))

def G1_to_pubkey(pt: G1Uncompressed) -> BLSPubkey:
    z = compress_G1(pt)
    return BLSPubkey(z.to_bytes(48, "big"))

def pubkey_to_G1(pubkey: BLSPubkey) -> G1Uncompressed:
    z = big_endian_to_int(pubkey)
    return decompress_G1(z)

#
# G2
#

def compress_G2(pt: G2Uncompressed) -> G2Compressed:
    """
    The compressed point (z1, z2) has the bit order:
    z1: (c_flag1, b_flag1, a_flag1, x1)
    z2: (c_flag2, b_flag2, a_flag2, x2)
    where
    - c_flag1 is always set to 1
    - b_flag1 indicates infinity when set to 1
    - a_flag1 helps determine the y-coordinate when decompressing,
    - a_flag2, b_flag2, and c_flag2 are always set to 0
    """
    if not is_on_curve(pt, b2):
        raise ValueError(
            "The given point is not on the twisted curve over FQ**2"
        )
    if is_inf(pt):
        return G2Compressed((POW_2_383 + POW_2_382, 0))
    x, y = normalize(pt)
    x_re, x_im = x.coeffs
    y_re, y_im = y.coeffs
    # Record the leftmost bit of y_im to the a_flag1
    # If y_im happens to be zero, then use the bit of y_re
    a_flag1 = (y_im * 2) // q if y_im > 0 else (y_re * 2) // q

    # Imaginary part of x goes to z1, real part goes to z2
    # c_flag1 = 1, b_flag1 = 0
    z1 = x_im + a_flag1 * POW_2_381 + POW_2_383
    # a_flag2 = b_flag2 = c_flag2 = 0
    z2 = x_re
    return G2Compressed((z1, z2))

def decompress_G2(p: G2Compressed) -> G2Uncompressed:
    """
    Recovers x and y coordinates from the compressed point (z1, z2).
    """
    z1, z2 = p

    # b_flag == 1 indicates the infinity point
    b_flag1 = (z1 % POW_2_383) // POW_2_382
    if b_flag1 == 1:
        return Z2

    x1 = z1 % POW_2_381
    x2 = z2
    # x1 is the imaginary part, x2 is the real part
    x = FQ2([x2, x1])
    y = modular_squareroot_in_FQ2(x**3 + b2)
    if y is None:
        raise ValueError("Failed to find a modular squareroot")

    # Choose the y whose leftmost bit of the imaginary part is equal to the a_flag1
    # If y_im happens to be zero, then use the bit of y_re
    a_flag1 = (z1 % POW_2_382) // POW_2_381
    y_re, y_im = y.coeffs
    if (y_im > 0 and (y_im * 2) // q != a_flag1) or (y_im == 0 and (y_re * 2) // q != a_flag1):
        y = FQ2((y * -1).coeffs)

    if not is_on_curve((x, y, FQ2([1, 0])), b2):
        raise ValueError(
            "The given point is not on the twisted curve over FQ**2"
        )
    return (x, y, FQ2([1, 0]))

def G2_to_signature(pt: G2Uncompressed) -> BLSSignature:
    z1, z2 = compress_G2(pt)
    return BLSSignature(
        z1.to_bytes(48, "big") +
        z2.to_bytes(48, "big")
    )

def signature_to_G2(signature: BLSSignature) -> G2Uncompressed:
    p = G2Compressed(
        (big_endian_to_int(signature[:48]), big_endian_to_int(signature[48:]))
    )
    return decompress_G2(p)


from typing import (
    cast,
    Tuple,
    Sequence,
    Union,
    TYPE_CHECKING,
)

if TYPE_CHECKING:
    from py_ecc.fields.field_elements import (  # noqa: F401
        FQ,
    )
    from py_ecc.fields.optimized_field_elements import (  # noqa: F401
        FQ as optimized_FQ,
    )

IntOrFQ = Union[int, "FQ"]

def prime_field_inv(a: int, n: int) -> int:
    """
    Extended euclidean algorithm to find modular inverses for integers
    """
    if a == 0:
        return 0
    lm, hm = 1, 0
    low, high = a % n, n
    while low > 1:
        r = high // low
        nm, new = hm - lm * r, high - low * r
        lm, low, hm, high = nm, new, lm, low
    return lm % n

# Utility methods for polynomial math
def deg(p: Sequence[Union[int, "FQ", "optimized_FQ"]]) -> int:
    d = len(p) - 1
    while p[d] == 0 and d:
        d -= 1
    return d

def poly_rounded_div(a: Sequence[IntOrFQ],
                     b: Sequence[IntOrFQ]) -> Tuple[IntOrFQ, ...]:
    dega = deg(a)
    degb = deg(b)
    temp = [x for x in a]
    o = [0 for x in a]
    for i in range(dega - degb, -1, -1):
        o[i] += int(temp[degb + i] / b[degb])
        for c in range(degb + 1):
            temp[c + i] -= o[c]
    return cast(Tuple[IntOrFQ, ...], tuple(o[:deg(o) + 1]))


#!/usr/bin/env python
# -*- coding: utf 8 -*-
"""
A vocabulary for parsing lithologic or stratigraphic decriptions.

:copyright: 2015 Agile Geoscience
:license: Apache 2.0
"""
from io import open
import json
import warnings
import re
from itertools import islice

from . import defaults

SPECIAL = ['synonyms', 'splitters', 'parts_of_speech', 'abbreviations']

class LexiconError(Exception):
    """
    Generic error class.
    """
    pass

class Lexicon(object):
    """
    A Lexicon is a dictionary of 'types' and regex patterns.

    Most commonly you will just load the default one.

    Args:
        params (dict): The dictionary to use. For an example, refer to the
            default lexicon in ``defaults.py``.
    """

    def __init__(self, params):
        for k, v in params.items():
            k = re.sub(' ', '_', k)
            setattr(self, k, v)

        # Make sure the special attributes are set.
        # We probably don't really need to do this.
        for attr in SPECIAL:
            if not getattr(self, attr, None):
                setattr(self, attr, None)

    def __repr__(self):
        return str(self.__dict__)

    def __str__(self):
        keys = self.__dict__.keys()
        counts = [len(v) for k, v in self.__dict__.items() if v]
        s = "Lexicon("
        for i in zip(keys, counts):
            s += "'{0}': {1} items, ".format(*i)
        s += ")"
        return s

    @classmethod
    def default(cls):
        """
        Makes the default lexicon, as provided in ``defaults.py``.

        Returns:
            Lexicon: The default lexicon.
        """
        return cls(defaults.LEXICON)

    @classmethod
    def from_json_file(cls, filename):
        """
        Load a lexicon from a JSON file.

        Args:
            filename (str): The path to a JSON dump.
        """
        with open(filename, 'r') as fp:
            return cls(json.load(fp))

    def find_word_groups(self, text, category, proximity=2):
        """
        Given a string and a category, finds and combines words into
        groups based on their proximity.

        Args:
            text (str): Some text.
            tokens (list): A list of regex strings.

        Returns:
            list. The combined strings it found.

        Example:
            COLOURS = [r"red(?:dish)?", r"grey(?:ish)?", r"green(?:ish)?"]
            s = 'GREYISH-GREEN limestone with RED or GREY sandstone.'
            find_word_groups(s, COLOURS) --> ['greyish green', 'red', 'grey']
        """
        f = re.IGNORECASE
        words = getattr(self, category)
        regex = re.compile(r'(\b' + r'\b|\b'.join(words) + r'\b)', flags=f)
        candidates = regex.finditer(text)

        starts, ends = [], []
        groups = []

        for item in candidates:
            starts.append(item.span()[0])
            ends.append(item.span()[1])
            groups.append(item.group().lower())

        new_starts = []  # As a check only.
        new_groups = []  # This is what I want.

        skip = False
        for i, g in enumerate(groups):
            if skip:
                skip = False
                continue
            if (i < len(groups)-1) and (starts[i+1]-ends[i] <= proximity):
                if g[-1] == '-':
                    sep = ''  # Don't insert spaces after hyphens.
                else:
                    sep = ' '
                new_groups.append(g + sep + groups[i+1])
                new_starts.append(starts[i])
                skip = True
            else:
                if g not in new_groups:
                    new_groups.append(g)
                    new_starts.append(starts[i])
                skip = False

        return new_groups

    def find_synonym(self, word):
        """
        Given a string and a dict of synonyms, returns the 'preferred'
        word. Case insensitive.

        Args:
            word (str): A word.

        Returns:
            str: The preferred word, or the input word if not found.

        Example:
            >>> syn = {'snake': ['python', 'adder']}
            >>> find_synonym('adder', syn)
            'snake'
            >>> find_synonym('rattler', syn)
            'rattler'

        TODO:
            Make it handle case, returning the same case it received.
        """
        if word and self.synonyms:
            # Make the reverse look-up table.
            reverse_lookup = {}
            for k, v in self.synonyms.items():
                for i in v:
                    reverse_lookup[i.lower()] = k.lower()

            # Now check words against this table.
            if word.lower() in reverse_lookup:
                return reverse_lookup[word.lower()]

        return word

    def expand_abbreviations(self, text):
        """
        Parse a piece of text and replace any abbreviations with their full
        word equivalents. Uses the lexicon.abbreviations dictionary to find
        abbreviations.

        Args:
            text (str): The text to parse.

        Returns:
            str: The text with abbreviations replaced.
        """
        if not self.abbreviations:
            raise LexiconError("No abbreviations in lexicon.")

        def chunks(data, SIZE=25):
            """
            Regex only supports 100 groups for munging callbacks. So we have to
            chunk the abbreviation dicitonary.
            """
            it = iter(data)
            for i in range(0, len(data), SIZE):
                yield {k: data[k] for k in islice(it, SIZE)}

        def cb(g):
            """Regex callback"""
            return self.abbreviations.get(g.group(0)) or g.group(0)

        # Special cases.

        # TODO: We should handle these with a special set of
        # replacements that are made before the others.
        text = re.sub(r'w/', r'wi', text)

        # Main loop.
        for subdict in chunks(self.abbreviations):
            regex = r'(\b' + r'\b)|(\b'.join(subdict.keys()) + r'\b)'
            text = re.sub(regex, cb, text)

        return text

    def get_component(self, text, required=False, first_only=True):
        """
        Takes a piece of text representing a lithologic description for one
        component, e.g. "Red vf-f sandstone" and turns it into a dictionary
        of attributes.

        TODO:
            Generalize this so that we can use any types of word, as specified
            in the lexicon.
        """
        component = {}

        for i, (category, words) in enumerate(self.__dict__.items()):

            # There is probably a more elegant way to do this.
            if category in SPECIAL:
                # There are special entries in the lexicon.
                continue

            groups = self.find_word_groups(text, category)

            if groups and first_only:
                groups = groups[:1]
            elif groups:
                # groups = groups
                pass
            else:
                groups = [None]
                if required:
                    with warnings.catch_warnings():
                        warnings.simplefilter("always")
                        w = "No lithology in lexicon matching '{0}'"
                        warnings.warn(w.format(text))

            filtered = [self.find_synonym(i) for i in groups]
            if first_only:
                component[category] = filtered[0]
            else:
                component[category] = filtered

        return component

    def split_description(self, text):
        """
        Split a description into parts, each of which can be turned into
        a single component.
        """
        # Protect some special sequences.
        t = re.sub(r'(\d) ?in\. ', r'\1 inch ', text)  # Protect.
        t = re.sub(r'(\d) ?ft\. ', r'\1 feet ', t)  # Protect.

        # Transform all part delimiters to first splitter.
        words = getattr(self, 'splitters')
        try:
            splitter = words[0].strip()
        except:
            splitter = 'with'
        t = re.sub(r'\,?\;?\.? ?((under)?(less than)? \d+%) (?=\w)', r' '+splitter+' \1 ', t)

        # Split.
        f = re.IGNORECASE
        pattern = re.compile(r'(?:' + r'|'.join(words) + r')', flags=f)
        parts = filter(None, pattern.split(t))

        return [i.strip() for i in parts]

    @property
    def categories(self):
        """
        Lists the categories in the lexicon, except the
        optional categories.

        Returns:
            list: A list of strings of category names.
        """
        keys = [k for k in self.__dict__.keys() if k not in SPECIAL]
        return keys