# -*- coding: utf-8 -*-
#
# This file contains code from python-tabulate, modified for SolveBio
#
# Copyright © 2011-2013 Sergey Astanin
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
from __future__ import print_function
from __future__ import absolute_import

from six.moves import map
from six.moves import range
from six.moves import zip
from six import string_types

from collections import namedtuple
from platform import python_version_tuple
import re
from .printing import TTY_COLS

if python_version_tuple()[0] < "3":
    from itertools import izip_longest
    _none_type = type(None)
    _int_type = int
    _float_type = float
    _text_type = str
    _binary_type = str
else:
    from itertools import zip_longest as izip_longest
    from functools import reduce
    _none_type = type(None)
    _int_type = int
    _float_type = float
    _text_type = str
    _binary_type = bytes

__all__ = ["tabulate"]
__version__ = "0.6"

Line = namedtuple("Line", ["begin", "hline", "sep", "end"])
DataRow = namedtuple("DataRow", ["begin", "sep", "end"])
TableFormat = namedtuple("TableFormat", ["lineabove", "linebelowheader",
                                         "linebetweenrows", "linebelow",
                                         "headerrow", "datarow",
                                         "padding", "usecolons",
                                         "with_header_hide",
                                         "without_header_hide"])
_format_defaults = {"padding": 0,
                    "usecolons": False,
                    "with_header_hide": [],
                    "without_header_hide": []}

_table_formats = {"simple":
                  TableFormat(lineabove=None,
                              linebelowheader=Line("", "-", "  ", ""),
                              linebetweenrows=None,
                              linebelow=Line("", "-", "  ", ""),
                              headerrow=DataRow("", "  ", ""),
                              datarow=DataRow("", "  ", ""),
                              padding=0,
                              usecolons=False,
                              with_header_hide=["linebelow"],
                              without_header_hide=[]),
                  "plain":
                  TableFormat(None, None, None, None,
                              DataRow("", "  ", ""), DataRow("", "  ", ""),
                              **_format_defaults),
                  "grid":
                  TableFormat(lineabove=Line("+", "-", "+", "+"),
                              linebelowheader=Line("+", "=", "+", "+"),
                              linebetweenrows=Line("+", "-", "+", "+"),
                              linebelow=Line("+", "-", "+", "+"),
                              headerrow=DataRow("|", "|", "|"),
                              datarow=DataRow("|", "|", "|"),
                              padding=1,
                              usecolons=False,
                              with_header_hide=[],
                              without_header_hide=["linebelowheader"]),
                  "pipe":
                  TableFormat(lineabove=None,
                              linebelowheader=Line("|", "-", "|", "|"),
                              linebetweenrows=None,
                              linebelow=None,
                              headerrow=DataRow("|", "|", "|"),
                              datarow=DataRow("|", "|", "|"),
                              padding=1,
                              usecolons=True,
                              with_header_hide=[],
                              without_header_hide=[]),
                  "orgmode":
                  TableFormat(lineabove=None,
                              linebelowheader=Line("|", "-", "+", "|"),
                              linebetweenrows=None,
                              linebelow=None,
                              headerrow=DataRow("|", "|", "|"),
                              datarow=DataRow("|", "|", "|"),
                              padding=1,
                              usecolons=False,
                              with_header_hide=[],
                              without_header_hide=["linebelowheader"])}

_invisible_codes = re.compile(r'\x1b\[\d*m')  # ANSI color codes

def simple_separated_format(separator):
    """
    Construct a simple TableFormat with columns separated by a separator.

    >>> tsv = simple_separated_format("\t") ; \
        tabulate([["foo", 1], ["spam", 23]], \
            tablefmt=tsv) == 'foo \\t 1\\nspam\\t23'
    True

    """
    return TableFormat(None, None, None, None,
                       headerrow=None, datarow=DataRow('', separator, ''),
                       **_format_defaults)

def _isconvertible(conv, string):
    try:
        conv(string)  # noqa
        return True
    except (TypeError, ValueError):
        return False

def _isnumber(string):
    """
    >>> _isnumber("123.45")
    True
    >>> _isnumber("123")
    True
    >>> _isnumber("spam")
    False
    """
    return _isconvertible(float, string)

def _isint(string):
    """
    >>> _isint("123")
    True
    >>> _isint("123.45")
    False
    """
    return type(string) is int or \
        (isinstance(string, _binary_type) or
         isinstance(string, string_types)) and \
        _isconvertible(int, string)

def _type(string, has_invisible=True):
    """
    The least generic type (type(None), int, float, str, unicode).

    >>> _type(None) is type(None)
    True
    >>> _type("foo") is type("")
    True
    >>> _type("1") is type(1)
    True
    >>> _type('\x1b[31m42\x1b[0m') is type(42)
    True
    >>> _type('\x1b[31m42\x1b[0m') is type(42)
    True

    """

    if has_invisible and \
       (isinstance(string, _text_type) or isinstance(string, _binary_type)):
        string = _strip_invisible(string)

    if string is None:
        return _none_type
    elif _isint(string):
        return _int_type
    elif _isnumber(string):
        return float
    elif isinstance(string, _binary_type):
        return _binary_type
    else:
        return _text_type

def _afterpoint(string):
    """
    Symbols after a decimal point, -1 if the string lacks the decimal point.

    >>> _afterpoint("123.45")
    2
    >>> _afterpoint("1001")
    -1
    >>> _afterpoint("eggs")
    -1
    >>> _afterpoint("123e45")
    2

    """
    if _isnumber(string):
        if _isint(string):
            return -1
        else:
            pos = string.rfind(".")
            pos = string.lower().rfind("e") if pos < 0 else pos
            if pos >= 0:
                return len(string) - pos - 1
            else:
                return -1  # no point
    else:
        return -1  # not a number

def _padleft(width, s, has_invisible=True):
    """
    Flush right.

    >>> _padleft(6, '\u044f\u0439\u0446\u0430') \
        == '  \u044f\u0439\u0446\u0430'
    True

    """
    iwidth = width + len(s) - len(_strip_invisible(s)) \
        if has_invisible else width
    fmt = "{0:>%ds}" % iwidth
    return fmt.format(s)

def _padright(width, s, has_invisible=True):
    """
    Flush left.

    >>> _padright(6, '\u044f\u0439\u0446\u0430') \
        == '\u044f\u0439\u0446\u0430  '
    True

    """
    iwidth = width + len(s) - len(_strip_invisible(s)) \
        if has_invisible else width
    fmt = "{0:<%ds}" % iwidth
    return fmt.format(s)

def _padboth(width, s, has_invisible=True):
    """
    Center string.

    >>> _padboth(6, '\u044f\u0439\u0446\u0430') \
        == ' \u044f\u0439\u0446\u0430 '
    True

    """
    iwidth = width + len(s) - len(_strip_invisible(s)) \
        if has_invisible else width
    fmt = "{0:^%ds}" % iwidth
    return fmt.format(s)

def _strip_invisible(s):
    "Remove invisible ANSI color codes."
    return re.sub(_invisible_codes, "", s)

def _visible_width(s):
    """
    Visible width of a printed string. ANSI color codes are removed.

    >>> _visible_width('\x1b[31mhello\x1b[0m'), _visible_width("world")
    (5, 5)

    """
    if isinstance(s, _text_type) or isinstance(s, _binary_type):
        return len(_strip_invisible(s))
    else:
        return len(_text_type(s))

def _align_column(strings, alignment, minwidth=0, has_invisible=True):
    """
    [string] -> [padded_string]

    >>> list(map(str,_align_column( \
        ["12.345", "-1234.5", "1.23", "1234.5", \
         "1e+234", "1.0e234"], "decimal")))
    ['   12.345  ', '-1234.5    ', '    1.23   ', \
     ' 1234.5    ', '    1e+234 ', '    1.0e234']

    """
    if alignment == "right":
        strings = [s.strip() for s in strings]
        padfn = _padleft
    elif alignment in "center":
        strings = [s.strip() for s in strings]
        padfn = _padboth
    elif alignment in "decimal":
        decimals = [_afterpoint(s) for s in strings]
        maxdecimals = max(decimals)
        strings = [s + (maxdecimals - decs) * " "
                   for s, decs in zip(strings, decimals)]
        padfn = _padleft
    else:
        strings = [s.strip() for s in strings]
        padfn = _padright

    if has_invisible:
        width_fn = _visible_width
    else:
        width_fn = len

    maxwidth = max(max(list(map(width_fn, strings))), minwidth)
    padded_strings = [padfn(maxwidth, s, has_invisible) for s in strings]
    return padded_strings

def _more_generic(type1, type2):
    types = {_none_type: 0, int: 1, float: 2, _text_type: 4}
    invtypes = {4: _text_type, 2: float, 1: int, 0: _none_type}
    moregeneric = max(types.get(type1, 4), types.get(type2, 4))
    return invtypes[moregeneric]

def _column_type(strings, has_invisible=True):
    """
    The least generic type all column values are convertible to.

    >>> _column_type(["1", "2"]) is _int_type
    True
    >>> _column_type(["1", "2.3"]) is _float_type
    True
    >>> _column_type(["1", "2.3", "four"]) is _text_type
    True
    >>> _column_type(["four", '\u043f\u044f\u0442\u044c']) is _text_type
    True
    >>> _column_type([None, "brux"]) is _text_type
    True
    >>> _column_type([1, 2, None]) is _int_type
    True

    """
    types = [_type(s, has_invisible) for s in strings]
    return reduce(_more_generic, types, int)

def _format(val, valtype, floatfmt, missingval=""):
    """
    Format a value accoding to its type.

    Unicode is supported:

    >>> hrow = ['\u0431\u0443\u043a\u0432\u0430', \
                '\u0446\u0438\u0444\u0440\u0430'] ; \
        tbl = [['\u0430\u0437', 2], ['\u0431\u0443\u043a\u0438', 4]] ; \
        good_result = '\\u0431\\u0443\\u043a\\u0432\\u0430      \
                        \\u0446\\u0438\\u0444\\u0440\\u0430\\n-------\
                          -------\\n\\u0430\\u0437             \
                          2\\n\\u0431\\u0443\\u043a\\u0438           4' ; \
        tabulate(tbl, headers=hrow) == good_result
    True

    """
    if val is None:
        return missingval

    if valtype in [int, _binary_type, _text_type]:
        return "{0}".format(val)
    elif valtype is float:
        return format(float(val), floatfmt)
    else:
        return "{0}".format(val)

def _align_header(header, alignment, width):
    if alignment == "left":
        return _padright(width, header)
    elif alignment == "center":
        return _padboth(width, header)
    else:
        return _padleft(width, header)

def _normalize_tabular_data(tabular_data, headers, sort=True):
    """
    Transform a supported data type to a list of lists, and a list of headers.

    Supported tabular data types:

    * list-of-lists or another iterable of iterables

    * 2D NumPy arrays

    * dict of iterables (usually used with headers="keys")

    * pandas.DataFrame (usually used with headers="keys")

    The first row can be used as headers if headers="firstrow",
    column indices can be used as headers if headers="keys".

    """

    if hasattr(tabular_data, "keys") and hasattr(tabular_data, "values"):
        # dict-like and pandas.DataFrame?
        if hasattr(tabular_data.values, "__call__"):
            # likely a conventional dict
            keys = list(tabular_data.keys())
            # columns have to be transposed
            rows = list(izip_longest(*list(tabular_data.values())))
        elif hasattr(tabular_data, "index"):
            # values is a property, has .index then
            # it's likely a pandas.DataFrame (pandas 0.11.0)
            keys = list(tabular_data.keys())
            # values matrix doesn't need to be transposed
            vals = tabular_data.values
            names = tabular_data.index
            rows = [[v] + list(row) for v, row in zip(names, vals)]
        else:
            raise ValueError("tabular data doesn't appear to be a dict "
                             "or a DataFrame")

        if headers == "keys":
            headers = list(map(_text_type, keys))  # headers should be strings

    else:  # it's, as usual, an iterable of iterables, or a NumPy array
        rows = list(tabular_data)

        if headers == "keys" and len(rows) > 0:  # keys are column indices
            headers = list(map(_text_type, list(range(len(rows[0])))))

    # take headers from the first row if necessary
    if headers == "firstrow" and len(rows) > 0:
        headers = list(map(_text_type, rows[0]))  # headers should be strings
        rows = rows[1:]

    headers = list(headers)

    rows = list(map(list, rows))

    if sort and len(rows) > 1:
        rows = sorted(rows, key=lambda x: x[0])

    # pad with empty headers for initial columns if necessary
    if headers and len(rows) > 0:
        nhs = len(headers)
        ncols = len(rows[0])
        if nhs < ncols:
            headers = [""] * (ncols - nhs) + headers

    return rows, headers

def _build_row(cells, padding, begin, sep, end):
    "Return a string which represents a row of data cells."

    pad = " " * padding
    padded_cells = [pad + cell + pad for cell in cells]

    # SolveBio: we're only displaying Key-Value tuples (dimension of 2).
    #  enforce that we don't wrap lines by setting a max
    #  limit on row width which is equal to TTY_COLS (see printing)
    rendered_cells = (begin + sep.join(padded_cells) + end).rstrip()
    if len(rendered_cells) > TTY_COLS:
        if not cells[-1].endswith(" ") and not cells[-1].endswith("-"):
            terminating_str = " ... "
        else:
            terminating_str = ""
        rendered_cells = "{0}{1}{2}".format(
            rendered_cells[:TTY_COLS - len(terminating_str) - 1],
            terminating_str, end)

    return rendered_cells

def _build_line(colwidths, padding, begin, fill, sep, end):
    "Return a string which represents a horizontal line."
    cells = [fill * (w + 2 * padding) for w in colwidths]
    return _build_row(cells, 0, begin, sep, end)

def _mediawiki_cell_attrs(row, colaligns):
    "Prefix every cell in a row with an HTML alignment attribute."
    alignment = {"left": '',
                 "right": 'align="right"| ',
                 "center": 'align="center"| ',
                 "decimal": 'align="right"| '}
    row2 = [alignment[a] + c for c, a in zip(row, colaligns)]
    return row2

def _line_segment_with_colons(linefmt, align, colwidth):
    """Return a segment of a horizontal line with optional colons which
    indicate column's alignment (as in `pipe` output format)."""
    fill = linefmt.hline
    w = colwidth
    if align in ["right", "decimal"]:
        return (fill[0] * (w - 1)) + ":"
    elif align == "center":
        return ":" + (fill[0] * (w - 2)) + ":"
    elif align == "left":
        return ":" + (fill[0] * (w - 1))
    else:
        return fill[0] * w

def _format_table(fmt, headers, rows, colwidths, colaligns):
    """Produce a plain-text representation of the table."""
    lines = []
    hidden = fmt.with_header_hide if headers else fmt.without_header_hide
    pad = fmt.padding
    headerrow = fmt.headerrow if fmt.headerrow else fmt.datarow

    if fmt.lineabove and "lineabove" not in hidden:
        lines.append(_build_line(colwidths, pad, *fmt.lineabove))

    if headers:
        lines.append(_build_row(headers, pad, *headerrow))

    if fmt.linebelowheader and "linebelowheader" not in hidden:
        begin, fill, sep, end = fmt.linebelowheader
        if fmt.usecolons:
            segs = [
                _line_segment_with_colons(fmt.linebelowheader, a, w + 2 * pad)
                for w, a in zip(colwidths, colaligns)]
            lines.append(_build_row(segs, 0, begin, sep, end))
        else:
            lines.append(_build_line(colwidths, pad, *fmt.linebelowheader))

    if rows and fmt.linebetweenrows and "linebetweenrows" not in hidden:
        # initial rows with a line below
        for row in rows[:-1]:
            lines.append(_build_row(row, pad, *fmt.datarow))
            lines.append(_build_line(colwidths, pad, *fmt.linebetweenrows))
        # the last row without a line below
        lines.append(_build_row(rows[-1], pad, *fmt.datarow))
    else:
        for row in rows:
            lines.append(_build_row(row, pad, *fmt.datarow))

    if fmt.linebelow and "linebelow" not in hidden:
        lines.append(_build_line(colwidths, pad, *fmt.linebelow))

    return "\n".join(lines)

def tabulate(tabular_data, headers=[], tablefmt="orgmode",
             floatfmt="g", aligns=[], missingval="", sort=True):
    list_of_lists, headers = _normalize_tabular_data(tabular_data, headers,
                                                     sort=sort)

    # optimization: look for ANSI control codes once,
    # enable smart width functions only if a control code is found
    plain_text = '\n'.join(
        ['\t'.join(map(_text_type, headers))] +
        ['\t'.join(map(_text_type, row)) for row in list_of_lists])

    has_invisible = re.search(_invisible_codes, plain_text)
    if has_invisible:
        width_fn = _visible_width
    else:
        width_fn = len

    # format rows and columns, convert numeric values to strings
    cols = list(zip(*list_of_lists))

    coltypes = list(map(_column_type, cols))
    cols = [[_format(v, ct, floatfmt, missingval) for v in c]
            for c, ct in zip(cols, coltypes)]

    # align columns
    if not aligns:
        # dynamic alignment by col type
        aligns = ["decimal" if ct in [int, float] else "left"
                  for ct in coltypes]

    minwidths = [width_fn(h) + 2 for h in headers] if headers \
        else [0] * len(cols)
    cols = [_align_column(c, a, minw, has_invisible)
            for c, a, minw in zip(cols, aligns, minwidths)]

    if headers:
        # align headers and add headers
        minwidths = [max(minw, width_fn(c[0]))
                     for minw, c in zip(minwidths, cols)]
        headers = [_align_header(h, a, minw)
                   for h, a, minw in zip(headers, aligns, minwidths)]
    else:
        minwidths = [width_fn(c[0]) for c in cols]
    rows = list(zip(*cols))

    if not isinstance(tablefmt, TableFormat):
        tablefmt = _table_formats.get(tablefmt, _table_formats["orgmode"])

    # make sure values don't have newlines or tabs in them
    rows = [[str(c).replace('\n', '').replace('\t', '').replace('\r', '')
            for c in r] for r in rows]
    return _format_table(tablefmt, headers, rows, minwidths, aligns)

if __name__ == "__main__":
    data = [
        ("gene_symbols", ["CPB1"]),
        ("clinical_significance", "other"),
        ("clinical_origin", ["somatic"]),
        ("alternate_alleles", ["T"]), ]
    print(tabulate(data,
                   headers=('Fields', 'Data'),
                   aligns=('right', 'left'), sort=True))
    print(tabulate(data,
                   headers=('Fields', 'Data'),
                   aligns=('right', 'left'), sort=False))


import os
import time

from ..client import client
from ..query import Query
from ..errors import NotFoundError

from .solveobject import convert_to_solve_object
from .apiresource import CreateableAPIResource
from .apiresource import ListableAPIResource
from .apiresource import UpdateableAPIResource
from .apiresource import DeletableAPIResource

from .task import Task
from .datasetfield import DatasetField
from .datasetexport import DatasetExport
from .datasetmigration import DatasetMigration

class Dataset(CreateableAPIResource,
              ListableAPIResource,
              DeletableAPIResource,
              UpdateableAPIResource):
    """
    Datasets are access points to data. Dataset names are unique
    within a vault folder.
    """
    RESOURCE_VERSION = 2

    LIST_FIELDS = (
        ('id', 'ID'),
        ('vault_name', 'Vault'),
        ('vault_object_path', 'Path'),
        ('documents_count', 'Documents'),
        ('description', 'Description'),
    )

    @classmethod
    def make_full_path(cls, vault_name, path, name, **kwargs):
        from solvebio import SolveError

        _client = kwargs.pop('client', None) or cls._client or client

        try:
            user = _client.get('/v1/user', {})
            domain = user['account']['domain']
        except SolveError as e:
            print("Error obtaining account domain: {0}".format(e))
            raise

        result = ':'.join([
            domain,
            vault_name,
            os.path.join('/' + path.lstrip('/'), name),
        ])

        return result

    @classmethod
    def get_by_full_path(cls, full_path, **kwargs):
        from solvebio import Object

        _client = kwargs.pop('client', None) or cls._client or client
        obj = Object.get_by_full_path(full_path, assert_type='dataset',
                                      client=_client)
        return Dataset.retrieve(obj['dataset_id'], client=_client, **kwargs)

    @classmethod
    def get_or_create_by_full_path(cls, full_path, **kwargs):
        from solvebio import Vault
        from solvebio import Object

        _client = kwargs.pop('client', None) or cls._client or client
        create_vault = kwargs.pop('create_vault', False)
        create_folders = kwargs.pop('create_folders', True)

        try:
            return Dataset.get_by_full_path(full_path, assert_type='dataset',
                                            client=_client)
        except NotFoundError:
            pass

        # Dataset not found, create it step-by-step
        full_path, parts = Object.validate_full_path(full_path, client=_client)

        if create_vault:
            vault = Vault.get_or_create_by_full_path(
                '{0}:{1}'.format(parts['domain'], parts['vault']),
                client=_client)
        else:
            vaults = Vault.all(account_domain=parts['domain'],
                               name=parts['vault'],
                               client=_client)
            if len(vaults.solve_objects()) == 0:
                raise Exception(
                    'Vault does not exist with name {0}:{1}'.format(
                        parts['domain'], parts['vault'])
                )
            vault = vaults.solve_objects()[0]

        # Create the folders to hold the dataset if they do not already exist.
        object_path = parts['path']
        curr_path = os.path.dirname(object_path)
        folders_to_create = []
        new_folders = []
        id_map = {'/': None}

        while curr_path != '/':
            try:
                obj = Object.get_by_path(curr_path,
                                         vault_id=vault.id,
                                         assert_type='folder',
                                         client=_client)
                id_map[curr_path] = obj.id
                break
            except NotFoundError:
                if not create_folders:
                    raise Exception('Folder {} does not exist.  Pass '
                                    'create_folders=True to auto-create '
                                    'missing folders')

                folders_to_create.append(curr_path)
                curr_path = '/'.join(curr_path.split('/')[:-1])
                if curr_path == '':
                    break

        for folder in reversed(folders_to_create):
            new_folder = Object.create(
                object_type='folder',
                vault_id=vault.id,
                filename=os.path.basename(folder),
                parent_object_id=id_map[os.path.dirname(folder)],
                client=_client
            )
            new_folders.append(new_folder)
            id_map[folder] = new_folder.id

        if os.path.dirname(object_path) == '/':
            parent_folder_id = None
        elif new_folders:
            parent_folder_id = new_folders[-1].id
        else:
            parent_folder_id = id_map[os.path.dirname(object_path)]

        return Dataset.create(name=os.path.basename(object_path),
                              vault_id=vault.id,
                              vault_parent_object_id=parent_folder_id,
                              client=_client,
                              **kwargs)

    def saved_queries(self, **params):
        from solvebio import SavedQuery

        if 'id' not in self or not self['id']:
            raise Exception(
                'No Dataset ID was provided. '
                'Please instantiate the Dataset '
                'object with an ID.')

        saved_queries = SavedQuery.all(
            dataset_id=self['id'],
            client=self._client,
            **params)

        return saved_queries

    def fields(self, name=None, **params):
        if 'fields_url' not in self:
            raise Exception(
                'Please use Dataset.retrieve({ID}) before looking '
                'up fields')

        if name:
            params.update({
                'name': name,
            })
            fields = self._client.get(self.fields_url, params)
            result = DatasetField.retrieve(fields['data'][0]['id'],
                                           client=self._client)
            return result

        response = self._client.get(self.fields_url, params)
        results = convert_to_solve_object(response, client=self._client)
        results.set_tabulate(
            ['name', 'data_type', 'entity_type', 'description'],
            headers=['Field', 'Data Type', 'Entity Type', 'Description'],
            aligns=['left', 'left', 'left', 'left'], sort=True)

        return results

    def template(self, **params):
        if 'template_url' not in self:
            raise Exception(
                'Please use Dataset.retrieve({ID}) before retrieving '
                'a template')

        response = self._client.get(self.template_url, params)
        return convert_to_solve_object(response, client=self._client)

    def commits(self, **params):
        if 'commits_url' not in self:
            raise Exception(
                'Please use Dataset.retrieve({ID}) before looking '
                'up commits')

        response = self._client.get(self.commits_url, params)
        results = convert_to_solve_object(response, client=self._client)
        results.set_tabulate(
            ['id', 'title', 'description', 'status', 'created_at'],
            headers=['ID', 'Title', 'Description', 'Status', 'Created'],
            aligns=['left', 'left', 'left', 'left', 'left'], sort=False)

        return results

    def imports(self, **params):
        if 'imports_url' not in self:
            raise Exception(
                'Please use Dataset.retrieve({ID}) before looking '
                'up imports')

        response = self._client.get(self.imports_url, params)
        results = convert_to_solve_object(response, client=self._client)
        results.set_tabulate(
            ['id', 'title', 'description', 'status', 'created_at'],
            headers=['ID', 'Title', 'Description', 'Status', 'Created'],
            aligns=['left', 'left', 'left', 'left', 'left'], sort=False)

        return results

    def _data_url(self):
        if 'data_url' not in self:
            if 'id' not in self or not self['id']:
                raise Exception(
                    'No Dataset ID was provided. '
                    'Please instantiate the Dataset '
                    'object with an ID.')
            # automatically construct the data_url from the ID
            return self.instance_url() + '/data'
        return self['data_url']

    def query(self, query=None, **params):
        self._data_url()  # raises an exception if there's no ID
        return Query(self['id'], query=query, client=self._client, **params)

    def lookup(self, *sbids):
        lookup_url = self._data_url() + '/' + ','.join(sbids)
        return self._client.get(lookup_url, {})['results']

    def _beacon_url(self):
        if 'beacon_url' not in self:
            if 'id' not in self or not self['id']:
                raise Exception(
                    'No Dataset ID was provided. '
                    'Please instantiate the Dataset '
                    'object with an ID.')
            # automatically construct the data_url from the ID
            self['beacon_url'] = self.instance_url() + '/beacon'
        return self['beacon_url']

    def beacon(self, **params):
        # raises an exception if there's no ID
        return self._client.get(self._beacon_url(), params)

    def import_file(self, path, **kwargs):
        """
        This is a shortcut to creating a DatasetImport. Can't use "import()"
        because of Python.
        """
        from . import Manifest
        from . import DatasetImport

        if 'id' not in self or not self['id']:
            raise Exception(
                'No Dataset ID found. '
                'Please instantiate or retrieve a dataset '
                'with an ID.')

        manifest = Manifest()
        manifest.add(path)
        return DatasetImport.create(
            dataset_id=self['id'],
            manifest=manifest.manifest,
            **kwargs)

    def export(self, format='json', follow=True, **kwargs):
        if 'id' not in self or not self['id']:
            raise Exception(
                'No Dataset ID was provided. '
                'Please instantiate the Dataset '
                'object with an ID.')

        export = DatasetExport.create(
            dataset_id=self['id'],
            format=format,
            **kwargs)

        if follow:
            export.follow()

        return export

    def migrate(self, target, follow=True, **kwargs):
        """
        Migrate the data from this dataset to a target dataset.

        Valid optional kwargs include:

        * source_params
        * target_fields
        * include_errors
        * commit_mode

        """
        if 'id' not in self or not self['id']:
            raise Exception(
                'No source dataset ID found. '
                'Please instantiate the Dataset '
                'object with an ID.')

        # Target can be provided as a Dataset, or as an ID.
        if isinstance(target, Dataset):
            target_id = target.id
        else:
            target_id = target

        migration = DatasetMigration.create(
            source_id=self['id'],
            target_id=target_id,
            **kwargs)

        if follow:
            migration.follow()

        return migration

    def activity(self, follow=False):
        statuses = ['running', 'queued', 'pending']
        activity = list(Task.all(target_object_id=self.id,
                                 status=','.join(statuses),
                                 client=self._client))

        print("Found {0} active task(s)".format(len(activity)))
        if not follow:
            return activity

        while True:
            if not activity:
                break

            for task in activity:
                task.follow()

            time.sleep(5.0)
            activity = self.activity()

        return activity

    #
    # Vault properties
    #
    @property
    def vault_object(self):
        from solvebio import Object
        return Object.retrieve(self['vault_object_id'], client=self._client)


"""Solvebio Object API resource"""
import os
import re
import base64
import binascii
import mimetypes

import requests

from solvebio import SolveError
from solvebio.utils.md5sum import md5sum

from ..client import client

from .apiresource import CreateableAPIResource
from .apiresource import ListableAPIResource
from .apiresource import SearchableAPIResource
from .apiresource import UpdateableAPIResource
from .apiresource import DeletableAPIResource
from .apiresource import DownloadableAPIResource

class Object(CreateableAPIResource,
             ListableAPIResource,
             DeletableAPIResource,
             SearchableAPIResource,
             UpdateableAPIResource,
             DownloadableAPIResource):
    """
    An object is a resource in a Vault.  It has three possible types,
    though more may be added later: folder, file, and SolveBio Dataset.
    """
    RESOURCE_VERSION = 2

    LIST_FIELDS = (
        ('id', 'ID'),
        ('object_type', 'Type'),
        ('full_path', 'Full Path'),
        ('description', 'Description'),
    )

    # Regex describing an object path.
    PATH_RE = re.compile(r'^[^\/]*(?P<path>(\/[^\/]*)+)$')

    @classmethod
    def validate_full_path(cls, full_path, **kwargs):
        """Helper method to parse a full or partial path and
        return a full path as well as a dict containing path parts.

        Uses the following rules when processing the path:

            * If no domain, uses the current user's account domain
            * If no vault, uses the current user's personal vault.
            * If no path, uses '/' (vault root)

        Returns a tuple containing:

            * The validated full_path
            * A dictionary with the components:
                * domain: the domain of the vault
                * vault: the name of the vault, without domain
                * vault_full_path: domain:vault
                * path: the object path within the vault
                * parent_path: the parent path to the object
                * filename: the object's filename (if any)
                * full_path: the validated full path

        The following components may be overridden using kwargs:

            * vault
            * path

        Object paths (also known as "paths") must begin with a forward slash.

        The following path formats are supported:

            domain:vault:/path -> object "path" in the root of "domain:vault"
            domain:vault/path  -> object "path" in the root of "domain:vault"
            vault:/path        -> object "path" in the root of "vault"
            vault/path         -> object "path" in the root of "vault"
            ~/path             -> object "path" in the root of personal vault
            vault/             -> root of "vault"
            ~/                 -> root of your personal vault

        The following two formats are not supported:

            path               -> invalid/ambiguous path (exception)
            vault:path         -> invalid/ambiguous path (exception)
            vault:path/path    -> unsupported, interpreted as domain:vault/path

        """
        from solvebio.resource.vault import Vault

        _client = kwargs.pop('client', None) or cls._client or client

        if not full_path:
            raise Exception(
                'Invalid path: ',
                'Full path must be in one of the following formats: '
                '"vault:/path", "domain:vault:/path", or "~/path"')

        # Parse the vault's full_path, using overrides if any
        input_vault = kwargs.get('vault') or full_path
        try:
            vault_full_path, path_dict = \
                Vault.validate_full_path(input_vault, client=_client)
        except Exception as err:
            raise Exception('Could not determine vault from "{0}": {1}'
                            .format(input_vault, err))

        if kwargs.get('path'):
            # Allow override of the object_path.
            full_path = '{0}:/{1}'.format(vault_full_path, kwargs['path'])

        match = cls.PATH_RE.match(full_path)
        if match:
            object_path = match.groupdict()['path']
        else:
            raise Exception(
                'Cannot find a valid object path in "{0}". '
                'Full path must be in one of the following formats: '
                '"vault:/path", "domain:vault:/path", or "~/path"'
                .format(full_path))

        # Remove double slashes
        object_path = re.sub('//+', '/', object_path)
        if object_path != '/':
            # Remove trailing slash
            object_path = object_path.rstrip('/')

        path_dict['path'] = object_path
        # TODO: parent_path and filename
        full_path = '{domain}:{vault}:{path}'.format(**path_dict)
        path_dict['full_path'] = full_path
        return full_path, path_dict

    @classmethod
    def get_by_full_path(cls, full_path, **params):
        # Don't pop client from params since **params is used below
        _client = params.get('client', None) or cls._client or client
        full_path, _ = cls.validate_full_path(full_path, client=_client)
        assert_type = params.pop('assert_type', None)
        params.update({'full_path': full_path})
        obj = cls._retrieve_helper('object', 'full_path', full_path,
                                   **params)
        if obj and assert_type and obj['object_type'] != assert_type:
            raise SolveError(
                "Expected a {} but found a {} at {}"
                .format(assert_type, obj['object_type'], full_path))

        return obj

    @classmethod
    def get_by_path(cls, path, **params):
        assert_type = params.pop('assert_type', None)
        params.update({'path': path})
        obj = cls._retrieve_helper('object', 'path', path, **params)
        if obj and assert_type and obj['object_type'] != assert_type:
            raise SolveError(
                "Expected a {} but found a {} at {}"
                .format(assert_type, obj['object_type'], path))

        return obj

    @classmethod
    def upload_file(cls, local_path, remote_path, vault_full_path, **kwargs):
        from solvebio import Vault
        from solvebio import Object

        _client = kwargs.pop('client', None) or cls._client or client

        local_path = os.path.expanduser(local_path)
        if os.stat(local_path).st_size == 0:
            print('Notice: Cannot upload empty file {0}'.format(local_path))
            return

        # Get vault
        vault = Vault.get_by_full_path(vault_full_path, client=_client)

        # Get MD5, mimetype, and file size for the object
        md5, _ = md5sum(local_path, multipart_threshold=None)
        _, mimetype = mimetypes.guess_type(local_path)
        size = os.path.getsize(local_path)

        # Lookup parent object
        if remote_path == '/':
            parent_object_id = None
        else:
            parent_obj = Object.get_by_full_path(
                ':'.join([vault.full_path, remote_path]),
                assert_type='folder', client=_client)
            parent_object_id = parent_obj.id

        description = kwargs.get(
            'description',
            'File uploaded via python client'
        )

        # Create the file, and upload it to the Upload URL
        obj = Object.create(
            vault_id=vault.id,
            parent_object_id=parent_object_id,
            object_type='file',
            filename=os.path.basename(local_path),
            md5=md5,
            mimetype=mimetype,
            size=size,
            description=description,
            client=_client
        )

        print('Notice: File created for {0} at {1}'.format(local_path,
                                                           obj.path))
        print('Notice: Upload initialized')

        upload_url = obj.upload_url

        headers = {
            'Content-MD5': base64.b64encode(binascii.unhexlify(md5)),
            'Content-Type': mimetype,
            'Content-Length': str(size),
        }

        # Use a session with a retry policy to handle connection errors.
        session = requests.Session()
        session.mount('https://', requests.adapters.HTTPAdapter(max_retries=5))
        upload_resp = session.put(upload_url,
                                  data=open(local_path, 'rb'),
                                  headers=headers)

        if upload_resp.status_code != 200:
            print('Notice: Upload status code for {0} was {1}'.format(
                local_path, upload_resp.status_code
            ))
            print('See error message below:')
            print(upload_resp.content)
            # Clean up the failed upload
            obj.delete(force=True)
        else:
            print('Notice: Successfully uploaded {0} to {1}'.format(local_path,
                                                                    obj.path))

        return obj

    @property
    def parent(self):
        """ Returns the parent object """
        if self['parent_object_id']:
            return self.retrieve(self['parent_object_id'], client=self._client)

        return None

    @property
    def vault(self):
        """ Returns the vault object """
        from . import Vault
        return Vault.retrieve(self['vault_id'], client=self._client)


# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function

import gzip
import json
import os
import re
import sys

import solvebio

from solvebio import Vault
from solvebio import Object
from solvebio.utils.files import check_gzip_path
from solvebio.errors import NotFoundError

def _upload_folder(domain, vault, base_remote_path,
                   base_local_path, local_start):

    # Create the upload root folder if it does not exist on the remote
    try:
        upload_root_path, _ = Object.validate_full_path(
            os.path.join(base_remote_path, local_start)
        )
        obj = Object.get_by_full_path(upload_root_path, assert_type='folder')
    except NotFoundError:
        base_remote_path, path_dict = \
            Object.validate_full_path(base_remote_path)

        if path_dict['path'] == '/':
            parent_object_id = None
        else:
            obj = Object.get_by_full_path(base_remote_path,
                                          assert_type='folder')
            parent_object_id = obj.id

        # Create base folder
        new_folder = Object.create(
            vault_id=vault.id,
            parent_object_id=parent_object_id,
            object_type='folder',
            filename=local_start
        )

        print('Notice: Folder created for {0} at {1}'.format(
            base_local_path,
            new_folder.path,
        ))

    for root, dirs, files in os.walk(base_local_path):

        # Create the sub-folders that do not exist on the remote
        for d in dirs:
            dirpath = os.path.join(
                base_remote_path,
                re.sub('^' + os.path.dirname(base_local_path), '', root).lstrip('/'),  # noqa
                d
            )

            try:
                Object.get_by_full_path(dirpath, object_type='folder')
            except NotFoundError:
                # Create the folder
                if os.path.dirname(dirpath.split(':')[-1]) == '/':
                    parent_object_id = None
                else:
                    parent_full_path = os.path.dirname(dirpath)
                    parent = Object.get_by_full_path(
                        parent_full_path, assert_type='folder')
                    parent_object_id = parent.id

                # Make the API call
                new_obj = Object.create(
                    vault_id=vault.id,
                    parent_object_id=parent_object_id,
                    object_type='folder',
                    filename=d,
                )

                print('Notice: Folder created for {0} at {1}'
                      .format(os.path.join(root, d), new_obj.path))

        # Upload the files that do not yet exist on the remote
        for f in files:
            file_full_path = os.path.join(
                base_remote_path,
                re.sub('^' + os.path.dirname(base_local_path),
                       '',
                       root).lstrip('/'),
                f,
            )
            try:
                Object.get_by_full_path(file_full_path)
            except NotFoundError:
                parent_full_path = os.path.dirname(
                    os.path.join(
                        base_remote_path,
                        re.sub('^' + os.path.dirname(base_local_path),
                               '',
                               root).lstrip('/'),
                        f,
                    )
                )
                parent = Object.get_by_full_path(
                    parent_full_path, assert_type='folder')
                Object.upload_file(os.path.join(root, f),
                                   parent.path, vault.full_path)

def create_dataset(args):
    """
    Attempt to create a new dataset given the following params:

        * template_id
        * template_file
        * capacity
        * create_vault
        * [argument] dataset name or full path

    NOTE: genome_build has been deprecated and is no longer used.

    """
    # For backwards compatibility, the "full_path" argument
    # can be a dataset filename, but only if vault and path
    # are set. If vault/path are both provided and there
    # are no forward-slashes in the "full_path", assume
    # the user has provided a dataset filename.
    if '/' not in args.full_path and args.vault and args.path:
        full_path, path_dict = Object.validate_full_path(
            '{0}:/{1}/{2}'.format(args.vault, args.path, args.full_path))
    else:
        full_path, path_dict = Object.validate_full_path(
            args.full_path, vault=args.vault, path=args.path)

    # Accept a template_id or a template_file
    if args.template_id:
        # Validate the template ID
        try:
            tpl = solvebio.DatasetTemplate.retrieve(args.template_id)
        except solvebio.SolveError as e:
            if e.status_code != 404:
                raise e
            print("No template with ID {0} found!"
                  .format(args.template_id))
            sys.exit(1)
    elif args.template_file:
        mode = 'r'
        fopen = open
        if check_gzip_path(args.template_file):
            mode = 'rb'
            fopen = gzip.open

        # Validate the template file
        with fopen(args.template_file, mode) as fp:
            try:
                tpl_json = json.load(fp)
            except:
                print('Template file {0} could not be loaded. Please '
                      'pass valid JSON'.format(args.template_file))
                sys.exit(1)

        tpl = solvebio.DatasetTemplate.create(**tpl_json)
        print("A new dataset template was created with id: {0}".format(tpl.id))
    else:
        print("Creating a new dataset {0} without a template."
              .format(full_path))
        tpl = None
        fields = []
        entity_type = None
        description = None

    if tpl:
        print("Creating new dataset {0} using the template '{1}'."
              .format(full_path, tpl.name))
        fields = tpl.fields
        entity_type = tpl.entity_type
        # include template used to create
        description = 'Created with dataset template: {0}'.format(str(tpl.id))

    return solvebio.Dataset.get_or_create_by_full_path(
        full_path,
        capacity=args.capacity,
        entity_type=entity_type,
        fields=fields,
        description=description,
        create_vault=args.create_vault,
    )

def upload(args):
    """
    Given a folder or file, upload all the folders and files contained
    within it, skipping ones that already exist on the remote.
    """
    base_remote_path, path_dict = Object.validate_full_path(
        args.full_path, vault=args.vault, path=args.path)

    # Assert the vault exists and is accessible
    vault = Vault.get_by_full_path(path_dict['vault_full_path'])

    # If not the vault root, validate remote path exists and is a folder
    if path_dict['path'] != '/':
        Object.get_by_full_path(base_remote_path, assert_type='folder')

    for local_path in args.local_path:
        local_path = local_path.rstrip('/')
        local_start = os.path.basename(local_path)

        if os.path.isdir(local_path):
            _upload_folder(path_dict['domain'], vault,
                           base_remote_path, local_path, local_start)
        else:
            Object.upload_file(local_path, path_dict['path'],
                               vault.full_path)

def import_file(args):
    """
    Given a dataset and a local path, upload and import the file(s).

    Command arguments (args):

        * create_dataset
        * template_id
        * full_path
        * vault (optional, overrides the vault in full_path)
        * path (optional, overrides the path in full_path)
        * commit_mode
        * capacity
        * file (list)
        * follow (default: False)

    """
    full_path, path_dict = Object.validate_full_path(
        args.full_path, vault=args.vault, path=args.path)

    # Ensure the dataset exists. Create if necessary.
    if args.create_dataset:
        dataset = create_dataset(args)
    else:
        try:
            dataset = solvebio.Dataset.get_by_full_path(full_path)
        except solvebio.SolveError as e:
            if e.status_code != 404:
                raise e

            print("Dataset not found: {0}".format(full_path))
            print("Tip: use the --create-dataset flag "
                  "to create one from a template")
            sys.exit(1)

    # Generate a manifest from the local files
    manifest = solvebio.Manifest()
    manifest.add(*args.file)

    # Create the manifest-based import
    imp = solvebio.DatasetImport.create(
        dataset_id=dataset.id,
        manifest=manifest.manifest,
        commit_mode=args.commit_mode
    )

    if args.follow:
        imp.follow()
    else:
        mesh_url = 'https://my.solvebio.com/activity/'
        print("Your import has been submitted, view details at: {0}"
              .format(mesh_url))


"""Solvebio Vault API resource"""
from ..client import client
from ..errors import NotFoundError

from .apiresource import CreateableAPIResource
from .apiresource import ListableAPIResource
from .apiresource import SearchableAPIResource
from .apiresource import UpdateableAPIResource
from .apiresource import DeletableAPIResource

import re

class Vault(CreateableAPIResource,
            ListableAPIResource,
            DeletableAPIResource,
            SearchableAPIResource,
            UpdateableAPIResource):
    """
    A vault is like a filesystem that can contain files, folder,
    and SolveBio datasets.  Vaults can be "connected" to external resources
    such as DNAnexus and SevenBridges projects, or Amazon S3 Buckets.
    Typically, vaults contain a series of datasets that are compatible with
    each other (i.e. they come from the same data source or project).
    """
    RESOURCE_VERSION = 2

    LIST_FIELDS = (
        ('id', 'ID'),
        ('full_path', 'Full Path'),
        ('provider', 'Provider'),
        ('description', 'Description'),
    )

    # Regex describing Vault full path.
    # NOTE: Not valid for object full paths.
    VAULT_PATH_RE = re.compile(
        # Non-greedy wildcard match for domain
        r'^(?:(?P<domain>[a-zA-Z0-9\-]+)\:)??'
        # Match vault or vault:/ or vault/
        r'(?P<vault>[^\/:]+)(?:\:?\/.*)?$')

    def __init__(self, vault_id, **kwargs):
        super(Vault, self).__init__(vault_id, **kwargs)

    def _object_list_helper(self, **params):
        from solvebio import Object

        params.update({
            'vault_id': self.id,
        })

        items = Object.all(client=self._client, **params)
        return items

    @classmethod
    def validate_full_path(cls, full_path, **kwargs):
        """Helper method to return a full path from a full or partial path.

            If no domain, assumes user's account domain
            If the vault is "~", assumes personal vault.

        Valid vault paths include:

            domain:vault
            domain:vault:/path
            domain:vault/path
            vault:/path
            vault
            ~/

        Invalid vault paths include:

            /vault/
            /path
            /
            :/

        Does not allow overrides for any vault path components.
        """
        _client = kwargs.pop('client', None) or cls._client or client

        full_path = full_path.strip()
        if not full_path:
            raise Exception(
                'Vault path "{0}" is invalid. Path must be in the format: '
                '"domain:vault:/path" or "vault:/path".'.format(full_path)
            )

        match = cls.VAULT_PATH_RE.match(full_path)
        if not match:
            raise Exception(
                'Vault path "{0}" is invalid. Path must be in the format: '
                '"domain:vault:/path" or "vault:/path".'.format(full_path)
            )
        path_parts = match.groupdict()

        # Handle the special case where "~" means personal vault
        if path_parts.get('vault') == '~':
            path_parts = dict(domain=None, vault=None)

        # If any values are None, set defaults from the user.
        if None in path_parts.values():
            user = _client.get('/v1/user', {})
            defaults = {
                'domain': user['account']['domain'],
                'vault': 'user-{0}'.format(user['id'])
            }
            path_parts = dict((k, v or defaults.get(k))
                              for k, v in path_parts.items())

        # Rebuild the full path
        full_path = '{domain}:{vault}'.format(**path_parts)
        path_parts['vault_full_path'] = full_path
        return full_path, path_parts

    def files(self, **params):
        return self._object_list_helper(object_type='file', **params)

    def folders(self, **params):
        return self._object_list_helper(object_type='folder', **params)

    def datasets(self, **params):
        return self._object_list_helper(object_type='dataset', **params)

    def objects(self, **params):
        return self._object_list_helper(**params)

    def ls(self, **params):
        return self._object_list_helper(**params)

    def _get_parent_folder(self, path):
        from solvebio import Object
        return Object.get_by_full_path(
            ':'.join([self.full_path, path]),
            assert_type='folder',
            client=self._client
        )

    def create_dataset(self, name, **params):
        from solvebio import Dataset

        params['vault_id'] = self.id
        path = params.pop('path', None)

        if path == '/' or path is None:
            params['vault_parent_object_id'] = None
        else:
            parent_object = self._get_parent_folder(path)
            params['vault_parent_object_id'] = parent_object.id

        params['name'] = name
        return Dataset.create(**params)

    def create_folder(self, filename, **params):
        from solvebio import Object

        path = params.pop('path', None)
        if path and path != '/':
            parent_object = self._get_parent_folder(path)
            params['parent_object_id'] = parent_object.id

        params.update({
            'filename': filename,
            'vault_id': self.id,
            'object_type': 'folder'
        })
        return Object.create(client=self._client, **params)

    def upload_file(self, local_path, remote_path, **kwargs):
        from solvebio import Object
        return Object.upload_file(
            local_path, remote_path, self.full_path, **kwargs)

    def search(self, query, **params):
        params.update({
            'query': query,
        })
        return self._object_list_helper(**params)

    @classmethod
    def get_by_full_path(cls, full_path, **kwargs):
        _client = kwargs.pop('client', None) or cls._client or client

        full_path, parts = cls.validate_full_path(full_path, client=_client)
        return Vault._retrieve_helper(
            'vault', 'name', full_path,
            account_domain=parts['domain'],
            name=parts['vault'],
            client=_client
        )

    @classmethod
    def get_or_create_by_full_path(cls, full_path, **kwargs):
        _client = kwargs.pop('client', None) or cls._client or client

        try:
            return Vault.get_by_full_path(full_path, client=_client)
        except NotFoundError:
            pass

        # Vault not found, create it
        parts = full_path.split(':', 2)
        vault_name = parts[-1]

        return Vault.create(name=vault_name, client=_client)

    @classmethod
    def get_personal_vault(cls, **kwargs):
        _client = kwargs.pop('client', None) or cls._client or client
        user = _client.get('/v1/user', {})
        # TODO - this will have to change if the format of the personal vaults
        # changes.
        name = 'user-{0}'.format(user['id'])
        return list(Vault.all(name=name, vault_type='user', client=_client))[0]

    @classmethod
    def get_or_create_uploads_path(cls, **kwargs):
        from solvebio import Object
        _client = kwargs.pop('client', None) or cls._client or client
        v = cls.get_personal_vault(client=_client)
        default_path = 'Uploads'
        full_path = '{0}:/{1}'.format(v.full_path, default_path)

        try:
            upload_dir = Object.get_by_full_path(
                full_path, assert_type='folder', client=_client)
        except NotFoundError:
            print("Uploads directory not found. Creating {0}"
                  .format(full_path))
            upload_dir = Object.create(
                vault_id=v.id,
                object_type='folder',
                filename=default_path,
                client=_client
            )

        return upload_dir.path


from __future__ import absolute_import

from six.moves.urllib.parse import urlparse

from ..errors import SolveError

def validate_api_host_url(url):
    """
    Validate SolveBio API host url.

    Valid urls must not be empty and
    must contain either HTTP or HTTPS scheme.
    """
    if not url:
        raise SolveError('No SolveBio API host is set')

    parsed = urlparse(url)
    if parsed.scheme not in ['http', 'https']:
        raise SolveError(
            'Invalid API host: %s. '
            'Missing url scheme (HTTP or HTTPS).' % url
        )

    elif not parsed.netloc:
        raise SolveError('Invalid API host: %s.' % url)

    return True


# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function

import os
import glob

from six.moves.urllib.parse import urlparse

import solvebio

class Manifest(object):
    """
    Manifests aren't strictly resources, they represent a list
    of remote files (URLs) with additional information that
    can be used for validation (size and MD5).
    """
    manifest = None

    def __init__(self):
        self.manifest = {'files': []}

    def add_file(self, path, **kwargs):
        default_upload_path = solvebio.Vault.get_or_create_uploads_path()
        vault = solvebio.Vault.get_personal_vault()
        print("Uploading file: {0} to {1}".format(path, default_upload_path))
        file_ = solvebio.Object.upload_file(path, default_upload_path,
                                            vault.full_path)
        print("Successfuly uploaded file {0} (id:{1} size:{2} md5:{3})"
              .format(path, file_.id, file_.size, file_.md5))

        self.manifest['files'].append({
            'object_id': file_.id,
            'name': file_.filename,
            'md5': file_.md5,
            'size': file_.size,
            'reader_params': kwargs.get('reader_params'),
            'entity_params': kwargs.get('entity_params'),
            'validation_params': kwargs.get('validation_params')
        })

    def add_url(self, url, **kwargs):
        manifest_item = dict(url=url, **kwargs)
        self.manifest['files'].append(manifest_item)

    def add(self, *args):
        """
        Add one or more files or URLs to the manifest.
        If files contains a glob, it is expanded.

        All files are uploaded to SolveBio. The Upload
        object is used to fill the manifest.
        """
        def _is_url(path):
            p = urlparse(path)
            return bool(p.scheme)

        for path in args:
            path = os.path.expanduser(path)
            if _is_url(path):
                self.add_url(path)
            elif os.path.isfile(path):
                self.add_file(path)
            elif os.path.isdir(path):
                for f in os.listdir(path):
                    self.add_file(f)
            elif glob.glob(path):
                for f in glob.glob(path):
                    self.add_file(f)
            else:
                raise ValueError(
                    'Path: "{0}" is not a valid format or does not exist. '
                    'Manifest paths must be files, directories, or URLs.'
                    .format(path)
                )


# -*- coding: utf-8 -*-
from __future__ import absolute_import

from .client import client

import logging
logger = logging.getLogger('solvebio')

class Annotator(object):
    """
    Runs the synchronous annotate endpoint against
    batches of results from a query.
    """
    CHUNK_SIZE = 100

    # Allows pre-setting a SolveClient
    _client = None

    def __init__(self, fields, **kwargs):
        self._client = kwargs.pop('client', None) or self._client or client

        self.buffer = []
        self.fields = fields

        # Pop annotator_params from kwargs
        annotator_param_keys = [
            'annotator',
            'debug',
            'include_errors',
        ]
        self.annotator_params = {}
        for key in annotator_param_keys:
            if key in kwargs:
                self.annotator_params[key] = kwargs.pop(key)

        self.data = kwargs.get('data')

    def annotate(self, records, **kwargs):
        """Annotate a set of records with stored fields.

        Args:
            records: A list or iterator (can be a Query object)
            chunk_size: The number of records to annotate at once (max 500).

        Returns:
            A generator that yields one annotated record at a time.
        """
        # Update annotator_params with any kwargs
        self.annotator_params.update(**kwargs)
        chunk_size = self.annotator_params.get('chunk_size', self.CHUNK_SIZE)

        chunk = []
        for i, record in enumerate(records):
            chunk.append(record)
            if (i + 1) % chunk_size == 0:
                for r in self._execute(chunk):
                    yield r
                chunk = []

        if chunk:
            for r in self._execute(chunk):
                yield r
            chunk = []

    def _execute(self, chunk):
        data = {
            'records': chunk,
            'fields': self.fields,
            'annotator_params': self.annotator_params,
            'data': self.data
        }

        for r in self._client.post('/v1/annotate', data)['results']:
            yield r

class Expression(object):
    """Runs a single SolveBio expression."""

    # Allows pre-setting a SolveClient
    _client = None

    def __init__(self, expr, **kwargs):
        self.expr = expr
        self._client = kwargs.get('client') or self._client or client

    def evaluate(self, data=None, data_type='string', is_list=False):
        """Evaluates the expression with the provided context and format."""
        payload = {
            'data': data,
            'expression': self.expr,
            'data_type': data_type,
            'is_list': is_list
        }
        res = self._client.post('/v1/evaluate', payload)
        return res['result']

    def __repr__(self):
        return '<Expression "{0}">'.format(self.expr)


# -*- coding: utf-8 -*-
"""A generic tabular data output formatter interface."""

from __future__ import unicode_literals
from collections import namedtuple

from cli_helpers.compat import (text_type, binary_type, int_types, float_types,
                                zip_longest)
from cli_helpers.utils import unique_items
from . import (delimited_output_adapter, vertical_table_adapter,
               tabulate_adapter, terminaltables_adapter, tsv_output_adapter)
from decimal import Decimal

import itertools

MISSING_VALUE = '<null>'
MAX_FIELD_WIDTH = 500

TYPES = {
    type(None): 0,
    bool: 1,
    int: 2,
    float: 3,
    Decimal: 3,
    binary_type: 4,
    text_type: 5
}

OutputFormatHandler = namedtuple(
    'OutputFormatHandler',
    'format_name preprocessors formatter formatter_args')

class TabularOutputFormatter(object):
    """An interface to various tabular data formatting libraries.

    The formatting libraries supported include:
      - `tabulate <https://bitbucket.org/astanin/python-tabulate>`_
      - `terminaltables <https://robpol86.github.io/terminaltables/>`_
      - a CLI Helper vertical table layout
      - delimited formats (CSV and TSV)

    :param str format_name: An optional, default format name.

    Usage::

      >>> from cli_helpers.tabular_output import TabularOutputFormatter
      >>> formatter = TabularOutputFormatter(format_name='simple')
      >>> data = ((1, 87), (2, 80), (3, 79))
      >>> headers = ('day', 'temperature')
      >>> print(formatter.format_output(data, headers))
        day    temperature
      -----  -------------
          1             87
          2             80
          3             79

    You can use any :term:`iterable` for the data or headers::

      >>> data = enumerate(('87', '80', '79'), 1)
      >>> print(formatter.format_output(data, headers))
        day    temperature
      -----  -------------
          1             87
          2             80
          3             79

    """

    _output_formats = {}

    def __init__(self, format_name=None):
        """Set the default *format_name*."""
        self._format_name = None

        if format_name:
            self.format_name = format_name

    @property
    def format_name(self):
        """The current format name.

        This value must be in :data:`supported_formats`.

        """
        return self._format_name

    @format_name.setter
    def format_name(self, format_name):
        """Set the default format name.

        :param str format_name: The display format name.
        :raises ValueError: if the format is not recognized.

        """
        if format_name in self.supported_formats:
            self._format_name = format_name
        else:
            raise ValueError('unrecognized format_name "{}"'.format(
                format_name))

    @property
    def supported_formats(self):
        """The names of the supported output formats in a :class:`tuple`."""
        return tuple(self._output_formats.keys())

    @classmethod
    def register_new_formatter(cls, format_name, handler, preprocessors=(),
                               kwargs=None):
        """Register a new output formatter.

        :param str format_name: The name of the format.
        :param callable handler: The function that formats the data.
        :param tuple preprocessors: The preprocessors to call before
            formatting.
        :param dict kwargs: Keys/values for keyword argument defaults.

        """
        cls._output_formats[format_name] = OutputFormatHandler(
            format_name, preprocessors, handler, kwargs or {})

    def format_output(self, data, headers, format_name=None,
                      preprocessors=(), column_types=None, **kwargs):
        """Format the headers and data using a specific formatter.

        *format_name* must be a supported formatter (see
        :attr:`supported_formats`).

        :param iterable data: An :term:`iterable` (e.g. list) of rows.
        :param iterable headers: The column headers.
        :param str format_name: The display format to use (optional, if the
            :class:`TabularOutputFormatter` object has a default format set).
        :param tuple preprocessors: Additional preprocessors to call before
                                    any formatter preprocessors.
        :param \*\*kwargs: Optional arguments for the formatter.
        :return: The formatted data.
        :rtype: str
        :raises ValueError: If the *format_name* is not recognized.

        """
        format_name = format_name or self._format_name
        if format_name not in self.supported_formats:
            raise ValueError('unrecognized format "{}"'.format(format_name))

        (_, _preprocessors, formatter,
         fkwargs) = self._output_formats[format_name]
        fkwargs.update(kwargs)
        if column_types is None:
            data = list(data)
            column_types = self._get_column_types(data)
        for f in unique_items(preprocessors + _preprocessors):
            data, headers = f(data, headers, column_types=column_types,
                              **fkwargs)
        return formatter(list(data), headers, column_types=column_types, **fkwargs)

    def _get_column_types(self, data):
        """Get a list of the data types for each column in *data*."""
        columns = list(zip_longest(*data))
        return [self._get_column_type(column) for column in columns]

    def _get_column_type(self, column):
        """Get the most generic data type for iterable *column*."""
        type_values = [TYPES[self._get_type(v)] for v in column]
        inverse_types = {v: k for k, v in TYPES.items()}
        return inverse_types[max(type_values)]

    def _get_type(self, value):
        """Get the data type for *value*."""
        if value is None:
            return type(None)
        elif type(value) in int_types:
            return int
        elif type(value) in float_types:
            return float
        elif isinstance(value, binary_type):
            return binary_type
        else:
            return text_type

def format_output(data, headers, format_name, **kwargs):
    """Format output using *format_name*.

    This is a wrapper around the :class:`TabularOutputFormatter` class.

    :param iterable data: An :term:`iterable` (e.g. list) of rows.
    :param iterable headers: The column headers.
    :param str format_name: The display format to use.
    :param \*\*kwargs: Optional arguments for the formatter.
    :return: The formatted data.
    :rtype: str

    """
    formatter = TabularOutputFormatter(format_name=format_name)
    return formatter.format_output(data, headers, **kwargs)

for vertical_format in vertical_table_adapter.supported_formats:
    TabularOutputFormatter.register_new_formatter(
        vertical_format, vertical_table_adapter.adapter,
        vertical_table_adapter.preprocessors,
        {'table_format': vertical_format, 'missing_value': MISSING_VALUE, 'max_field_width': None})

for delimited_format in delimited_output_adapter.supported_formats:
    TabularOutputFormatter.register_new_formatter(
        delimited_format, delimited_output_adapter.adapter,
        delimited_output_adapter.preprocessors,
        {'table_format': delimited_format, 'missing_value': '', 'max_field_width': None})

for tabulate_format in tabulate_adapter.supported_formats:
    TabularOutputFormatter.register_new_formatter(
        tabulate_format, tabulate_adapter.adapter,
        tabulate_adapter.preprocessors +
        (tabulate_adapter.style_output_table(tabulate_format),),
        {'table_format': tabulate_format, 'missing_value': MISSING_VALUE, 'max_field_width': MAX_FIELD_WIDTH})

for terminaltables_format in terminaltables_adapter.supported_formats:
    TabularOutputFormatter.register_new_formatter(
        terminaltables_format, terminaltables_adapter.adapter,
        terminaltables_adapter.preprocessors +
        (terminaltables_adapter.style_output_table(terminaltables_format),),
        {'table_format': terminaltables_format, 'missing_value': MISSING_VALUE, 'max_field_width': MAX_FIELD_WIDTH})

for tsv_format in tsv_output_adapter.supported_formats:
    TabularOutputFormatter.register_new_formatter(
        tsv_format, tsv_output_adapter.adapter,
        tsv_output_adapter.preprocessors,
        {'table_format': tsv_format, 'missing_value': '', 'max_field_width': None})


# -*- coding: utf-8 -*-
"""Format adapter for the tabulate module."""

from __future__ import unicode_literals

from cli_helpers.utils import filter_dict_by_key
from .preprocessors import (convert_to_string, truncate_string, override_missing_value,
                            style_output, HAS_PYGMENTS, Terminal256Formatter, StringIO)

import tabulate

supported_markup_formats = ('mediawiki', 'html', 'latex', 'latex_booktabs',
                            'textile', 'moinmoin', 'jira')
supported_table_formats = ('plain', 'simple', 'grid', 'fancy_grid', 'pipe',
                           'orgtbl', 'psql', 'rst')
supported_formats = supported_markup_formats + supported_table_formats

preprocessors = (override_missing_value, convert_to_string, truncate_string, style_output)

def style_output_table(format_name=""):
    def style_output(data, headers, style=None,
                     table_separator_token='Token.Output.TableSeparator', **_):
        """Style the *table* a(e.g. bold, italic, and colors)

        .. NOTE::
            This requires the `Pygments <http://pygments.org/>`_ library to
            be installed. You can install it with CLI Helpers as an extra::
                $ pip install cli_helpers[styles]

        Example usage::

            from cli_helpers.tabular_output import tabulate_adapter
            from pygments.style import Style
            from pygments.token import Token

            class YourStyle(Style):
                default_style = ""
                styles = {
                    Token.Output.TableSeparator: '#ansigray'
                }

            headers = ('First Name', 'Last Name')
            data = [['Fred', 'Roberts'], ['George', 'Smith']]
            style_output_table = tabulate_adapter.style_output_table('psql')
            style_output_table(data, headers, style=CliStyle)

            data, headers = style_output(data, headers, style=YourStyle)
            output = tabulate_adapter.adapter(data, headers, style=YourStyle)

        :param iterable data: An :term:`iterable` (e.g. list) of rows.
        :param iterable headers: The column headers.
        :param str/pygments.style.Style style: A Pygments style. You can `create
            your own styles <http://pygments.org/docs/styles/#creating-own-styles>`_.
        :param str table_separator_token: The token type to be used for the table separator.
        :return: data and headers.
        :rtype: tuple

        """
        if style and HAS_PYGMENTS and format_name in supported_table_formats:
            formatter = Terminal256Formatter(style=style)

            def style_field(token, field):
                """Get the styled text for a *field* using *token* type."""
                s = StringIO()
                formatter.format(((token, field),), s)
                return s.getvalue()

            def addColorInElt(elt):
                if not elt:
                    return elt
                if elt.__class__ == tabulate.Line:
                    return tabulate.Line(*(style_field(table_separator_token, val) for val in elt))
                if elt.__class__ == tabulate.DataRow:
                    return tabulate.DataRow(*(style_field(table_separator_token, val) for val in elt))
                return elt

            srcfmt = tabulate._table_formats[format_name]
            newfmt = tabulate.TableFormat(
                *(addColorInElt(val) for val in srcfmt))
            tabulate._table_formats[format_name] = newfmt

        return iter(data), headers
    return style_output

def adapter(data, headers, table_format=None, preserve_whitespace=False,
            **kwargs):
    """Wrap tabulate inside a function for TabularOutputFormatter."""
    keys = ('floatfmt', 'numalign', 'stralign', 'showindex', 'disable_numparse')
    tkwargs = {'tablefmt': table_format}
    tkwargs.update(filter_dict_by_key(kwargs, keys))

    if table_format in supported_markup_formats:
        tkwargs.update(numalign=None, stralign=None)

    tabulate.PRESERVE_WHITESPACE = preserve_whitespace

    return iter(tabulate.tabulate(data, headers, **tkwargs).split('\n'))


# -*- coding: utf-8 -*-
"""Read and write an application's config files."""

from __future__ import unicode_literals
import io
import logging
import os

from configobj import ConfigObj, ConfigObjError
from validate import ValidateError, Validator

from .compat import MAC, text_type, UserDict, WIN

logger = logging.getLogger(__name__)

class ConfigError(Exception):
    """Base class for exceptions in this module."""
    pass

class DefaultConfigValidationError(ConfigError):
    """Indicates the default config file did not validate correctly."""
    pass

class Config(UserDict, object):
    """Config reader/writer class.

    :param str app_name: The application's name.
    :param str app_author: The application author/organization.
    :param str filename: The config filename to look for (e.g. ``config``).
    :param dict/str default: The default config values or absolute path to
                             config file.
    :param bool validate: Whether or not to validate the config file.
    :param bool write_default: Whether or not to write the default config
                               file to the user config directory if it doesn't
                               already exist.
    :param tuple additional_dirs: Additional directories to check for a config
                                  file.
    """

    def __init__(self, app_name, app_author, filename, default=None,
                 validate=False, write_default=False, additional_dirs=()):
        super(Config, self).__init__()
        #: The :class:`ConfigObj` instance.
        self.data = ConfigObj()

        self.default = {}
        self.default_file = self.default_config = None
        self.config_filenames = []

        self.app_name, self.app_author = app_name, app_author
        self.filename = filename
        self.write_default = write_default
        self.validate = validate
        self.additional_dirs = additional_dirs

        if isinstance(default, dict):
            self.default = default
            self.update(default)
        elif isinstance(default, text_type):
            self.default_file = default
        elif default is not None:
            raise TypeError(
                '"default" must be a dict or {}, not {}'.format(
                    text_type.__name__, type(default)))

        if self.write_default and not self.default_file:
            raise ValueError('Cannot use "write_default" without specifying '
                             'a default file.')

        if self.validate and not self.default_file:
            raise ValueError('Cannot use "validate" without specifying a '
                             'default file.')

    def read_default_config(self):
        """Read the default config file.

        :raises DefaultConfigValidationError: There was a validation error with
                                              the *default* file.
        """
        if self.validate:
            self.default_config = ConfigObj(configspec=self.default_file,
                                            list_values=False, _inspec=True,
                                            encoding='utf8')
            valid = self.default_config.validate(Validator(), copy=True,
                                                 preserve_errors=True)
            if valid is not True:
                for name, section in valid.items():
                    if section is True:
                        continue
                    for key, value in section.items():
                        if isinstance(value, ValidateError):
                            raise DefaultConfigValidationError(
                                'section [{}], key "{}": {}'.format(
                                    name, key, value))
        elif self.default_file:
            self.default_config, _ = self.read_config_file(self.default_file)

        self.update(self.default_config)

    def read(self):
        """Read the default, additional, system, and user config files.

        :raises DefaultConfigValidationError: There was a validation error with
                                              the *default* file.
        """
        if self.default_file:
            self.read_default_config()
        return self.read_config_files(self.all_config_files())

    def user_config_file(self):
        """Get the absolute path to the user config file."""
        return os.path.join(
            get_user_config_dir(self.app_name, self.app_author),
            self.filename)

    def system_config_files(self):
        """Get a list of absolute paths to the system config files."""
        return [os.path.join(f, self.filename) for f in get_system_config_dirs(
            self.app_name, self.app_author)]

    def additional_files(self):
        """Get a list of absolute paths to the additional config files."""
        return [os.path.join(f, self.filename) for f in self.additional_dirs]

    def all_config_files(self):
        """Get a list of absolute paths to all the config files."""
        return (self.additional_files() + self.system_config_files() +
                [self.user_config_file()])

    def write_default_config(self, overwrite=False):
        """Write the default config to the user's config file.

        :param bool overwrite: Write over an existing config if it exists.
        """
        destination = self.user_config_file()
        if not overwrite and os.path.exists(destination):
            return

        with io.open(destination, mode='wb') as f:
            self.default_config.write(f)

    def write(self, outfile=None, section=None):
        """Write the current config to a file (defaults to user config).

        :param str outfile: The path to the file to write to.
        :param None/str section: The config section to write, or :data:`None`
                                 to write the entire config.
        """
        with io.open(outfile or self.user_config_file(), 'wb') as f:
            self.data.write(outfile=f, section=section)

    def read_config_file(self, f):
        """Read a config file *f*.

        :param str f: The path to a file to read.
        """
        configspec = self.default_file if self.validate else None
        try:
            config = ConfigObj(infile=f, configspec=configspec,
                               interpolation=False, encoding='utf8')
        except ConfigObjError as e:
            logger.warning(
                'Unable to parse line {} of config file {}'.format(
                    e.line_number, f))
            config = e.config

        valid = True
        if self.validate:
            valid = config.validate(Validator(), preserve_errors=True,
                                    copy=True)
        if bool(config):
            self.config_filenames.append(config.filename)

        return config, valid

    def read_config_files(self, files):
        """Read a list of config files.

        :param iterable files: An iterable (e.g. list) of files to read.
        """
        errors = {}
        for _file in files:
            config, valid = self.read_config_file(_file)
            self.update(config)
            if valid is not True:
                errors[_file] = valid
        return errors or True

def get_user_config_dir(app_name, app_author, roaming=True, force_xdg=True):
    """Returns the config folder for the application.  The default behavior
    is to return whatever is most appropriate for the operating system.

    For an example application called ``"My App"`` by ``"Acme"``,
    something like the following folders could be returned:

    macOS (non-XDG):
      ``~/Library/Application Support/My App``
    Mac OS X (XDG):
      ``~/.config/my-app``
    Unix:
      ``~/.config/my-app``
    Windows 7 (roaming):
      ``C:\\Users\<user>\AppData\Roaming\Acme\My App``
    Windows 7 (not roaming):
      ``C:\\Users\<user>\AppData\Local\Acme\My App``

    :param app_name: the application name. This should be properly capitalized
                     and can contain whitespace.
    :param app_author: The app author's name (or company). This should be
                       properly capitalized and can contain whitespace.
    :param roaming: controls if the folder should be roaming or not on Windows.
                    Has no effect on non-Windows systems.
    :param force_xdg: if this is set to `True`, then on macOS the XDG Base
                      Directory Specification will be followed. Has no effect
                      on non-macOS systems.

    """
    if WIN:
        key = 'APPDATA' if roaming else 'LOCALAPPDATA'
        folder = os.path.expanduser(os.environ.get(key, '~'))
        return os.path.join(folder, app_author, app_name)
    if MAC and not force_xdg:
        return os.path.join(os.path.expanduser(
            '~/Library/Application Support'), app_name)
    return os.path.join(
        os.path.expanduser(os.environ.get('XDG_CONFIG_HOME', '~/.config')),
        _pathify(app_name))

def get_system_config_dirs(app_name, app_author, force_xdg=True):
    r"""Returns a list of system-wide config folders for the application.

    For an example application called ``"My App"`` by ``"Acme"``,
    something like the following folders could be returned:

    macOS (non-XDG):
      ``['/Library/Application Support/My App']``
    Mac OS X (XDG):
      ``['/etc/xdg/my-app']``
    Unix:
      ``['/etc/xdg/my-app']``
    Windows 7:
      ``['C:\ProgramData\Acme\My App']``

    :param app_name: the application name. This should be properly capitalized
                     and can contain whitespace.
    :param app_author: The app author's name (or company). This should be
                       properly capitalized and can contain whitespace.
    :param force_xdg: if this is set to `True`, then on macOS the XDG Base
                      Directory Specification will be followed. Has no effect
                      on non-macOS systems.

    """
    if WIN:
        folder = os.environ.get('PROGRAMDATA')
        return [os.path.join(folder, app_author, app_name)]
    if MAC and not force_xdg:
        return [os.path.join('/Library/Application Support', app_name)]
    dirs = os.environ.get('XDG_CONFIG_DIRS', '/etc/xdg')
    paths = [os.path.expanduser(x) for x in dirs.split(os.pathsep)]
    return [os.path.join(d, _pathify(app_name)) for d in paths]

def _pathify(s):
    """Convert spaces to hyphens and lowercase a string."""
    return '-'.join(s.split()).lower()


# -*- coding: utf-8 -*-
"""Various utility functions and helpers."""

import binascii
import re

from cli_helpers.compat import binary_type, text_type

def bytes_to_string(b):
    """Convert bytes *b* to a string.

    Hexlify bytes that can't be decoded.

    """
    if isinstance(b, binary_type):
        try:
            return b.decode('utf8')
        except UnicodeDecodeError:
            return '0x' + binascii.hexlify(b).decode('ascii')
    return b

def to_string(value):
    """Convert *value* to a string."""
    if isinstance(value, binary_type):
        return bytes_to_string(value)
    else:
        return text_type(value)

def truncate_string(value, max_width=None):
    """Truncate string values."""
    if isinstance(value, text_type) and max_width is not None and len(value) > max_width:
        return value[:max_width]
    return value

def intlen(n):
    """Find the length of the integer part of a number *n*."""
    pos = n.find('.')
    return len(n) if pos < 0 else pos

def filter_dict_by_key(d, keys):
    """Filter the dict *d* to remove keys not in *keys*."""
    return {k: v for k, v in d.items() if k in keys}

def unique_items(seq):
    """Return the unique items from iterable *seq* (in order)."""
    seen = set()
    return [x for x in seq if not (x in seen or seen.add(x))]

_ansi_re = re.compile('\033\[((?:\d|;)*)([a-zA-Z])')

def strip_ansi(value):
    """Strip the ANSI escape sequences from a string."""
    return _ansi_re.sub('', value)

def replace(s, replace):
    """Replace multiple values in a string"""
    for r in replace:
        s = s.replace(*r)
    return s


# -*- coding: utf-8 -*-
"""A tsv data output adapter"""

from __future__ import unicode_literals

from .preprocessors import bytes_to_string, override_missing_value, convert_to_string
from itertools import chain
from cli_helpers.utils import replace

supported_formats = ('tsv',)
preprocessors = (override_missing_value, bytes_to_string, convert_to_string)

def adapter(data, headers, **kwargs):
    """Wrap the formatting inside a function for TabularOutputFormatter."""
    for row in chain((headers,), data):
        yield "\t".join((replace(r, (('\n', r'\n'), ('\t', r'\t'))) for r in row))


# -*- coding: utf-8 -*-
"""Common development tasks for setup.py to use."""

import re
import subprocess
import sys

from setuptools import Command

class BaseCommand(Command, object):
    """The base command for project tasks."""

    user_options = []

    default_cmd_options = ('verbose', 'quiet', 'dry_run')

    def __init__(self, *args, **kwargs):
        super(BaseCommand, self).__init__(*args, **kwargs)
        self.verbose = False

    def initialize_options(self):
        """Override the distutils abstract method."""
        pass

    def finalize_options(self):
        """Override the distutils abstract method."""
        # Distutils uses incrementing integers for verbosity.
        self.verbose = bool(self.verbose)

    def call_and_exit(self, cmd, shell=True):
        """Run the *cmd* and exit with the proper exit code."""
        sys.exit(subprocess.call(cmd, shell=shell))

    def call_in_sequence(self, cmds, shell=True):
        """Run multiple commmands in a row, exiting if one fails."""
        for cmd in cmds:
            if subprocess.call(cmd, shell=shell) == 1:
                sys.exit(1)

    def apply_options(self, cmd, options=()):
        """Apply command-line options."""
        for option in (self.default_cmd_options + options):
            cmd = self.apply_option(cmd, option,
                                    active=getattr(self, option, False))
        return cmd

    def apply_option(self, cmd, option, active=True):
        """Apply a command-line option."""
        return re.sub(r'{{{}\:(?P<option>[^}}]*)}}'.format(option),
                      '\g<option>' if active else '', cmd)

class lint(BaseCommand):
    """A PEP 8 lint command that optionally fixes violations."""

    description = 'check code against PEP 8 (and fix violations)'

    user_options = [
        ('branch=', 'b', 'branch or revision to compare against (e.g. master)'),
        ('fix', 'f', 'fix the violations in place')
    ]

    def initialize_options(self):
        """Set the default options."""
        self.branch = 'master'
        self.fix = False
        super(lint, self).initialize_options()

    def run(self):
        """Run the linter."""
        cmd = 'pep8radius {branch} {{fix: --in-place}}{{verbose: -vv}}'
        cmd = cmd.format(branch=self.branch)
        self.call_and_exit(self.apply_options(cmd, ('fix', )))

class test(BaseCommand):
    """Run the test suites for this project."""

    description = 'run the test suite'

    user_options = [
        ('all', 'a', 'test against all supported versions of Python'),
        ('coverage', 'c', 'measure test coverage')
    ]

    unit_test_cmd = ('pytest{quiet: -q}{verbose: -v}{dry_run: --setup-only}'
                     '{coverage: --cov-report= --cov=cli_helpers}')
    test_all_cmd = 'tox{verbose: -v}{dry_run: --notest}'
    coverage_cmd = 'coverage report'

    def initialize_options(self):
        """Set the default options."""
        self.all = False
        self.coverage = False
        super(test, self).initialize_options()

    def run(self):
        """Run the test suites."""
        if self.all:
            cmd = self.apply_options(self.test_all_cmd)
            self.call_and_exit(cmd)
        else:
            cmds = (self.apply_options(self.unit_test_cmd, ('coverage', )), )
            if self.coverage:
                cmds += (self.apply_options(self.coverage_cmd), )
            self.call_in_sequence(cmds)

class docs(BaseCommand):
    """Use Sphinx Makefile to generate documentation."""

    description = 'generate the Sphinx HTML documentation'

    clean_docs_cmd = 'make -C docs clean'
    html_docs_cmd = 'make -C docs html'
    view_docs_cmd = 'open docs/build/html/index.html'

    def run(self):
        """Generate and view the documentation."""
        cmds = (self.clean_docs_cmd, self.html_docs_cmd, self.view_docs_cmd)
        self.call_in_sequence(cmds)