import requests
import importlib
import logging
import datetime
import pandas as pd
import pickle
from collections import defaultdict, namedtuple
from civicpy import DATA_ROOT

CACHE = dict()

CACHE_FILE = DATA_ROOT / 'CACHE.pkl'

COORDINATE_TABLE = None
COORDINATE_TABLE_START = None
COORDINATE_TABLE_STOP = None
COORDINATE_TABLE_CHR = None

HPO_TERMS = dict()

FRESH_DELTA = datetime.timedelta(days=7)

MODULE = importlib.import_module('civicpy.civic')

API_URL = 'https://civicdb.org/api'

LINKS_URL = 'https://civicdb.org/links'

UNMARKED_PLURALS = {'evidence'}

CIVIC_TO_PYCLASS = {
    'evidence_items': 'evidence'
}

CoordinateQuery = namedtuple('CoordinateQuery', ['chr', 'start', 'stop', 'alt', 'key'], defaults=(None, None))

def pluralize(string):
    if string in UNMARKED_PLURALS:
        return f'{string}_items'
    if string.endswith('s'):
        return string
    return string + 's'

def singularize(string):
    string = string.rstrip('s')
    if string == 'evidence_item':
        string = 'evidence'
    return string

def search_url(element):
    element = pluralize(element).lower()
    return '/'.join([API_URL, element, 'search'])

def snake_to_camel(snake_string):
    words = snake_string.split('_')
    cap_words = [x.capitalize() for x in words]
    return ''.join(cap_words)

def element_lookup_by_id(element_type, element_id):
    e_string = pluralize(element_type.lower())
    url = '/'.join([API_URL, e_string, str(element_id)])
    resp = requests.get(url)
    resp.raise_for_status()
    resp_dict = resp.json()
    return resp_dict

def get_class(element_type):
    e_string = singularize(element_type)
    class_string = snake_to_camel(e_string)
    cls = getattr(MODULE, class_string, CivicAttribute)
    return cls

def save_cache():
    with open(CACHE_FILE, 'wb') as pf:
        pickle.dump(CACHE, pf)

def load_cache():
    with open(CACHE_FILE, 'rb') as pf:
        old_cache = pickle.load(pf)
    c = dict()
    variants = set()
    for k, v in old_cache.items():
        if isinstance(k, str):
            c[k] = v
        elif isinstance(k, int):
            c[hash(v)] = v
            if v.type == 'variant':
                variants.add(v)
        else:
            raise ValueError
    MODULE.CACHE = c
    for k, v in MODULE.CACHE.items():
        if isinstance(k, str):
            continue
        v.update()
    _build_coordinate_table(variants)

class CivicRecord:

    _SIMPLE_FIELDS = {'id', 'type'}
    _COMPLEX_FIELDS = set()

    def __init__(self, partial=False, **kwargs):
        self._incomplete = set()
        self._partial = partial
        simple_fields = sorted(self._SIMPLE_FIELDS, reverse=True)
        simple_fields = sorted(simple_fields, key=lambda x: x in CivicRecord._SIMPLE_FIELDS, reverse=True)
        for field in simple_fields:
            try:
                self.__setattr__(field, kwargs[field])
            except KeyError:
                try:
                    object.__getattribute__(self, field)
                except AttributeError:
                    if partial and field not in CivicRecord._SIMPLE_FIELDS:
                        self._incomplete.add(field)     # Allow for incomplete data when partial flag set
                    else:
                        raise AttributeError(f'Expected {field} attribute for {self.type}, none found.')

        for field in self._COMPLEX_FIELDS:
            try:
                v = kwargs[field]
            except KeyError:
                if partial:
                    self._incomplete.add(field)
                    continue
                else:
                    raise AttributeError(f'Expected {field} attribute for {self.type}, none found.')
            is_compound = isinstance(v, list)
            cls = get_class(field)
            if is_compound:
                result = list()
                for data in v:
                    if isinstance(data, dict):
                        data['type'] = data.get('type', singularize(field))
                        result.append(cls(partial=True, **data))
                    else:
                        result.append(data)
                self.__setattr__(field, result)
            else:
                t = v.get('type', field)
                v['type'] = CIVIC_TO_PYCLASS.get(t, t)
                self.__setattr__(field, cls(partial=True, **v))

        self._partial = bool(self._incomplete)
        if not isinstance(self, CivicAttribute) and not self._partial and self.__class__.__name__ != 'CivicRecord':
            CACHE[hash(self)] = self

    def __repr__(self):
        return f'<CIViC {self.type} {self.id}>'

    def __getattr__(self, item):
        if self._partial and item in self._incomplete:
            self.update()
        return object.__getattribute__(self, item)

    def __hash__(self):
        return hash(f'{self.type}:{self.id}')

    def __eq__(self, other):
        return hash(self) == hash(other)

    def __setstate__(self, state):
        self.__dict__ = state

    def update(self, allow_partial=True, force=False, **kwargs):
        """Updates record and returns True if record is complete after update, else False."""
        if kwargs:
            self.__init__(partial=allow_partial, force=force, **kwargs)
            return not self._partial

        if not force and CACHE.get(hash(self)):
            cached = CACHE[hash(self)]
            for field in self._SIMPLE_FIELDS | self._COMPLEX_FIELDS:
                v = getattr(cached, field)
                setattr(self, field, v)
            self._partial = False
            logging.info(f'Loading {str(self)} from cache')
            return True
        resp_dict = element_lookup_by_id(self.type, self.id)
        self.__init__(partial=False, **resp_dict)
        return True

    @property
    def site_link(self):
        return '/'.join([LINKS_URL, self.type, str(self.id)])

class Variant(CivicRecord):
    _SIMPLE_FIELDS = CivicRecord._SIMPLE_FIELDS.union({
        'allele_registry_id',
        'civic_actionability_score',
        'description',
        'entrez_id',
        'entrez_name',
        'gene_id',
        'name'})
    _COMPLEX_FIELDS = CivicRecord._COMPLEX_FIELDS.union({
        'assertions',
        'clinvar_entries',
        'coordinates',
        # 'errors',
        'evidence_items',
        'hgvs_expressions',
        # 'lifecycle_actions',
        # 'provisional_values',
        'sources',
        'variant_aliases',
        'variant_groups',
        'variant_types'})

    def __init__(self, **kwargs):
        # Handle overloaded evidence_items from some advanced search views
        evidence_items = kwargs.get('evidence_items')
        kwargs['type'] = 'variant'
        if evidence_items and not isinstance(evidence_items, list):
                del(kwargs['evidence_items'])
        super().__init__(**kwargs)

    @property
    def evidence_sources(self):
        sources = set()
        for evidence in self.evidence_items:
            if evidence.source is not None:
                sources.add(evidence.source)
        return sources

    @property
    def aliases(self):
        return self.variant_aliases

    @property
    def groups(self):
        return self.variant_groups

    @property
    def types(self):
        return self.variant_types

    @property
    def evidence(self):
        return self.evidence_items

    @property
    def gene(self):
        return _get_element_by_id('gene', self.gene_id)

class Gene(CivicRecord):
    _SIMPLE_FIELDS = CivicRecord._SIMPLE_FIELDS.union(
        {'description', 'entrez_id', 'name'})
    _COMPLEX_FIELDS = CivicRecord._COMPLEX_FIELDS.union({
        'aliases',
        # 'errors',                 # TODO: Add support for these fields in advanced search endpoint
        # 'lifecycle_actions',
        # 'provisional_values',
        # 'sources',
        'variants'
    })

class Evidence(CivicRecord):
    _SIMPLE_FIELDS = CivicRecord._SIMPLE_FIELDS.union({
        'clinical_significance',
        'description',
        'drug_interaction_type',
        'evidence_direction',
        'evidence_level',
        'evidence_type',
        'gene_id',
        'name',
        'open_change_count',
        'rating',
        'status',
        'variant_id',
        'variant_origin'})
    _COMPLEX_FIELDS = CivicRecord._COMPLEX_FIELDS.union({
        'assertions',
        'disease',
        'drugs',
        'errors',
        'fields_with_pending_changes',
        'lifecycle_actions',
        'phenotypes',
        'source'})

    @property
    def variant(self):
        return get_variants_by_ids([self.variant_id])[0]

class Assertion(CivicRecord):
    _SIMPLE_FIELDS = CivicRecord._SIMPLE_FIELDS.union({
        'allele_registry_id',
        'amp_level',
        'clinical_significance',
        'description',
        'drug_interaction_type',
        'evidence_direction',
        'evidence_item_count',
        'evidence_type',
        'fda_companion_test',
        'fda_regulatory_approval',
        'name',
        'nccn_guideline',
        'nccn_guideline_version',
        'open_change_count',
        'pending_evidence_count',
        'status',
        'summary',
        'variant_origin'
    })

    _COMPLEX_FIELDS = CivicRecord._COMPLEX_FIELDS.union({
        'acmg_codes',
        'disease',
        'drugs',
        'evidence_items',
        'gene',
        'lifecycle_actions',
        'phenotypes',
        'variant'
    })

    @property
    def evidence(self):
        return self.evidence_items

    @property
    def hpo_ids(self):
        return [x.hpo_id for x in self.phenotypes if x.hpo_id]

class CivicAttribute(CivicRecord, dict):

    _SIMPLE_FIELDS = {'type'}
    _COMPLEX_FIELDS = set()

    def __repr__(self):
        try:
            _id = self.id
        except AttributeError:
            return f'<CIViC Attribute {self.type}>'
        else:
            return f'<CIViC Attribute {self.type} {self.id}>'

    def __init__(self, **kwargs):
        kwargs['partial'] = False
        for k, v in kwargs.items():
            self.__setattr__(k, v)
        super().__init__(**kwargs)

    def __hash__(self):
        try:
            _id = self.id
        except AttributeError:
            raise NotImplementedError
        if _id is not None:
            return CivicRecord.__hash__(self)
        else:
            raise ValueError

    @property
    def site_link(self):
        return None

    def update(self):
        return NotImplementedError

class Drug(CivicAttribute):
    _SIMPLE_FIELDS = CivicRecord._SIMPLE_FIELDS.union({'pubchem_id'})

class Disease(CivicAttribute):
    _SIMPLE_FIELDS = CivicRecord._SIMPLE_FIELDS.union({'display_name', 'doid', 'url'})

def get_cached(element_type, element_id):
    klass = get_class(element_type)
    r = klass(type=element_type, id=element_id, partial=True)
    return CACHE.get(hash(r), False)

def _has_all_cached_fresh(element):
    s = '{}_all_cached'.format(pluralize(element))
    if CACHE.get(s, False):
        return CACHE[s] + FRESH_DELTA > datetime.datetime.now()
    return False

def _get_elements_by_ids(element, id_list=[], allow_cached=True, get_all=False):
    if allow_cached and not get_all:
        cached = [get_cached(element, element_id) for element_id in id_list]
        if all(cached):
            logging.info(f'Loading {pluralize(element)} from cache')
            return cached
    elif allow_cached and _has_all_cached_fresh(element):
        cached = [get_cached(element, element_id) for element_id in CACHE['{}_all_ids'.format(pluralize(element))]]
        logging.info(f'Loading {pluralize(element)} from cache')
        return cached
    if id_list and get_all:
        raise ValueError('Please pass list of ids or use the get_all flag, not both.')
    if get_all:
        payload = _construct_get_all_payload()
        logging.warning('Getting all {}. This may take a couple of minutes...'.format(pluralize(element)))
    else:
        payload = _construct_query_payload(id_list)
    url = search_url(element)
    response = requests.post(url, json=payload)
    response.raise_for_status()
    cls = get_class(element)
    elements = [cls(**x) for x in response.json()['results']]
    CACHE['{}_all_cached'.format(pluralize(element))] = datetime.datetime.now()
    CACHE['{}_all_ids'.format(pluralize(element))] = [x['id'] for x in response.json()['results']]
    return elements

def _get_element_by_id(element, id, allow_cached=True):
    return _get_elements_by_ids(element, [id], allow_cached)[0]

def _construct_get_all_payload():
    queries = [
        {
            'field': 'id',
            'condition': {
                'name': 'is_greater_than',
                'parameters': [
                    -1
                ]
            }
        }
    ]
    payload = {
        'operator': 'OR',
        'queries': queries
    }
    return payload

def _construct_query_payload(id_list):
    queries = list()
    for element_id in id_list:
        query = {
            'field': 'id',
            'condition': {
                'name': 'is_equal_to',
                'parameters': [
                    element_id
                ]
            }
        }
        queries.append(query)
    payload = {
        'operator': 'OR',
        'queries': queries
    }
    return payload

def get_assertions_by_ids(assertion_id_list=[], get_all=False):
    logging.info('Getting assertions...')
    assertions = _get_elements_by_ids('assertion', assertion_id_list, get_all=get_all)
    logging.info('Caching variant details...')
    variant_ids = [x.variant.id for x in assertions]    # Add variants to cache
    _get_elements_by_ids('variant', variant_ids)
    logging.info('Caching gene details...')
    gene_ids = [x.gene.id for x in assertions]          # Add genes to cache
    _get_elements_by_ids('gene', gene_ids)
    for assertion in assertions:                        # Load from cache
        assertion.variant.update()
        assertion.gene.update()
    return assertions

def get_assertion_by_id(assertion_id):
    return get_assertions_by_ids([assertion_id])[0]

def get_all_assertion_ids():
    return _get_all_element_ids('assertions')

def get_all_assertions():
    return get_assertions_by_ids(get_all=True)

def search_assertions_by_coordinates(coordinates, search_mode='any'):
    variants = search_variants_by_coordinates(coordinates, search_mode=search_mode)
    assertions = set()
    for v in variants:
        if v.assertions:
            assertions.update(v.assertions)
    return list(assertions)

def get_variants_by_ids(variant_id_list):
    logging.info('Getting variants...')
    variants = _get_elements_by_ids('variant', variant_id_list)
    gene_ids = set()
    for variant in variants:
        gene_ids.add(variant.gene_id)
    if gene_ids:
        logging.info('Caching gene details...')
        _get_elements_by_ids('gene', gene_ids)
    return variants

def get_variant_by_id(variant_id):
    return get_variants_by_ids([variant_id])[0]

def _build_coordinate_table(variants):
    variant_records = list()
    for v in variants:
        c = v.coordinates
        start = getattr(c, 'start', None)
        stop = getattr(c, 'stop', None)
        chr = getattr(c, 'chromosome', None)
        alt = getattr(c, 'variant_bases', None)
        if all([start, stop, chr]):
            variant_records.append([chr, start, stop, alt, hash(v)])
        else:
            continue
        start = getattr(c, 'start2', None)
        stop = getattr(c, 'stop2', None)
        chr = getattr(c, 'chromosome2', None)
        if all([start, stop, chr]):
            variant_records.append([chr, start, stop, None, hash(v)])
    df = pd.DataFrame.from_records(
        variant_records,
        columns=['chr', 'start', 'stop', 'alt', 'v_hash']
    ).sort_values(by=['chr', 'start', 'stop', 'alt'])
    MODULE.COORDINATE_TABLE = df
    MODULE.COORDINATE_TABLE_START = df.start.sort_values()
    MODULE.COORDINATE_TABLE_STOP = df.stop.sort_values()
    MODULE.COORDINATE_TABLE_CHR = df.chr.sort_values()

def get_all_variants(allow_cached=True):
    precached = _has_all_cached_fresh('variants')
    variants = _get_all_genes_and_variants(allow_cached)['variants']
    if not (precached and allow_cached):
        _build_coordinate_table(variants)
    return variants

def search_variants_by_coordinates(coordinate_query, search_mode='any'):
    """
    Search the cache for variants matching provided coordinates using the corresponding search mode.

    :param coordinate_query: A civic CoordinateQuery object
                        start: the genomic start coordinate of the query
                        stop: the genomic end coordinate of the query
                        chr: the GRCh37 chromosome of the query (e.g. "7", "X")
                        alt: the alternate allele at the coordinate [optional]

    :param search_mode: ['any', 'include_smaller', 'include_larger', 'exact']
                        any: any overlap between a query and a variant is a match
                        include_smaller: variants must fit within the coordinates of the query
                        include_larger: variants must encompass the coordinates of the query
                        exact: variants must match coordinates precisely, as well as alternate
                               allele, if provided
                        search_mode is 'exact' by default

    :return:            Returns a list of variant hashes matching the coordinates and search_mode
    """
    get_all_variants()
    ct = COORDINATE_TABLE
    start_idx = COORDINATE_TABLE_START
    stop_idx = COORDINATE_TABLE_STOP
    chr_idx = COORDINATE_TABLE_CHR
    start = int(coordinate_query.start)
    stop = int(coordinate_query.stop)
    chromosome = str(coordinate_query.chr)
    # overlapping = (start <= ct.stop) & (stop >= ct.start)
    left_idx = chr_idx.searchsorted(chromosome)
    right_idx = chr_idx.searchsorted(chromosome, side='right')
    chr_ct_idx = chr_idx[left_idx:right_idx].index
    right_idx = start_idx.searchsorted(stop, side='right')
    start_ct_idx = start_idx[:right_idx].index
    left_idx = stop_idx.searchsorted(start)
    stop_ct_idx = stop_idx[left_idx:].index
    match_idx = chr_ct_idx & start_ct_idx & stop_ct_idx
    m_df = ct.loc[match_idx, ]
    if search_mode == 'any':
        var_digests = m_df.v_hash.to_list()
        return [CACHE[v] for v in var_digests]
    elif search_mode == 'include_smaller':
        match_idx = (start <= m_df.start) & (stop >= m_df.stop)
    elif search_mode == 'include_larger':
        match_idx = (start >= m_df.start) & (stop <= m_df.stop)
    elif search_mode == 'exact':
        match_idx = (start == m_df.stop) & (stop == m_df.start)
        if coordinate_query.alt:
            match_idx = match_idx & (coordinate_query.alt == m_df.alt)
    else:
        raise ValueError("unexpected search mode")
    var_digests = m_df.loc[match_idx,].v_hash.to_list()
    return [CACHE[v] for v in var_digests]

# TODO: Refactor this method
def bulk_search_variants_by_coordinates(sorted_queries, search_mode='any'):
    """
    An interator to search the cache for variants matching the set of sorted coordinates and yield
    matches corresponding to the search mode.

    :param sorted_queries:  A list of civic CoordinateQuery objects, sorted by coordinate.
                            start: the genomic start coordinate of the query
                            stop: the genomic end coordinate of the query
                            chr: the GRCh37 chromosome of the query (e.g. "7", "X")
                            alt: the alternate allele at the coordinate [optional]

    :param search_mode: ['any', 'include_smaller', 'include_larger', 'exact']
                        any: any overlap between a query and a variant is a match
                        include_smaller: variants must fit within the coordinates of the query
                        include_larger: variants must encompass the coordinates of the query
                        exact: variants must match coordinates precisely, as well as alternate
                               allele, if provided
                        search_mode is 'exact' by default

    :yield:            Yields (query, match) tuples for each identified match
    """

    def is_sorted(prev_q, current_q):
        if prev_q['chr'] < current_q['chr']:
            return True
        if prev_q['chr'] > current_q['chr']:
            return False
        if prev_q['start'] < current_q['start']:
            return True
        if prev_q['start'] > current_q['start']:
            return False
        if prev_q['stop'] < current_q['stop']:
            return True
        if prev_q['stop'] > current_q['stop']:
            return False
        return True

    ct_pointer = 0
    query_pointer = 0
    last_query_pointer = -1
    match_start = None
    ct = MODULE.COORDINATE_TABLE
    matches = defaultdict(list)
    Match = namedtuple('Match', ct.columns)
    while query_pointer < len(sorted_queries) and ct_pointer < len(ct):
        if last_query_pointer != query_pointer:
            q = sorted_queries[query_pointer]
            if match_start is not None:
                ct_pointer = match_start
                match_start = None
            last_query_pointer = query_pointer
        c = ct.iloc[ct_pointer]
        q_chr = str(q.chr)
        c_chr = c.chr
        if q_chr < c_chr:
            query_pointer += 1
            continue
        if q_chr > c_chr:
            ct_pointer += 1
            continue
        q_start = int(q.start)
        c_start = c.start
        q_stop = int(q.stop)
        c_stop = c.stop
        if q_start > c_stop:
            ct_pointer += 1
            continue
        if q_stop < c_start:
            query_pointer += 1
            continue
        if search_mode == 'any':
            matches[q].append(c.to_dict())
        elif search_mode == 'exact' and q_start == c_start and q_stop == c_stop:
            q_alt = q.alt
            c_alt = c.alt
            if not (q_alt and c_alt and q_alt != c_alt):
                matches[q].append(Match(**c.to_dict()))
        elif search_mode == 'include_smaller':
            raise NotImplementedError
        elif search_mode == 'include_larger':
            raise NotImplementedError
        if match_start is None:
            match_start = ct_pointer
        ct_pointer += 1
    return dict(matches)

def get_all_variant_ids():
    return _get_all_element_ids('variants')

def _get_all_genes_and_variants(allow_cached=True):
    variants = _get_elements_by_ids('variants', get_all=True, allow_cached=allow_cached)
    genes = _get_elements_by_ids('gene', get_all=True, allow_cached=allow_cached)
    for variant in variants:
        variant.gene.update()
    return {'genes': genes, 'variants': variants}

def _get_all_element_ids(element):
    url = f'https://civicdb.org/api/{element}?count=100000'
    resp = requests.get(url)
    resp.raise_for_status()
    return [x['id'] for x in resp.json()['records']]

def get_genes_by_ids(gene_id_list):
    logging.info('Getting genes...')
    genes = _get_elements_by_ids('gene', gene_id_list)  # Advanced search results are incomplete
    variant_ids = set()
    for gene in genes:
        for variant in gene.variants:
            variant_ids.add(variant.id)
    if variant_ids:
        logging.info('Caching variant details...')
        _get_elements_by_ids('variant', variant_ids)
    for gene in genes:
        for variant in gene.variants:
            variant.update()
    return genes

def get_gene_by_id(gene_id):
    return get_genes_by_ids([gene_id])[0]

def get_all_gene_ids():
    return _get_all_element_ids('genes')

def get_all_genes():
    return _get_all_genes_and_variants()['genes']

def get_all_evidence_ids():
    return _get_all_element_ids('evidence_items')

def get_HPO_terms_by_ids(hpo_id_list):
    if not HPO_TERMS:
        _load_HPO()
    return [HPO_TERMS[x] for x in hpo_id_list]

def _load_HPO():
    url = 'https://civicdb.org/api/phenotypes?count=100000'
    resp = requests.get(url)
    resp.raise_for_status()
    for h in resp.json():
        HPO_TERMS[h['id']] = h


"""
Command-line interface for asana-hub.
"""

import argparse
import logging
import sys
import os
import traceback
import multiprocessing

try:
    from asana import Client
    from asana import error as asana_errors
    from github import Github

    import urllib3
    import certifi

    # Setup pool manager and SSL verification.
    http = urllib3.PoolManager(
        cert_reqs='CERT_REQUIRED', # Force certificate check.
        ca_certs=certifi.where(),  # Path to the Certifi bundle.
    )

    try:
        import urllib3.contrib.pyopenssl
        urllib3.contrib.pyopenssl.inject_into_urllib3()
    except:
        logging.debug("pyopenssl not detected.\n"
            "to install pyopenssl: "
            "pip install pyopenssl ndg-httpsclient pyasn1")

except ImportError:
    raise Exception("Could not import required packages.\n"
        "Did you pip install -r requirements.txt ?")

import transport

from .json_data import JSONData
from .action import Action

class ToolApp(object):

    """Represents the AsanaHub app.

    Attributes:

        mp:
            `multiprocessing.Pool`. Pool for multithreaded processing.
    """

    @classmethod
    def uniqify(cls, seq):
        """Returns a unique list of seq"""
        seen = set()
        seen_add = seen.add
        return [ x for x in seq if x not in seen and not seen_add(x)]

    def authenticate(self):
        """Connects to Github and Asana and authenticates via OAuth."""
        if self.oauth:
            return False

        # Save asana.
        self.settings.apply('api-asana', self.args.asana_api,
            "enter asana api key")

        # Save github.com
        self.settings.apply('api-github', self.args.github_api,
            "enter github.com token")

        logging.debug("authenticating asana api.")
        self.asana = Client.basic_auth(self.settings['api-asana'])
        self.asana_errors = asana_errors
        self.asana_me = self.asana.users.me()
        logging.debug("authenticating github api")
        self.github = Github(self.settings['api-github'])
        self.github_user = self.github.get_user()

        self.oauth = True

    @classmethod
    def _list_select(cls, lst, prompt, offset=0):
        """Given a list of values and names, accepts the index value or name."""

        inp = raw_input("select %s: " % prompt)
        assert inp, "value required."

        try:
            return lst[int(inp)+offset]
        except ValueError:
            return inp
        except IndexError:
            assert False, "bad value."

    def prompt_repo(self):

        # Select org
        orgs = [None]
        print "0) %s [you]" % self.github_user.name
        for idx, org in enumerate(self.github_user.get_orgs()):
            orgs.append(org)
            print "%d) %s" % (idx+1, org.name)

        org = self._list_select(orgs, "org")
        ctx = org or self.github_user

        repos = []
        for idx, repo in enumerate(ctx.get_repos()):
            repos.append(repo)
            print "%d) %s" % (idx, repo.name)

        repo = self._list_select(repos, "repo")
        logging.debug("repo id: %d", repo.id)

        return repo

    def prompt_project(self):

        # Select workspace
        workspaces = []
        for idx, workspace in enumerate(self.asana_me['workspaces']):
            workspaces.append(workspace)
            print "%d) %s" % (idx, workspace['name'])

        workspace = self._list_select(workspaces, "workspace")

        # Select workspace
        as_projects = self.asana.projects.find_by_workspace(workspace['id'],
            iterator_type=None)

        projects = []

        for idx, project in enumerate(as_projects):
            projects.append(project)
            print "%d) %s" % (idx, project['name'])

        project = self._list_select(projects, "project")

        return project

    @classmethod
    def make_asana_url(cls, project_id, task_id):
        """Returns a URL to an asana task."""
        return "https://app.asana.com/0/%d/%d" % (project_id, task_id)

    ##################
    ### Issue Data ###
    ##################

    @classmethod
    def _issue_data_key(cls, namespace):
        """Returns key for issue_data in data."""
        return 'issue_data_%s' % namespace

    def save_issue_data_task(self, issue, task_id, namespace='open'):
        """Saves a issue data (tasks, etc.) to local data.

        Args:
            issue:
                `int`. Github issue number.
            task:
                `int`. Asana task ID.
            namespace:
                `str`. Namespace for storing this issue.
        """

        issue_data = self.get_saved_issue_data(issue, namespace)

        if not issue_data.has_key('tasks'):
            issue_data['tasks'] = [task_id]
        elif task_id not in issue_data['tasks']:
            issue_data['tasks'].append(task_id)

    def has_saved_issue_data(self, issue, namespace='open'):
        issue_data_key = self._issue_data_key(namespace)
        issue_data = self.data.get(issue_data_key,
            {})

        if isinstance(issue, int):
            issue_number = str(issue)
        elif isinstance(issue, basestring):
            issue_number = issue
        else:
            issue_number = issue.number

        return issue_data.has_key(str(issue_number))

    def get_saved_issue_data(self, issue, namespace='open'):
        """Returns issue data from local data.

        Args:
            issue:
                `int`. Github issue number.
            namespace:
                `str`. Namespace for storing this issue.
        """

        if isinstance(issue, int):
            issue_number = str(issue)
        elif isinstance(issue, basestring):
            issue_number = issue
        else:
            issue_number = issue.number

        issue_data_key = self._issue_data_key(namespace)
        issue_data = self.data.get(issue_data_key,
            {})

        _data = issue_data.get(str(issue_number), {})
        issue_data[str(issue_number)] = _data
        return _data

    def move_saved_issue_data(self, issue, ns, other_ns):
        """Moves an issue_data from one namespace to another."""

        if isinstance(issue, int):
            issue_number = str(issue)
        elif isinstance(issue, basestring):
            issue_number = issue
        else:
            issue_number = issue.number

        issue_data_key = self._issue_data_key(ns)
        other_issue_data_key = self._issue_data_key(other_ns)
        issue_data = self.data.get(issue_data_key,
            {})
        other_issue_data = self.data.get(other_issue_data_key,
            {})

        _id = issue_data.pop(issue_number, None)
        if _id:
            other_issue_data[issue_number] = _id

        self.data[other_issue_data_key] = other_issue_data
        self.data[issue_data_key] = issue_data

    #################
    ### Task Data ###
    #################

    @classmethod
    def _task_data_key(cls):
        """Returns key for task_data in data."""
        return 'task-data'

    def has_saved_task_data(self, task):
        task_data_key = self._task_data_key()
        task_data = self.data.get(task_data_key,
            {})

        if isinstance(task, int):
            task_number = str(task)
        elif isinstance(task, basestring):
            task_number = task
        else:
            task_number = task['id']

        return task_data.has_key(str(task_number))

    def get_saved_task_data(self, task):
        """Returns task data from local data.

        Args:
            task:
                `int`. Asana task number.
        """

        if isinstance(task, int):
            task_number = str(task)
        elif isinstance(task, basestring):
            task_number = task
        else:
            task_number = task['id']

        task_data_key = self._task_data_key()
        task_data = self.data.get(task_data_key, {})

        _data = task_data.get(str(task_number), {})
        task_data[str(task_number)] = _data
        return _data

    #############
    ### Misc. ###
    #############

    def get_asana_task(self, asana_task_id):
        """Retrieves a task from asana."""

        try:
            return self.asana.tasks.find_by_id(asana_task_id)
        except asana_errors.NotFoundError:
            return None
        except asana_errors.ForbiddenError:
            return None

    def sync_data(self):

        # Updates transport data
        transport.data.update(self.data.data)

    def flush(self):

        transport.flush(callback=self.flush_settings)

    def flush_settings(self):

        for setting in transport.iter_settings():
            task = setting.pop('task')
            if task == "save_issue_data_task":
                self.save_issue_data_task(**setting)
            elif task == "add_tags_to_task":
                self.add_tags_to_task(**setting)
            else:
                raise Exception("Unknown settings task: %s" % task)

    def add_tags_to_task(self, task_id, tag_ids):
        task_data = self.get_saved_task_data(task_id)
        task_tag_ids = task_data.get('tags') or []

        logging.debug("\t\t - added %d tags to %s", len(tag_ids), task_id)
        task_data['tags'] = self.uniqify(task_tag_ids + tag_ids)

    def __init__(self, version):
        """Accepts version of the app."""

        # Setup settings
        self.version = version
        self.exit_code = 999
        self.oauth = False

        # Setup logging
        self.logger = logging.getLogger()
        formatter = logging.Formatter("%(message)s")
        self.logger.setLevel(logging.DEBUG)
        self.logger.propagate = False
        self.logger.handlers = []
        ch = logging.StreamHandler()
        ch.setLevel(logging.INFO)
        ch.setFormatter(formatter)
        self.logger.addHandler(ch)

        parser = argparse.ArgumentParser(description='Loudr Utility Tool')

        # Default paths for settings and data
        def_settings_file = os.path.join(
            os.path.expanduser("~"),
            '.asana-hub',
            )

        def_data_file = os.path.expanduser("./.asana-hub.proj")

        # Load actions
        actions = {}
        choices = []
        help_msgs = ''

        for action in Action.iter_actions():
            actions[action.name] = action
            choices.append(action.name)
            help_msgs += '\n-  %s: %s' % (action.name, action.__doc__)

        help_msgs += '\n'

        parser.add_argument(
            'action',
            action='store',
            nargs=1,
            help=help_msgs,
            choices=choices,
            )

        parser.add_argument(
            '-vv', '--verbose',
            action='store_true',
            dest='verbose',
            help="use debugging verbosity.",
            )

        parser.add_argument(
            '-s', '--settings-file',
            action='store',
            nargs='?',
            dest='settings_file',
            default=def_settings_file,
            help="path to save oauth api keys.",
            )

        parser.add_argument(
            '-d', '--data-file',
            action='store',
            nargs='?',
            dest='data_file',
            default=def_data_file,
            help="path to save repository and project based data.",
            )

        # Add action arguments.
        for action in actions.values():
            action.add_arguments(parser)

        # Add actions from the parent class. (The settings)
        Action.add_arguments(parser)

        parser.add_argument('-v', '--version', action='version',
            version='%(prog)s ' + '%s' % version)

        self.args = parser.parse_args()

        if len(sys.argv) < 2:
            parser.print_help()
            self.exit_code = 1
            return

        if self.args.verbose:
            ch.setLevel(logging.DEBUG)

        logging.debug("Loading settings")

        # Load settings
        self.settings = JSONData(filename=self.args.settings_file,
            args=self.args, version=version)

        self.data = JSONData(filename=self.args.data_file,
            args=self.args, version=version)

        # Load action method and call.
        try:
            action_name = self.args.action[0]
            action_class = actions.get(action_name)
            if not action_class:
                raise NotImplementedError(
                    "%s is not implemented." % action_name)

            # Instantiate and run
            action = action_class(app=self, args=self.args)

            # Authenticate app
            self.authenticate()

            # Begin transporters
            transport.start(self)

            # Run action
            action.run()

            # Flush transport, calling flush_settings every pass
            transport.flush(callback=self.flush_settings)

            # Flush settings queue
            logging.debug("Flushing settings updates")
            self.flush_settings()

        except AssertionError as exc:
            logging.error("Error: %s", unicode(exc))
            logging.debug("%s", traceback.format_exc())
            self.exit_code = 1
            return
        except Exception as exc:
            logging.exception("Exception: %r", exc)
            self.exit_code = 129
            return
        finally:

            # Shutdown transport
            transport.shutdown()

            # Save settings
            self.settings.save()
            # Save data
            self.data.save()

        self.exit_code = 0


"""
Maintains JSON based data file manipulated as a dictionary.
"""

import json

class JSONData(object):

    def __init__(self, filename, args, version):
        """
        Args:
            filename:
                Filename for database.
            args:
                Program arguments.
            version:
                Version of file.
        """
        self.args = args
        self.version = version

        self.filename = filename

        try:
            with open(self.filename, 'rb') as file:
                self.data = json.load(file)
        except IOError:
            self.data = {}

    def assert_version(self):
        """Asserts that the version and data file exists."""

        if not self.has_key('version'):
            raise Exception("`asana-hub connect` must be run first.")

        self.data['version'] = self.version

    def save(self):
        """Save data."""

        with open(self.filename, 'wb') as file:
            self.prune()
            self.data['version'] = self.version
            json.dump(self.data,
                file,
                sort_keys=True, indent=2)

    def __setitem__(self, key, value):
        """Set a value by key."""
        self.data[key] = value

    def __getitem__(self, key):
        """Get a value by key."""
        return self.data[key]

    def prune(self, data=None):
        if data is None:
            data = self.data

        empty_keys = [k for k, v in data.iteritems() if not v]
        for k in empty_keys:
            del data[k]

        for v in data.values():
            if isinstance(v, dict):
                self.prune(data=v)

    def apply(self, key, value, prompt=None,
        on_load=lambda a: a, on_save=lambda a: a):
        """Applies a setting value to a key, if the value is not `None`.

        Returns without prompting if either of the following:
            * `value` is not `None`
            * already present in the dictionary

        Args:
            prompt:
                May either be a string to prompt via `raw_input` or a
                method (callable) that returns the value.

            on_load:
                lambda. Value is passed through here after loaded.

            on_save:
                lambda. Value is saved as this value.
        """

        # Reset value if flag exists without value
        if value == '':
            value = None
            if key and self.data.has_key(key): del self.data[key]

        # If value is explicitly set from args.
        if value is not None:
            value = on_load(value)
            if key: self.data[key] = on_save(value)
            return value

        elif not key or not self.has_key(key):
            if callable(prompt):
                value = prompt()
            elif prompt is not None:
                value = raw_input(prompt + ": ")

            if value is None:
                if self.data.has_key(key): del self.data[key]
                return None

            self.data[key] = on_save(value)
            return value

        return on_load(self.data[key])

    def assert_key(self, key):
        assert self.data.get(key), "%s missing from data" % key

    def has_key(self, *args, **kwargs):
        return self.data.has_key(*args, **kwargs)

    def get(self, key, default_value=None):
        try:
            return self.data[key]
        except KeyError:
            if default_value is not None:
                self.data[key] = default_value

            return default_value


"""
issue action

Creates a new github issue and asana task.

"""

import logging

from ..action import Action

class PullRequest(Action):
    """Creates a new github pull-request for an exist issue and a branch."""

    # name of action
    name = "pr"

    @classmethod
    def add_arguments(cls, parser):
        """Add arguments to the parser for collection in app.args.

        Args:
            parser:
                `argparse.ArgumentParser`. Parser.
                Arguments added here are server on
                self.args.
        """

        parser.add_argument(
            '-i', '--issue',
            action='store',
            nargs='?',
            const='',
            dest='issue',
            help="[pr] issue #",
            )

        parser.add_argument(
            '-br', '--branch',
            action='store',
            nargs='?',
            const='',
            dest='branch',
            help="[pr] branch",
            )

        parser.add_argument(
            '-tbr', '--target-branch',
            action='store',
            nargs='?',
            const='',
            default='master',
            dest='target_branch',
            help="[pr] name of branch to pull changes into\n(defaults to: master)",
            )

    def run(self):
        app = self.app

        repo, project = self.get_repo_and_project()
        project_id = project['id']

        # Collect title and body
        issue = app.settings.apply(None, app.args.issue,
            "issue # to create PR for",
            )

        assert issue, "issue required"
        issue = repo.get_issue(int(issue))
        assert issue, "issue could not be found"

        branch = app.settings.apply(None, app.args.branch,
            "branch to create pull-request from [via api]",
            ) or ''

        # Get issue data to create pull request tasks list.
        issue_data = app.get_saved_issue_data(issue)

        issue_tasks = issue_data.get('tasks', [])
        issue_data['tasks'] = issue_tasks

        # pull_requests is a list of pull request numbers
        issue_prs = issue_data.get('pull_requests', [])
        issue_data['pull_requests'] = issue_prs

        asana_msgs = ''
        for task in issue_tasks:
            asana_msgs += '\n * [#%d](%s)' % (
                task,
                app.make_asana_url(project_id, task)
                )

        if asana_msgs:
            asana_msgs = '## Asana Tasks\n' + asana_msgs

        # Create pull request.
        pull_request = repo.create_pull(
            title=issue.title,
            head=branch,
            base="master",
            body="Fixes #%d - %s\n"
                "\n## Testing:\n\n"
                "%s"
                "" % (
                    issue.number,
                    issue.title,
                    asana_msgs,
                    )
            )

        # Post asana task.
        title = "PR #%d" % pull_request.number
        body = "%s" % (
            pull_request.html_url
            )

        task_ids = []
        for task_id in issue_tasks:
            task = app.asana.tasks.add_subtask(
                task_id,
                {
                'name': title,
                'notes': body,
                # TODO: Correct assignee.
                'assignee': 'me',
                })
            task_ids.append(task['id'])

            app.save_issue_data_task(issue.number, task['id'])

        # Add pull request to local data
        if pull_request.number not in issue_prs:
            issue_prs.append(pull_request.number)

        logging.info("github pull_request #%d created:\n%s\n",
            pull_request.number, pull_request.html_url)


# -*- coding: utf-8 -*-
"""
transport queue handlers

Provides an interface for multiprocessing transport queues.

"""

import logging
import re
import multiprocessing
import Queue
import urllib3
import certifi

from asana import Client
from asana import error as asana_errors
from github import Github

import tool

mem = multiprocessing.Manager()

data = mem.dict()
"""Transient data about github and asana repo."""

shutdown_event = mem.Event()
"""Shutdown event"""

queue = mem.Queue()
"""Multiprocessing transport queue."""

settings_queue = mem.Queue()
"""Multprocessing queue for updating settings."""

processes = []
"""Contains running workers."""

ASANA_SECTION_RE = re.compile(r'## Asana Tasks:\s+(.*#(\d{12,}))+', re.M)
"""Regular exprsssion to catch malformed data due to too many tasks."""

def transport_task(func):
    """Decorator for retrying tasks with special cases."""

    def wrapped_func(*args, **kwargs):
        tries = 0
        while True:
            try:
                try:
                    return func(*args, **kwargs)

                except (asana_errors.InvalidRequestError,
                        asana_errors.NotFoundError), exc:
                    logging.warn("warning: invalid request: %r", exc)

                except asana_errors.ForbiddenError, exc:
                    logging.warn("forbidden error: %r", exc)

                except asana_errors.NotFoundError, exc:
                    logging.warn("not found error: %r", exc)

                return None
            except asana_errors.RetryableAsanaError, retry_exc:
                tries += 1
                logging.warn("retry exception %r on try %d", retry_exc, tries)

                if tries >= 3:
                    raise
            except Exception, exc:
                logging.exception("Exception in transport.")
                return

    return wrapped_func

class TransportWorker(object):

    """Represents a single thread worker that responds to a queue of tasks.
    """

    def __init__(self, settings):
        self.settings = settings
        self.asana = Client.basic_auth(self.settings['api-asana'])
        self.asana_me = self.asana.users.me()
        self.github = Github(self.settings['api-github'])
        self.github_user = self.github.get_user()

    def run(self):

        while True:

            if shutdown_event.is_set():
                logging.debug("got shutdown signal")
                break

            try:
                packet = queue.get(timeout=5)
            except Queue.Empty:
                continue

            # Handle packet
            packet_task = packet.pop('task')
            method = getattr(self, packet_task, None)
            if not method:
                raise Exception("Packet method '%s' is not supported." %
                                packet_task)

            logging.debug("running packet: %s", packet_task)
            method(**packet)

    @transport_task
    def create_missing_task(self,
                            asana_workspace_id,
                            name,
                            assignee,
                            projects,
                            completed,
                            issue_number,
                            issue_html_url,
                            issue_state,
                            issue_body,
                            tasks,
                            labels,
                            label_tag_map):

        """Creates a missing task."""

        task = self.asana.tasks.create_in_workspace(
            asana_workspace_id,
            {
                'name': name,
                'notes': issue_body,
                'assignee': assignee,
                'projects': projects,
                'completed': completed,
            })

        # Announce task git issue
        task_id = task['id']

        put("create_story",
            task_id=task_id,
            text="Git Issue #%d: \n"
                  "%s" % (
                    issue_number,
                    issue_html_url,
                    )
            )

        put("apply_tasks_to_issue",
            tasks=[task_id],
            issue_number=issue_number,
            issue_body=issue_body,
            )

        # Save task to drive
        put_setting("save_issue_data_task",
                    issue=issue_number,
                    task_id=task_id,
                    namespace=issue_state)

        tasks.append(task_id)

        # Sync tags/labels
        put("sync_tags",
            tasks=tasks,
            labels=labels,
            label_tag_map=label_tag_map)

    @transport_task
    def get_repo(self):
        return self.github.get_repo(data['github-repo'])

    @transport_task
    def add_tag(self, task_id, tag_id):

        if not task_id or not tag_id:
            return

        self.asana.tasks.add_tag(task=task_id, tag=tag_id)

    @transport_task
    def sync_tags(self, tasks, labels, label_tag_map):

        for task_id in tasks:
            tag_ids = []

            added_tags = 0
            for label in labels:
                tag_id = label_tag_map.get(label)
                if not tag_id:
                    continue
                # if tag_id in tag_ids:
                #     continue
                tag_ids.append(tag_id)
                put("add_tag",
                    task_id=task_id,
                    tag_id=tag_id)
                added_tags += 1

            if added_tags:
                put_setting("add_tags_to_task",
                            task_id=task_id,
                            tag_ids=tag_ids)

    @transport_task
    def create_story(self, task_id, text):
        self.asana.stories.create_on_task(task_id,
                                          { 'text': text })

    @transport_task
    def issue_edit(self, issue_number, body):

        repo = self.get_repo()
        issue = repo.get_issue(issue_number)
        issue.edit(body=body)

    @transport_task
    def apply_tasks_to_issue(self, tasks, issue_number, issue_body):
        """Applies task numbers to an issue."""
        issue_body = issue_body
        task_numbers = format_task_numbers_with_links(tasks)
        if task_numbers:
            new_body = ASANA_SECTION_RE.sub('', issue_body)
            new_body = new_body + "\n## Asana Tasks:\n\n%s" % task_numbers
            put("issue_edit",
                issue_number=issue_number,
                body=new_body)
            return new_body

        return issue_body

    @transport_task
    def update_task(self, task_id, params):
        self.asana.tasks.update(task_id, params)

def run_worker(settings):
    try:
        worker = TransportWorker(settings)
        worker.run()
    except:
        shutdown_event.set()
        raise

def put(task, **kwargs):
    kwargs['task'] = task
    queue.put(kwargs)

def put_setting(task, **kwargs):
    """Pushes a setting to the queue."""
    kwargs['task'] = task
    settings_queue.put(kwargs)

def flush(callback=None):
    """Waits until queue is empty."""

    while True:
        if shutdown_event.is_set():
            return

        if callable(callback):
            callback()

        try:
            item = queue.get(timeout=1)
            queue.put(item)  # put it back, we're just peeking.
        except Queue.Empty:
            return

def issue_edit(issue, **kwargs):
    """Saves an issue"""
    put("issue_edit",
        issue_number=issue.number,
        **kwargs)

def task_create(asana_workspace_id, name, notes, assignee, projects,
                completed, **kwargs):
    """Creates a task"""
    put("task_create",
        asana_workspace_id=asana_workspace_id,
        name=name,
        notes=notes,
        assignee=assignee,
        projects=projects,
        completed=completed,
        **kwargs)

def start(app):

    logging.debug("Starting multiprocess workers")
    for _ in range(multiprocessing.cpu_count()):
        process = multiprocessing.Process(target=run_worker,
                                          kwargs={
                                            'settings': app.settings.data,
                                          })
        process.start()
        processes.append(process)

def shutdown():
    logging.debug("Shutting down transporter")

    shutdown_event.set()
    for p in processes:
        p.join()

def is_shutdown():
    """Returns True if the app is requesting a global shutdown."""
    return shutdown_event.is_set()

def iter_settings():
    """Yields items from the settings queue."""

    while True:
        try:
            item = settings_queue.get(timeout=1)
            yield item
        except Queue.Empty:
            return

def format_task_numbers_with_links(tasks):
    """Returns formatting for the tasks section of asana."""

    project_id = data.get('asana-project', None)

    def _task_format(task_id):
        if project_id:
            asana_url = tool.ToolApp.make_asana_url(project_id, task_id)
            return "[#%d](%s)" % (task_id, asana_url)
        else:
            return "#%d" % task_id

    return "\n".join([_task_format(tid) for tid in tasks])


"""Project"""
from __future__ import absolute_import, division, print_function, unicode_literals

class GenProject(object):

    """Genesais project annotation."""

    def __init__(self, data, gencloud):
        for field in data:
            setattr(self, field, data[field])

        self.gencloud = gencloud
        self.id = getattr(self, 'id', None)  # pylint: disable=invalid-name
        self.name = getattr(self, 'name', None)

    def data_types(self):
        """Return a list of data types."""
        data = self.gencloud.project_data(self.id)
        return sorted(set(d.type for d in data))

    def data(self, **query):
        """Query for Data object annotation."""
        data = self.gencloud.project_data(self.id)
        query['case_ids__contains'] = self.id
        ids = set(d['id'] for d in self.gencloud.api.dataid.get(**query)['objects'])
        return [d for d in data if d.id in ids]

    def find(self, filter_str):
        """Filter Data object annotation."""
        raise NotImplementedError()

    def __str__(self):
        return self.name or 'n/a'

    def __repr__(self):
        return u"GenProject: {} - {}".format(self.id, self.name)