"""Functionality to build a list of routes."""
import hashlib

from o2g.osm.models import Route
from o2g.gtfs.gtfs_misc import map_osm_route_type_to_gtfs

def build_routes(relations):
    for rel in relations.values():
        route = build_route(rel)
        if route:
            yield route

def build_route(relation):
    """Extract information of one route."""
    if relation.tags.get('type') != 'route':
        # Build route only for relations of type `route`
        return

    short_name = create_route_short_name(relation)
    color = relation.tags.get('color')
    return\
        Route(relation.id,
              short_name,
              create_route_long_name(relation, short_name),
              map_osm_route_type_to_gtfs(relation.tags.get('route')),
              'https://www.openstreetmap.org/relation/{}'.format(relation.id),
              color.strip('#') if color else '',
              get_agency_id(relation))

def create_route_short_name(relation):
    """Create a meaningful route short name."""
    return relation.tags.get('ref') or ''

def create_route_long_name(relation, short_name):
    """Create a meaningful route name."""
    if relation.tags.get('from') and relation.tags.get('to'):
        return "{0}-to-{1}".format(relation.tags.get('from'),
                                   relation.tags.get('to'))
    name = relation.tags.get('name') or\
        relation.tags.get('alt_name') or\
        "OSM Route No. {}".format(relation.id)

    # Drop route_short_name from this one if it contains it
    if short_name and name.startswith(short_name):
        # Drop it
        return name[len(short_name):]
    return name

def get_agency_id(relation):
    """Construct an id for agency using its tags."""
    op = relation.tags.get('operator')
    if op:
        return int(hashlib.sha256(op.encode('utf-8')).hexdigest(), 16) % 10**8
    return -1


import logging

from o2g.osm.handlers import RelationHandler, NodeHandler, WayHandler
from o2g.osm.builders import build_routes, build_stops, build_agencies,\
    build_shapes

class TransitDataExporter(object):
    def __init__(self, filename):
        self.filename = filename
        self.rh = None
        self.nh = None
        self.wh = None

    @property
    def agencies(self):
        return build_agencies(self.rh.relations, self.nh.nodes, self.wh.ways)

    @property
    def routes(self):
        return build_routes(self.rh.relations)

    @property
    def stops(self):
        return build_stops(self.rh.relations, self.nh.nodes)

    @property
    def shapes(self):
        return build_shapes(self.rh.relations, self.nh.nodes, self.wh.ways)

    def process(self):
        """Process the files and collect necessary data."""

        # Extract relations
        self.rh = RelationHandler()
        self.rh.apply_file(self.filename)

        logging.debug('Found %d public transport relations.', len(self.rh.relations))

        # Collect ids of interest
        node_ids, stop_node_ids, way_ids, reverse_map = self.__collect_ids()

        # Extract nodes
        self.nh = NodeHandler(node_ids)
        self.nh.apply_file(self.filename, locations=True)

        count = 0
        for idx, missing_node_id in enumerate(self.nh.missing_node_ids):
            count += 1
            logging.warning(
                '[no data] missing stop node. rel: https://osm.org/relation/%s node: https://osm.org/node/%s.',
                reverse_map[missing_node_id], missing_node_id)

        if count:
            logging.warning(
                '%d nodes that appear in relations are missing.',
                count)
        else:
            logging.debug('Lucky you! All relation member nodes were found.')

        # Extract ways
        self.wh = WayHandler(way_ids)
        self.wh.apply_file(self.filename, locations=True)

    def __collect_ids(self):
        node_ids = set()
        stop_node_ids = set()
        way_ids = set()
        reverse_map = {}

        for rel in self.rh.relations.values():
            for mtype, ref, role in rel.member_info:
                reverse_map[ref] = rel.id

                if mtype in ['n', 'node']:
                    node_ids.add(ref)
                    if role in ['stop', 'halt', 'platform']:
                        stop_node_ids.add(ref)

                elif mtype in ['w', 'way']:
                    way_ids.add(ref)

                elif mtype in ['r', 'relation']:
                    logging.warning(
                        '[Rel: %s]: super-relations are not supported yet. ref: %s',
                        rel.id, ref)
                else:
                    logging.warning(
                        '[Rel: %s]: unknown member type %s, ref: %s',
                        rel.id, mtype, ref)

        return node_ids, stop_node_ids, way_ids, reverse_map


import osmium as o

from o2g.osm.models import Relation

class RelationHandler(o.SimpleHandler):
    def __init__(self):
        super(RelationHandler, self).__init__()
        self.relations = {}
        self.versions = {}

    @property
    def transit_route_types(self):
        """A list of default OSM route types.

        Mainly train and bus types.
        """
        return ['bus', 'ex-bus',        # Bus
                'tram', 'light_rail',   # Tram
                'subway',               # Subway
                'rail', 'railway']      # Rail

    def relation(self, rel):
        """Process each relation."""
        rel_type = rel.tags.get('type')
        if any([rel.deleted,
                not rel.visible,
                not self.is_new_version(rel),
                rel_type not in ['route', 'public_transport']]):
            return

        route_tag = rel.tags.get('route')
        if rel_type == 'route' and route_tag not in self.transit_route_types:
            return

        public_transport = rel.tags.get('public_transport')
        if rel_type == 'public_transport' and public_transport != 'stop_area':
            return

        self.relations[rel.id] = \
            Relation(rel.id, {
                'type': rel_type,
                'public_transport': public_transport,
                'route': route_tag,
                'operator': rel.tags.get('operator'),
                'color': rel.tags.get('color'),
                'ref': rel.tags.get('ref'),
                'from': rel.tags.get('from'),
                'to': rel.tags.get('to'),
                'name': rel.tags.get('name'),
                'alt_name': rel.tags.get('alt_name'),
                'url': rel.tags.get('url'),
                'contact_website': rel.tags.get('contact:website')},
                [(member.type, member.ref, member.role) for member in rel.members])

        self.versions[rel.id] = rel.version

    def is_new_version(self, relation):
        return relation.id not in self.versions or \
            relation.version > self.versions[relation.id]


"""Tools to generate dummy GTFS feeds."""
import datetime
from math import radians, cos, sin, asin, sqrt
from collections import namedtuple, defaultdict

from o2g.osm.models import Agency

# Represents dummy GTFS data
DummyData = namedtuple('DummyData',
                       ['calendar', 'stop_times', 'trips', 'frequencies'])

def create_dummy_data(routes, stops):
    """Create `calendar`, `stop_times`, `trips` and `shapes`.

    :return: DummyData namedtuple
    """
    # Build stops per route auxiliary map
    stops_per_route = defaultdict(lambda: [])
    stops_map = {}
    for s in stops:
        if not s.route_id:
            continue
        stops_per_route[s.route_id].append(s)
        stops_map[s.stop_id] = s

    calendar = _create_dummy_calendar()

    trips = \
        _create_dummy_trips(
            routes,
            stops_per_route,
            calendar)

    stop_times = _create_dummy_stoptimes(trips, stops_per_route)
    frequencies = _create_dummy_frequencies(trips)

    return DummyData(calendar, stop_times, trips, frequencies)

def patch_agencies(agencies):
    """Fill the fields that are necessary for passing transitfeed checks."""
    # First return the unknown agency entry
    yield Agency(-1, 'http://hiposfer.com', 'Unknown agency', 'Europe/Berlin')

    # Then return the rest.
    for agency_id, agency_url, agency_name, agency_timezone in agencies:
        if not agency_url:
            agency_url = 'http://hiposfer.com'
        if not agency_timezone:
            # Set everything to one time zone to get rid of transitfeeds error.
            agency_timezone = 'Europe/Berlin'
        yield Agency(agency_id, agency_url, agency_name, agency_timezone)

def _create_dummy_calendar():
    return [{'service_id': 'WE', 'monday': 0, 'tuesday': 0, 'wednesday': 0,
             'thursday': 0,
             'friday': 0, 'saturday': 1, 'sunday': 1, 'start_date': 20170101,
             'end_date': 20190101},
            {'service_id': 'WD', 'monday': 1, 'tuesday': 1, 'wednesday': 1,
             'thursday': 1,
             'friday': 1, 'saturday': 0, 'sunday': 0, 'start_date': 20170101,
             'end_date': 20190101}]

def _create_dummy_trips(routes, stops_per_route, calendar):
    trips = []

    for route in routes:
        route_id = route.route_id

        if len(stops_per_route.get(route_id, [])) < 2:
            continue

        for cal_idx, cal in enumerate(calendar):
            # For the sake of simplicity, we assume 3 trips per service day.
            for idx in range(3):

                trip_id = \
                    '{sequence}{cal_idx}.{route_id}'.format(
                        sequence=idx + 1,
                        cal_idx=cal_idx,
                        route_id=route_id)

                trip = {'route_id': route_id,
                        'service_id': cal['service_id'],
                        'trip_id': trip_id,
                        'trip_headsign':
                            '[Dummy]{}'.format(route.route_long_name),
                        # Use route_id, i.e. relation_id as shape_id
                        'shape_id': route_id,
                        # Used for generating stop times.
                        'sequence': idx}
                trips.append(trip)

    return trips

def _create_dummy_stoptimes(trips, stops_per_route):
    stoptimes = []
    for trip in trips:
        # One service every 20 minutes from the base station
        first_service_time = \
            datetime.datetime(2017, 1, 1, 5, 0, 0) + \
            datetime.timedelta(minutes=20) * trip['sequence']
        stoptimes.extend(
            _create_dummy_trip_stoptimes(
                trip['trip_id'],
                stops_per_route.get(trip['route_id'], []),
                first_service_time))

    return stoptimes

def _create_dummy_trip_stoptimes(trip_id, stops, first_service_time):
    """Create station stop times for each trip."""
    waiting = datetime.timedelta(seconds=30)
    arrival = first_service_time
    last_departure = first_service_time
    last_departure_hour = (arrival + waiting).hour
    last_stop = None

    departure_hour = None
    arrival_hour = None
    for stop_sequence, stop in enumerate(stops):

        # Avoid time travels
        arrival = last_departure + get_time_from_last_stop(last_stop, stop)
        departure = arrival + waiting

        # Cover the case when the arrival time falls into the next day
        if arrival.hour < last_departure_hour:
            diff = last_departure_hour
            arrival_hour = arrival.hour + diff
            departure_hour = departure.hour + diff
            last_departure_hour = departure.hour + diff
        else:
            arrival_hour = arrival.hour
            departure_hour = departure.hour
            last_departure_hour = departure.hour

        # Cover the case when adding waiting time to the arrival time
        # falls into the next day
        if departure.hour < arrival.hour:
            diff = last_departure_hour
            departure_hour = departure.hour + diff
            last_departure_hour = departure.hour + diff

        yield {'trip_id': trip_id,
               'arrival_time': '{:02}:{}'.format(
                   arrival_hour,
                   arrival.strftime('%M:%S')),
               'departure_time': '{:02}:{}'.format(
                   departure_hour,
                   departure.strftime('%M:%S')),
               'stop_id': stop.stop_id,
               'stop_sequence': stop_sequence}

        last_stop = stop
        last_departure = departure

def get_time_from_last_stop(src_stop, dst_stop):
    if not src_stop:
        return datetime.timedelta()

    # Average public transport speed
    average_speed_kmh = 20

    distance_km = \
        haversine(src_stop.stop_lon,
                  src_stop.stop_lat,
                  dst_stop.stop_lon,
                  dst_stop.stop_lat)
    return \
        datetime.timedelta(
            hours=distance_km/average_speed_kmh)

# https://stackoverflow.com/a/4913653
def haversine(lon1, lat1, lon2, lat2):
    """
    Calculate the great circle distance between two
    points on the earth (specified in decimal degrees)
    """
    # convert decimal degrees to radians
    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])

    # haversine formula
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
    c = 2 * asin(sqrt(a))
    r = 6371  # Radius of earth in kilometers. Use 3956 for miles
    return c * r

def _create_dummy_frequencies(trips):
    """Create station stop frequencies."""
    for trip in trips:
        yield {'trip_id': trip['trip_id'],
               'start_time': '04:30:00',
               'end_time': '08:30:00',
               'headway_secs': '1200'}
        yield {'trip_id': trip['trip_id'],
               'start_time': '08:30:00',
               'end_time': '20:30:00',
               'headway_secs': '1800'}
        yield {'trip_id': trip['trip_id'],
               'start_time': '20:30:00',
               'end_time': '25:30:00',
               'headway_secs': '2800'}


"""Utilities for writing GTFS feeds."""
import os
import io
import csv
import shutil
import zipfile
from collections import OrderedDict

class GTFSWriter(object):
    """GTFS feed writer."""
    def __init__(self):
        self._buffers = {}
        self._csv_writers = {}
        self._files = {}

        for name, csv_headers in self.headers.items():
            self._buffers[name] = io.StringIO()
            self._csv_writers[name] =\
                csv.writer(self._buffers[name], lineterminator='\n')
            self._csv_writers[name].writerow(csv_headers)

    def _add_records(self, name, records, sortkey=None):
        if sortkey:
            records = sorted(records, key=lambda x: x[sortkey])
        for rec in records:
            if not isinstance(rec, dict):
                record = rec._asdict()
            else:
                record = rec
            csv_record = OrderedDict.fromkeys(self.headers[name])
            # Update csv with keys present in headers. Skip anything else.
            csv_record.update(
                {k: v for k, v in record.items() if k in self.headers[name]})
            self._csv_writers[name].writerow([v for v in csv_record.values()])

    @property
    def headers(self):
        """Map of filename to headers for every GTFS file.

        Only headers present in this map will be considered."""
        return {
            'agency': ['agency_id', 'agency_name', 'agency_url',
                       'agency_timezone', 'agency_lang', 'agency_phone',
                       'agency_fare_url', 'agency_email'],

            'stops': ['stop_id', 'stop_code', 'stop_name', 'stop_desc',
                      'stop_lat', 'stop_lon', 'zone_id', 'stop_url',
                      'location_type', 'parent_station', 'stop_timezone',
                      'wheelchair_boarding'],

            'routes': ['route_id', 'agency_id', 'route_short_name',
                       'route_long_name', 'route_desc', 'route_type',
                       'route_url', 'route_color', 'route_text_color'],

            'trips': ['route_id', 'service_id', 'trip_id', 'trip_headsign',
                      'shape_id'],

            'calendar': ['service_id', 'monday', 'tuesday', 'wednesday',
                         'thursday',
                         'friday', 'saturday', 'sunday', 'start_date',
                         'end_date'],

            'stop_times': ['trip_id', 'arrival_time', 'departure_time',
                           'stop_id',
                           'stop_sequence'],

            'shapes': ['shape_id', 'shape_pt_lat', 'shape_pt_lon',
                       'shape_pt_sequence'],

            'frequencies': ['trip_id', 'start_time', 'end_time',
                            'headway_secs'],

            'feedinfo': ['feed_publisher_name', 'feed_publisher_url',
                         'feed_lang', 'feed_start_date', 'feed_end_date',
                         'feed_version', 'feed_contact_email',
                         'feed_contact_url']}

    def add_agencies(self, agencies):
        self._add_records('agency', agencies)

    def add_stops(self, stops):
        self._add_records('stops', stops)

    def add_routes(self, routes):
        self._add_records('routes', routes, sortkey=0)

    def add_calendar(self, weekly_schedules):
        self._add_records('calendar', weekly_schedules)

    def add_trips(self, trips):
        self._add_records('trips', trips)

    def add_stop_times(self, stop_times):
        self._add_records('stop_times', stop_times)

    def add_shapes(self, shapes):
        self._add_records('shapes', shapes)

    def add_frequencies(self, frequencies):
        self._add_records('frequencies', frequencies)

    def add_feedinfo(self, info):
        self._add_records('feedinfo', [info])

    def add_file(self, name, path):
        self._files[name] = path

    def write_zipped(self, filepath):
        """Write the GTFS feed in the given file."""
        with zipfile.ZipFile(filepath, mode='w', compression=zipfile.ZIP_DEFLATED) as zfile:
            for name, buffer in self._buffers.items():
                encoded_values = io.BytesIO(buffer.getvalue().encode('utf-8'))
                zfile.writestr('{}.txt'.format(name),
                               encoded_values.getbuffer())
            for name, path in self._files.items():
                zfile.write(path, arcname=name)

    def write_unzipped(self, destination):
        """Write GTFS text files in the given path."""
        for name, buffer in self._buffers.items():
            with open(os.path.join(destination,
                                   '{}.txt'.format(name)),
                      'w', encoding='utf-8') as file:
                file.write(buffer.getvalue())
        for name, path in self._files.items():
            shutil.copy(path, os.path.join(destination, name))


"""Functionality to build a list of agencies."""
import hashlib

# from timezonefinder import TimezoneFinder

from o2g.osm.models import Agency

# TZ_FINDER = TimezoneFinder()

def build_agencies(relations, nodes, ways):
    extracted = []
    for rel in relations.values():
        agency = build_agency(rel, nodes)
        if agency and agency.agency_id not in extracted:
            extracted.append(agency.agency_id)
            yield agency#._replace(agency_timezone=_guess_timezone(rel, nodes, ways))

def build_agency(relation, nodes):
    """Extract agency information."""
    # TODO: find out the operator for routes without operator tag.
    # See: http://wiki.openstreetmap.org/wiki/Key:operator
    # Quote from the above link:
    #
    #    If the vast majority of a certain object in an area is operated by a certain
    #    organization and only very few by others then it may be sufficient to only tag the
    #    exceptions. For example, when nearly all roads in an area are managed by a local
    #    authority then it would be sufficient to only tag those that are not with an operator
    #    tag.
    op = relation.tags.get('operator')
    agency_url = relation.tags.get('url') or relation.tags.get('contact_website')

    if not op:
        return

    agency_id = int(hashlib.sha256(op.encode('utf8')).hexdigest(), 16) % 10**8

    return Agency(agency_id, agency_url, op, '')

# def _guess_timezone(relation, nodes, ways):
#     """Guess timezone of the relation by looking at it's nodes."""
#     lon, lat = _get_first_coordinate(relation, nodes, ways)
#     timezone = TZ_FINDER.timezone_at(lng=lon, lat=lat)
#     if not timezone:
#         logging.debug('No timezone found for (%s, %s)', lon, lat)
#     return timezone

# def _get_first_coordinate(relation, nodes, ways):
#     for member_id, member_role in relation.member_info:
#         if member_id in nodes:
#             return nodes[member_id].lon, nodes[member_id].lat
#         elif member_id in ways:
#             return _get_first_way_coordinate(member_id, ways)
#     logging.debug('No node found for relation %s', relation.id)
#     return 0, 0

# def _get_first_way_coordinate(way_id, ways):
#     # Pick the first node
#     return ways[way_id].points[0]


"""Functionality to build a list of stops."""
import logging

from o2g.osm.models import Stop

def build_stops(relations, nodes):
    visited_stops_ids = set()
    stop_to_station_map = {}

    # First gsocess all stop_areas
    for rel in relations.values():
        if rel.tags.get('public_transport') != 'stop_area':
            continue

        station = build_parent_stop(rel, nodes)
        if not station:
            continue

        for _, member_id, _ in rel.member_info:
            stop_to_station_map[member_id] = station.stop_id

        visited_stops_ids.add(station.stop_id)
        yield station

    for rel in relations.values():
        if rel.tags.get('public_transport') == 'stop_area':
            continue

        for stop in extract_stops(rel, nodes, visited_stops_ids, stop_to_station_map):
            if stop:
                yield stop

def build_parent_stop(relation, nodes):
    # One stop per stop_area is necessary.
    station_node = None
    some_node = None
    for member_type, member_id, member_role in relation.member_info:
        if member_id not in nodes:
            continue
        some_node = nodes[member_id]
        if not some_node:
            raise Exception()
        if some_node.tags.get('public_transport') == 'station' or \
            some_node.tags.get('amenity') == 'bus_station':
            station_node = some_node

    if not station_node and not some_node:
        logging.debug('No parent node found: https://osm.org/relation/%s' % relation.id)
        return

    if not station_node:
        logging.warning(
            'stop_area without station: https://osm.org/relation/%s.'\
            ' Using node https://osm.org/node/%s instead.' % (relation.id, some_node.id))
        station_node = some_node

    return Stop(
        station_node.id,
        station_node.tags.get('name') or "Unnamed stop_area.",
        station_node.lon,
        station_node.lat,
        None,
        _map_wheelchair(station_node.tags.get('wheelchair')),
        1,  # A station in GTFS terms
        '')  # Blank values since stations can't contain other stations.

def extract_stops(relation, nodes, visited_stop_ids, stop_to_station_map):
    """Extract stops in a relation."""
    # member_role: stop, halt, platform, terminal, etc.
    for member_type, member_id, member_role in relation.member_info:

        if member_id not in visited_stop_ids and \
            member_id in nodes and\
                member_role in ('stop', 'halt'):

            location_type = ''

            visited_stop_ids.add(member_id)
            yield Stop(
                member_id,
                nodes[member_id].tags.get('name') or
                "Unnamed {} stop.".format(relation.tags.get('route')),
                nodes[member_id].lon if member_id in nodes else '',
                nodes[member_id].lat if member_id in nodes else '',
                relation.id,
                _map_wheelchair(nodes[member_id].tags.get('wheelchair')),
                location_type,
                stop_to_station_map.get(member_id, ''))

def _map_wheelchair(osm_value):
    if not osm_value:
        return 0
    elif osm_value == 'limited':
        return 1
    elif osm_value == 'yes':
        return 1
    elif osm_value == 'no':
        return 2
    else:
        logging.warning('Unknown OSM wheelchair value %s', osm_value)
        return 0


"""Functionality to build a list of shapes."""
import logging

from o2g.osm.models import Shape

def build_shapes(relations, nodes, ways):
    for rel in relations.values():
        for record in build_shape(rel, nodes, ways):
            if record:
                yield record

def build_shape(relation, nodes, ways):
    """Extract shape of one route."""
    sequence_index = 0

    for member_type, member_id, member_role in relation.member_info:

        if member_id in nodes:
            yield Shape(
                relation.id,
                nodes[member_id].lat,
                nodes[member_id].lon,
                sequence_index)

            sequence_index += 1

        # Do we need to consider ways too? It dramatically increases the number of shapes.
        elif member_id in ways:
            continue
        #     for point in ways[member_id].points:
        #         shape = Shape(
        #             relation.id,
        #             point.lat,
        #             point.lon,
        #             sequence_index)
        #         sequence_index += 1

        else:
            # Ignore excessive logging for now.
            pass
            # if member_type == 'n':
            #     member_osm_type = 'node'
            # elif member_type == 'w':
            #     member_osm_type = 'way'
            # else:
            #     member_osm_type = member_type
            # logging.warning('[no data] https://osm.org/relation/%s missing https://osm.org/%s/%s.',
            #                 relation.id, member_osm_type, member_id)


# Copyright (c) 2013 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from u2flib_host.constants import APDU_OK, INS_GET_VERSION
from u2flib_host.yubicommon.compat import int2byte
from u2flib_host import exc
import struct

class U2FDevice(object):

    """
    A handle to a U2F device.
    device.open() needs to be called prior to using the device, and
    device.close() should be called when the device is no longer needed, to
    ensure that any held resources are released.
    As an aternative, the 'with' statement can be used:

        with device as dev:
            dev.send_apdu(...)
    """

    def __enter__(self):
        self.open()
        return self

    def __exit__(self, type, value, traceback):
        self.close()

    def __del__(self):
        self.close()

    def open(self):
        """
        Opens the device for use.
        """
        pass

    def close(self):
        """
        Closes the device, making it available for use by others.
        """
        pass

    def get_supported_versions(self):
        """
        Gets a list of supported U2F versions from the device.
        """
        if not hasattr(self, '_versions'):
            try:
                self._versions = [self.send_apdu(INS_GET_VERSION).decode()]
            except exc.APDUError as e:
                # v0 didn't support the instruction.
                self._versions = ['v0'] if e.code == 0x6d00 else []

        return self._versions

    def _do_send_apdu(self, apdu_data):
        """
        Sends an APDU to the device, and returns the response.
        """
        # Subclasses should implement this.
        raise NotImplementedError('_do_send_apdu not implemented!')

    def send_apdu(self, ins, p1=0, p2=0, data=b''):
        """
        Sends an APDU to the device, and waits for a response.
        """
        if data is None:
            data = b''
        elif isinstance(data, int):
            data = int2byte(data)

        size = len(data)
        l0 = size >> 16 & 0xff
        l1 = size >> 8 & 0xff
        l2 = size & 0xff
        apdu_data = struct.pack('B B B B B B B %is B B' % size,
                                0, ins, p1, p2, l0, l1, l2, data, 0x00, 0x00)
        try:
            resp = self._do_send_apdu(apdu_data)
        except Exception as e:
            # TODO Use six.reraise if/when Six becomes an agreed dependency.
            raise exc.DeviceError(e)
        status = struct.unpack('>H', resp[-2:])[0]
        data = resp[:-2]
        if status != APDU_OK:
            raise exc.APDUError(status)
        return data


# Copyright (c) 2013 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from __future__ import print_function

from u2flib_host import u2f, exc, __version__
from u2flib_host.constants import APDU_USE_NOT_SATISFIED
from u2flib_host.utils import u2str
from u2flib_host.yubicommon.compat import text_type

import time
import json
import argparse
import sys

def authenticate(devices, params, facet, check_only):
    """
    Interactively authenticates a AuthenticateRequest using an attached U2F
    device.
    """
    for device in devices[:]:
        try:
            device.open()
        except:
            devices.remove(device)

    try:
        prompted = False
        while devices:
            removed = []
            for device in devices:
                try:
                    return u2f.authenticate(device, params, facet, check_only)
                except exc.APDUError as e:
                    if e.code == APDU_USE_NOT_SATISFIED:
                        if check_only:
                            sys.stderr.write('\nCorrect U2F device present!\n')
                            sys.exit(0)
                        if not prompted:
                            sys.stderr.write('\nTouch the flashing U2F device '
                                             'to authenticate...\n')
                            prompted = True
                    else:
                        removed.append(device)
                except exc.DeviceError:
                    removed.append(device)
            devices = [d for d in devices if d not in removed]
            for d in removed:
                d.close()
            time.sleep(0.25)
    finally:
        for device in devices:
            device.close()
    sys.stderr.write('\nThe required U2F device is not present!\n')
    sys.exit(1)

def parse_args():
    parser = argparse.ArgumentParser(
        description="Authenticaties an AuthenticateRequest.\n"
        "Takes a JSON formatted AuthenticateRequest object on stdin, and "
        "returns the resulting AuthenticateResponse on stdout.",
        add_help=True
    )
    parser.add_argument('-v', '--version', action='version',
                        version='%(prog)s ' + __version__)
    parser.add_argument('facet', help='the facet for the challenge')
    parser.add_argument('-c', '--check-only', action="store_true", help='Check if '
                        'the key handle is correct only, don\'t sign.')
    parser.add_argument('-i', '--infile', help='specify a file to read '
                        'AuthenticateRequest from, instead of stdin')
    parser.add_argument('-o', '--outfile', help='specify a file to write '
                        'the AuthenticateResponse to, instead of stdout')
    parser.add_argument('-s', '--soft', help='Specify a soft U2F token file to use')
    return parser.parse_args()

def main():
    args = parse_args()

    facet = text_type(args.facet)
    if args.infile:
        with open(args.infile, 'r') as f:
            data = f.read()
    else:
        if sys.stdin.isatty():
            sys.stderr.write('Enter AuthenticateRequest JSON data...\n')
        data = sys.stdin.read()

    params = json.loads(data)
    if args.soft:
        from u2flib_host.soft import SoftU2FDevice
        devices = [SoftU2FDevice(args.soft)]
    else:
        devices = u2f.list_devices()
    result = authenticate(devices, params, facet, args.check_only)

    if args.outfile:
        with open(args.outfile, 'w') as f:
            json.dump(result, f)
        sys.stderr.write('Output written to %s\n' % args.outfile)
    else:
        sys.stderr.write('\n---Result---\n')
        print(json.dumps(result))

if __name__ == '__main__':
    main()


# Copyright (c) 2013 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from u2flib_host.constants import INS_ENROLL, INS_SIGN
from u2flib_host.utils import websafe_decode, websafe_encode
from u2flib_host.appid import verify_facet
from u2flib_host.yubicommon.compat import string_types, int2byte

from hashlib import sha256
import json

VERSION = 'U2F_V2'

def register(device, data, facet):
    """
    Register a U2F device

    data = {
        "version": "U2F_V2",
        "challenge": string, //b64 encoded challenge
        "appId": string, //app_id
    }

    """

    if isinstance(data, string_types):
        data = json.loads(data)

    if data['version'] != VERSION:
        raise ValueError('Unsupported U2F version: %s' % data['version'])

    app_id = data.get('appId', facet)
    verify_facet(app_id, facet)
    app_param = sha256(app_id.encode('utf8')).digest()

    client_data = {
        'typ': 'navigator.id.finishEnrollment',
        'challenge': data['challenge'],
        'origin': facet
    }
    client_data = json.dumps(client_data)
    client_param = sha256(client_data.encode('utf8')).digest()

    request = client_param + app_param

    p1 = 0x03
    p2 = 0
    response = device.send_apdu(INS_ENROLL, p1, p2, request)

    return {
        'registrationData': websafe_encode(response),
        'clientData': websafe_encode(client_data)
    }

def authenticate(device, data, facet, check_only=False):
    """
    Signs an authentication challenge

    data = {
        'version': "U2F_V2",
        'challenge': websafe_encode(self.challenge),
        'appId': self.binding.app_id,
        'keyHandle': websafe_encode(self.binding.key_handle)
    }

    """

    if isinstance(data, string_types):
        data = json.loads(data)

    if data['version'] != VERSION:
        raise ValueError('Unsupported U2F version: %s' % data['version'])

    app_id = data.get('appId', facet)
    verify_facet(app_id, facet)
    app_param = sha256(app_id.encode('utf8')).digest()

    key_handle = websafe_decode(data['keyHandle'])

    # Client data
    client_data = {
        'typ': 'navigator.id.getAssertion',
        'challenge': data['challenge'],
        'origin': facet
    }
    client_data = json.dumps(client_data)
    client_param = sha256(client_data.encode('utf8')).digest()

    request = client_param + app_param + int2byte(
        len(key_handle)) + key_handle

    p1 = 0x07 if check_only else 0x03
    p2 = 0
    response = device.send_apdu(INS_SIGN, p1, p2, request)

    return {
        'clientData': websafe_encode(client_data),
        'signatureData': websafe_encode(response),
        'keyHandle': data['keyHandle']
    }


# Copyright (c) 2013 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from __future__ import print_function

from u2flib_host import u2f, exc, __version__
from u2flib_host.constants import APDU_USE_NOT_SATISFIED
from u2flib_host.utils import u2str
from u2flib_host.yubicommon.compat import text_type

import time
import json
import argparse
import sys

def register(devices, params, facet):
    """
    Interactively registers a single U2F device, given the RegistrationRequest.
    """
    for device in devices[:]:
        try:
            device.open()
        except:
            devices.remove(device)

    sys.stderr.write('\nTouch the U2F device you wish to register...\n')
    try:
        while devices:
            removed = []
            for device in devices:
                try:
                    return u2f.register(device, params, facet)
                except exc.APDUError as e:
                    if e.code == APDU_USE_NOT_SATISFIED:
                        pass
                    else:
                        removed.append(device)
                except exc.DeviceError:
                    removed.append(device)
            devices = [d for d in devices if d not in removed]
            for d in removed:
                d.close()
            time.sleep(0.25)
    finally:
        for device in devices:
            device.close()
    sys.stderr.write('\nUnable to register with any U2F device.\n')
    sys.exit(1)

def parse_args():
    parser = argparse.ArgumentParser(
        description="Registers a U2F device.\n"
        "Takes a JSON formatted RegisterRequest object on stdin, and returns "
        "the resulting RegistrationResponse on stdout.",
        add_help=True
    )
    parser.add_argument('-v', '--version', action='version',
                        version='%(prog)s ' + __version__)
    parser.add_argument('facet', help='the facet for registration')
    parser.add_argument('-i', '--infile', help='specify a file to read '
                        'RegistrationRequest from, instead of stdin')
    parser.add_argument('-o', '--outfile', help='specify a file to write '
                        'the RegistrationResponse to, instead of stdout')
    parser.add_argument('-s', '--soft', help='Specify a soft U2F device file '
                        'to use')
    return parser.parse_args()

def main():
    args = parse_args()

    facet = text_type(args.facet)
    if args.infile:
        with open(args.infile, 'r') as f:
            data = f.read()
    else:
        if sys.stdin.isatty():
            sys.stderr.write('Enter RegistrationRequest JSON data...\n')
        data = sys.stdin.read()
    params = json.loads(data)

    if args.soft:
        from u2flib_host.soft import SoftU2FDevice
        devices = [SoftU2FDevice(args.soft)]
    else:
        devices = u2f.list_devices()
    result = register(devices, params, facet)

    if args.outfile:
        with open(args.outfile, 'w') as f:
            json.dump(result, f)
        sys.stderr.write('Output written to %s\n' % args.outfile)
    else:
        sys.stderr.write('\n---Result---\n')
        print(json.dumps(result))

if __name__ == '__main__':
    main()


# Copyright (c) 2013 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from u2flib_host.yubicommon.compat import text_type

from base64 import urlsafe_b64decode, urlsafe_b64encode
from hashlib import sha256

__all__ = [
    'u2str',
    'websafe_encode',
    'websafe_decode'
]

def u2str(data):
    """Recursively converts unicode objects to UTF-8 encoded byte strings."""
    if isinstance(data, dict):
        return {u2str(k): u2str(v) for k, v in data.items()}
    elif isinstance(data, list):
        return [u2str(x) for x in data]
    elif isinstance(data, text_type):
        return data.encode('utf-8')
    else:
        return data

def websafe_decode(data):
    if isinstance(data, text_type):
        data = data.encode('ascii')
    data += b'=' * (-len(data) % 4)
    return urlsafe_b64decode(data)

def websafe_encode(data):
    if isinstance(data, text_type):
        data = data.encode('ascii')
    return urlsafe_b64encode(data).replace(b'=', b'').decode('ascii')


#!/usr/bin/env python
#
# Copyright (c) 2011-2013, Shopkick Inc.
# All rights reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ---
# Author: John Egan <jwegan@gmail.com>

import functools
import inspect

import flawless.client.client
from flawless.lib.utils import im_func, im_self

def wrap_function(func=None, error_threshold=None, reraise_exception=True, save_current_stack_trace=True):
    ''' Wraps a function with reporting to errors backend '''
    # This if/else allows wrap_function to behave like a normal decorator when
    # used like:
    #         @wrap_function
    #         def some_func():
    #
    # However, it also allows wrap_function to also be passed keyword arguments
    # like the following:
    #         @wrap_function(error_threshold=3, reraise_exception=False)
    #         def some_func():
    if func:
        return flawless.client.client._wrap_function_with_error_decorator(
            func=func,
            error_threshold=error_threshold,
            reraise_exception=reraise_exception,
            save_current_stack_trace=save_current_stack_trace)
    else:
        return functools.partial(flawless.client.client._wrap_function_with_error_decorator,
                                 error_threshold=error_threshold,
                                 reraise_exception=reraise_exception,
                                 save_current_stack_trace=save_current_stack_trace)

def wrap_class(cls, error_threshold=None):
    ''' Wraps a class with reporting to errors backend by decorating each function of the class.
            Decorators are injected under the classmethod decorator if they exist.
    '''
    methods = inspect.getmembers(cls, inspect.ismethod) + inspect.getmembers(cls, inspect.isfunction)
    for method_name, method in methods:
        wrapped_method = flawless.client.client._wrap_function_with_error_decorator(
            method if not im_self(method) else im_func(method),
            save_current_stack_trace=False,
            error_threshold=error_threshold,
        )
        if im_self(method):
            wrapped_method = classmethod(wrapped_method)
        setattr(cls, method_name, wrapped_method)
    return cls

class WrapClassMetaclass(type):
    '''Specify FlawlessMetaClass as the metaclass for your class if you want Flawless to wrap a base class
    that other classes inherit from. Only methods defined in the base class that uses this metaclass will be wrapped.
    (i.e. new methods defined only in the subclass will not be wrapped)'''
    def __init__(cls, name, bases, dct):
        cls = wrap_class(cls)
        return super(WrapClassMetaclass, cls).__init__(name, bases, dct)


#!/usr/bin/env python
#
# Copyright (c) 2011-2013, Shopkick Inc.
# All rights reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ---
# Author: John Egan <jwegan@gmail.com>

from __future__ import absolute_import
import __main__

import cgi
try:
    from email.mime.text import MIMEText
except ImportError:
    from email.MIMEText import MIMEText
import collections
import datetime
from functools import reduce
import logging
import os
import os.path
import re
import smtplib
import subprocess
import sys
import threading
import time
import traceback
import urllib
try:
    import urllib.parse
    urlencode = urllib.parse.urlencode
except ImportError:
    import urllib
    urlencode = urllib.urlencode

from future.utils import iteritems

import flawless.lib.config
from flawless.lib.data_structures.lru_cache import ExpiringLRUCache
from flawless.lib.data_structures import prefix_tree
from flawless.lib.storage import DiskStorage
from flawless.lib.version_control.repo import get_repository
import flawless.server.api.ttypes as api_ttypes

try:
    import pprint
except:
    pprint = None

log = logging.getLogger(__name__)
config = flawless.lib.config.get()

############################## EXTEND THRIFT TTYPES ##############################

def code_identifier_equality(self, other):
    # We allow people to whitelist entire files, or functions by setting function_name to None or
    # code-fragment to None
    if self.filename != other.filename:
        return False
    if self.function_name and other.function_name and self.function_name != other.function_name:
        return False

    # Compress whitespace characters to make the comparissions more forgiving
    self_fragment = None if not self.code_fragment else re.sub("\s+", " ", self.code_fragment.strip())
    other_fragment = None if not other.code_fragment else re.sub("\s+", " ", other.code_fragment.strip())
    if (self_fragment and other_fragment and self_fragment not in other_fragment and
            other_fragment not in self_fragment):
        return False

    return True

api_ttypes.CodeIdentifier.__eq__ = code_identifier_equality
api_ttypes.KnownError.__eq__ = code_identifier_equality
api_ttypes.ErrorKey.__hash__ = lambda self: reduce(lambda x, y: x ^ hash(y), iteritems(self.__dict__), 1)

############################## BASE SERVICE CLASS ##############################

class FlawlessServiceBaseClass(object):
    """Contains numerous shared helper functions for the ThriftService & WebService classes"""

    def __init__(self,
                 storage_factory=DiskStorage,
                 thread_cls=threading.Thread,
                 open_process_func=subprocess.Popen,
                 smtp_client_cls=smtplib.SMTP,
                 time_func=time.time):
        self.__dict__.update({k: v for k, v in iteritems(locals()) if k != 'self'})
        self.extract_base_path_patterns = [re.compile('^.*/%s/?(.*)$' % d) for d in
                                           config.report_runtime_package_directory_names]
        self.raise_exception_pattern = re.compile('^\s*raise[ \n]')
        self.only_blame_patterns = [re.compile(p) for p in config.only_blame_filepaths_matching]
        self._read_whitelist_configs()

    ############################## Parse Config Files ##############################

    def _read_whitelist_configs(self):
        config_storage = self.storage_factory(partition=None)
        config_storage.open()

        # Whitelists
        self.building_blocks = self._parse_whitelist(config_storage["building_blocks"] or
                                                     api_ttypes.CodeIdentifierList())
        self.third_party_whitelist = self._parse_whitelist(config_storage["third_party_whitelist"] or
                                                           api_ttypes.CodeIdentifierList())
        self.known_errors = self._parse_whitelist(config_storage["known_errors"] or
                                                  api_ttypes.KnownErrorList())
        self.ignored_exceptions = set((config_storage["ignored_exceptions"] or
                                       api_ttypes.IgnoredExceptionList()).exceptions)

        # Watchers
        self.watch_all_errors, self.watch_only_if_blamed = self._parse_watchers_file(config_storage["watch_list"] or
                                                                                api_ttypes.WatchList())
        self.disownerships = self._parse_disownership(config_storage["disownership_list"] or
                                                      api_ttypes.FileDisownershipList())

        # Email remapping
        self.email_remapping = config_storage["email_remapping"] or api_ttypes.EmailRemapping()

        config_storage.close()

    def _parse_whitelist(self, code_identifier_list):
        parsed_objs = collections.defaultdict(list)
        for item in code_identifier_list.identifiers:
            parsed_objs[item.filename].append(item)
        return parsed_objs

    def _parse_watchers_file(self, watch_list):
        all_error_tree = prefix_tree.FilePathTree()
        blame_only_tree = prefix_tree.FilePathTree()

        for watch in watch_list.watches:
            tree = all_error_tree if watch.watch_all_errors else blame_only_tree
            if watch.filepath not in tree:
                tree[watch.filepath] = list()
            tree[watch.filepath].append(watch.email)

        # Set all_error_tree to have accumulator that will allow us to find everyone who was watching
        # the file or a parent of the file
        all_error_tree.set_accumulator(
            accumulator_intializer=list(),
            accumulator_func=lambda x, y: x + y if y else x,
        )

        return all_error_tree, blame_only_tree

    def _parse_disownership(self, disownership_list):
        disownership_tree = prefix_tree.FilePathTree()
        for entry in disownership_list.disownerships:
            if entry.filepath not in disownership_tree:
                disownership_tree[entry.filepath] = list()
            disownership_tree[entry.filepath].append(entry)

        # Set disownership_tree to have accumulator that will allow us to find everyone who has disowned
        # the file or a parent of the file
        disownership_tree.set_accumulator(
            accumulator_intializer=list(),
            accumulator_func=lambda x, y: x + y if y else x,
        )
        return disownership_tree

    ############################## Timestamp Helpers ##############################

    def _partition_for_ms(self, epoch_ms):
        timestamp_date = self._convert_epoch_ms(cls=datetime.date, epoch_ms=epoch_ms)
        timestamp_date = timestamp_date - datetime.timedelta(days=timestamp_date.isoweekday() % 7)
        file_path = os.path.join(config.data_dir_path, "flawless-errors-" + timestamp_date.strftime("%Y-%m-%d"))
        return file_path

    def _epoch_ms(self):
        return int(self.time_func() * 1000)

    def _convert_epoch_ms(self, cls, epoch_ms=None):
        if not epoch_ms:
            epoch_ms = self._epoch_ms()
        return cls.fromtimestamp(epoch_ms / 1000.)

    ############################## Traceback/Line Type Helpers ##############################

    def _blame_line(self, traceback):
        '''Figures out which line in traceback is to blame for the error.
        Returns a 3-tuple of (ErrorKey, StackTraceEntry, [email recipients])'''
        key = None
        blamed_entry = None
        email_recipients = []
        for stack_line in traceback:
            line_type = self._get_line_type(stack_line)
            if line_type == api_ttypes.LineType.THIRDPARTY_WHITELIST:
                return None, None, None, True
            elif line_type in [api_ttypes.LineType.DEFAULT, api_ttypes.LineType.KNOWN_ERROR]:
                filepath = self._get_basepath(stack_line.filename)
                entry = api_ttypes.CodeIdentifier(filepath, stack_line.function_name, stack_line.text)
                blamed_entry = entry
                key = api_ttypes.ErrorKey(filepath, stack_line.line_number, stack_line.function_name, stack_line.text)
                if filepath in self.watch_all_errors:
                    email_recipients.extend(self.watch_all_errors[filepath])
        return (key, blamed_entry, email_recipients, False)

    def _get_line_type(self, line):
        filepath = self._get_basepath(line.filename)
        if not filepath:
            return api_ttypes.LineType.BAD_FILEPATH

        entry = api_ttypes.CodeIdentifier(filepath, line.function_name, line.text)
        if entry in self.third_party_whitelist[filepath]:
            return api_ttypes.LineType.THIRDPARTY_WHITELIST
        elif not self._matches_filepath_pattern(filepath):
            return api_ttypes.LineType.IGNORED_FILEPATH
        elif entry in self.building_blocks[filepath]:
            return api_ttypes.LineType.BUILDING_BLOCK
        elif entry in self.known_errors[filepath]:
            return api_ttypes.LineType.KNOWN_ERROR
        elif self.raise_exception_pattern.match(line.text):
            return api_ttypes.LineType.RAISED_EXCEPTION
        else:
            return api_ttypes.LineType.DEFAULT

    def _matches_filepath_pattern(self, filepath):
        '''Given a filepath, and a list of regex patterns, this function returns true
        if filepath matches any one of those patterns'''
        if not self.only_blame_patterns:
            return True

        for pattern in self.only_blame_patterns:
            if pattern.match(filepath):
                return True
        return False

    def _get_basepath(self, filename):
        for pattern in self.extract_base_path_patterns:
            match = pattern.match(filename)
            if match:
                return match.group(1)
        return None

    def _format_traceback(self, err_info, append_locals=True, show_full_stack=False, include_err_info=False,
                          linebreak="
", spacer=" ", start_bold="",
                          end_bold="", escape_func=cgi.escape):
        request = err_info.last_error_data
        parts = []
        if include_err_info:
            parts.append("{b}Current Exception Count:{xb} {count}{lb}".format(
                b=start_bold, xb=end_bold, count=err_info.error_count, lb=linebreak))
            parts.append("{b}Last Occurred:{xb} {last_occurrence}{lb}".format(
                b=start_bold, xb=end_bold, last_occurrence=err_info.last_occurrence, lb=linebreak))
            parts.append("{b}Last Occurrence Logged From:{xb} {hostname}{lb}".format(
                b=start_bold, xb=end_bold, hostname=request.hostname, lb=linebreak))

        if request.exception_type:
            parts.append("{b}Exception Type:{xb} {type}{lb}".format(
                b=start_bold, xb=end_bold, type=request.exception_type, lb=linebreak))

        # Traceback
        parts.append("{b}Traceback (most recent call last):{xb}".format(b=start_bold, xb=end_bold))
        formatted_stack = [
            '{sp}{sp}File "{filename}", line {line}, in {function}{lb}{sp}{sp}{sp}{sp}{code}'.format(
                sp=spacer, lb=linebreak, filename=l.filename, line=l.line_number,
                function=l.function_name, code=escape_func(l.text),
            )
            for l in request.traceback
        ]
        parts.extend(formatted_stack)
        parts.append(escape_func(request.exception_message))

        def decode(value):
            if hasattr(value, 'decode'):
                return value.decode('UTF-8', 'replace')
            return value

        # Frame Locals
        parts.append(linebreak * 2 + "{b}Stack Frame:{xb}".format(b=start_bold, xb=end_bold))
        types_to_show = [api_ttypes.LineType.KNOWN_ERROR,
                         api_ttypes.LineType.DEFAULT,
                         api_ttypes.LineType.RAISED_EXCEPTION]
        frames_to_show = [l for l in request.traceback if l.frame_locals is not None and
                          (self._get_line_type(l) in types_to_show or show_full_stack)]
        for l in frames_to_show:
            line_info = '{sp}{sp}{b}File "{filename}", line {line}, in {function}{xb}'.format(
                sp=spacer, filename=l.filename, line=l.line_number, function=l.function_name,
                b=start_bold, xb=end_bold,
            )
            local_vals = ['{sp}{sp}{sp}{sp}{name}={value}'.format(
                          sp=spacer, name=name, value=escape_func(decode(value)))
                          for name, value in sorted(l.frame_locals.items())]
            parts.append(line_info)
            parts.extend(local_vals or [spacer * 4 + "No variables in this frame"])

        # Additional Information
        if request.additional_info:
            parts.append(linebreak * 2 + "{b}Additional Information:{xb}".format(b=start_bold, xb=end_bold))
            parts.append(
                escape_func(decode(request.additional_info)).replace("\n", linebreak)
            )

        return linebreak.join(parts)

############################## THRIFT SERVICE ##############################

class FlawlessThriftServiceHandler(FlawlessServiceBaseClass):
    """Handler for Thrift server API"""

    ############################## CONSTANTS ##############################

    # Validates that email address is valid. Does not attempt to be RFC compliant
    #     local part: any alphanumeric or ., %, +, \, -, _
    #     domain part: any alphanumeric. Dashes or periods allowed as long as they are not followed
    #                                by a period
    #     top level domain: between 2 to 4 alpha chracters
    VALIDATE_EMAIL_PATTERN = \
        re.compile(r"^[A-Za-z0-9.%+\-_]+@(?:(?:[a-zA-Z0-9]+-?)*[a-zA-Z0-9]\.)+[A-Za-z]{2,4}$")

    ############################## Init ##############################

    def __init__(self, *args, **kwargs):
        super(FlawlessThriftServiceHandler, self).__init__(*args, **kwargs)
        self.number_of_git_blames_running = 0

        self.repository = get_repository(open_process_func=self.open_process_func)
        self.emailed_errors_cache = ExpiringLRUCache(size=1000, expiration_seconds=60 * 5)

        self.lock = threading.RLock()
        self.errors_seen = None
        self._refresh_errors_seen()

        self.persist_thread = self.thread_cls(target=self._run_background_update_thread)
        self.persist_thread.daemon = True
        self.persist_thread.start()

    ############################## Update Thread ##############################

    def _refresh_errors_seen(self, epoch_ms=None):
        prefix = self._partition_for_ms(epoch_ms)
        with self.lock:
            if self.errors_seen is None:
                self.errors_seen = self.storage_factory(prefix)
                self.errors_seen.open()
            elif prefix != self.errors_seen.partition:
                # Order matters here since there can be a race condition if not done correctly
                old_errors_seen = self.errors_seen
                new_errors_seen = self.storage_factory(prefix)
                new_errors_seen.open()
                self.errors_seen = new_errors_seen
                old_errors_seen.sync()
                old_errors_seen.close()

    def _run_background_update_thread(self):
        while True:
            time.sleep(300)
            tasks_to_run = [
                lambda: self.errors_seen.sync(),
                lambda: self._refresh_errors_seen(),
                lambda: self.repository.update(),
                lambda: self._read_whitelist_configs(),
            ]
            # Run all items in try/except block because we don't want our background thread
            # to die.
            for task in tasks_to_run:
                try:
                    task()
                except:
                    self._handle_flawless_issue("
".join(traceback.format_exception(*sys.exc_info())))

    ############################## Misc Helper Funcs ##############################

    def ping(self):
        return True

    def _sendmail(self, to_addresses, subject, body):
        invalid_addresses = [e for e in to_addresses if not bool(self.VALIDATE_EMAIL_PATTERN.match(e))]
        if invalid_addresses:
            to_addresses = [e for e in to_addresses if e not in invalid_addresses]
            self._handle_flawless_issue(
                "Invalid email address found. Not sending to: %s" % ", ".join(invalid_addresses),
                log_func=log.warning,
            )

        msg = MIMEText(body.encode("UTF-8"), "html", "UTF-8")
        msg["From"] = config.smtp_from or "flawless@%s" % config.email_domain_name
        msg["To"] = ", ".join(to_addresses)
        msg["Subject"] = subject

        host, port = config.smtp_host.split(":")
        smtp_client = self.smtp_client_cls(host, int(port))

        if config.smtp_use_tls:
            smtp_client.starttls()
        if config.smtp_user and config.smtp_password:
            smtp_client.login(config.smtp_user, config.smtp_password)

        smtp_client.sendmail(msg["From"], to_addresses, msg.as_string())
        smtp_client.quit()

    def _get_email(self, email):
        '''Given an email address, check the email_remapping table to see if the email
        should be sent to a different address. This function also handles overriding
        the email domain if ignore_vcs_email_domain is set or the domain was missing'''
        if not email or "@" not in email:
            return None

        if email in self.email_remapping.remap:
            return self.email_remapping.remap[email]
        prefix, domain = email.split("@", 2)
        if prefix in self.email_remapping.remap:
            return self.email_remapping.remap[prefix]
        if "." not in domain or config.ignore_vcs_email_domain:
            return "%s@%s" % (prefix, config.email_domain_name)
        return email

    def _get_entry(self, entry, entry_tree):
        '''Helper function for retrieving a particular entry from the prefix trees'''
        for e in entry_tree[entry.filename]:
            if entry == e:
                return e

    def _handle_flawless_issue(self, message, log_func=log.error):
        log_func(message)
        if config.default_contact:
            self._sendmail([config.default_contact], "Unexpected problem on Flawless Server", message)

    ############################## Record Error ##############################

    def record_error(self, request):
        t = self.thread_cls(target=self._record_error, args=[request])
        t.start()

    def _record_error(self, request):
        log.debug("Recieved error from %s for %s" % (request.hostname, request.exception_message))

        # Skip ignored exceptions (ex: connection errors)
        if request.exception_type and request.exception_type in self.ignored_exceptions:
            return

        # Figure out which line in the stack trace is to blame for the error
        key, blamed_entry, email_recipients, was_whitelisted = self._blame_line(request.traceback)
        if not key:
            if not was_whitelisted:
                name_map = api_ttypes.LineType._VALUES_TO_NAMES
                line_types = [("%s:%d" % (stack_line.filename, stack_line.line_number),
                               name_map.get(self._get_line_type(stack_line)))
                              for stack_line in request.traceback]
                log.info("Unable to blame: %s" % str(line_types))
            return

        # If this error hasn't been reported before, then find the dev responsible
        err_info = None
        if key not in self.errors_seen:
            # If flawless is being flooded with errors, limit the number of git blames so the
            # service doesn't fall over. We don't use a thread safe counter, because 10
            # git blames is just a soft limit
            if self.number_of_git_blames_running > config.max_concurrent_git_blames:
                log.error("Unable to process %s because %d git blames already running" %
                          (str(key), self.number_of_git_blames_running))
                return
            try:
                self.number_of_git_blames_running += 1
                email, last_touched_ts = self.repository.blame(key.filename, key.line_number)
            finally:
                self.number_of_git_blames_running -= 1

            dev_email = self._get_email(email)
            if key.filename in self.disownerships:
                remapped_owners = {entry.email: entry.designated_email for entry in self.disownerships[key.filename]}
                dev_email = remapped_owners.get(dev_email, dev_email)

            last_touched_ts = last_touched_ts or 0
            cur_time = self._convert_epoch_ms(datetime.datetime).strftime("%Y-%m-%d %H:%M:%S")
            mod_time = self._convert_epoch_ms(datetime.datetime, epoch_ms=last_touched_ts * 1000)
            mod_time = mod_time.strftime("%Y-%m-%d %H:%M:%S")
            known_entry = self._get_entry(blamed_entry, self.known_errors)
            err_info = api_ttypes.ErrorInfo(error_count=1,
                                            developer_email=dev_email or "unknown",
                                            date=mod_time,
                                            email_sent=False,
                                            last_occurrence=cur_time,
                                            is_known_error=bool(known_entry),
                                            last_error_data=request)
            self.errors_seen[key] = err_info
            log.info("Error %s caused by %s on %s" % (str(key), dev_email, mod_time))

            if not dev_email:
                self._handle_flawless_issue("Unable to do blame for %s from %s. You may want to consider setting "
                                            "only_blame_filepaths_matching in your flawless.cfg " %
                                            (str(key), request.hostname))
                err_info.email_sent = True
                return
        # If we've already seen this error then update the error count
        elif key in self.errors_seen:
            err_info = self.errors_seen[key]
            err_info.error_count += request.error_count or 1
            err_info.last_error_data = request
            cur_dt = self._convert_epoch_ms(datetime.datetime)
            err_info.last_occurrence = cur_dt.strftime("%Y-%m-%d %H:%M:%S")
            self.errors_seen[key] = err_info

        # Figure out if we should send an email or not
        send_email = False
        known_entry = None
        if blamed_entry not in self.known_errors[blamed_entry.filename]:
            # If it is an unknown error, then it must meet certain criteria. The code must have been
            # touched after report_only_after_minimum_date so errors in old code can be ignored. It
            # also has to have occurred at least report_error_threshold times (although the client
            # is allowed to override that value).
            if (not err_info.email_sent and err_info.date >= config.report_only_after_minimum_date and
                    err_info.error_count >= (request.error_threshold or config.report_error_threshold)):
                send_email = True
        else:
            # If it is a known error, we allow fine grainted control of how frequently emails will
            # be sent. An email will be sent if it has passed the min_alert_threshold, and/or this
            # is the Nth occurrence as defined alert_every_n_occurrences. If it has passed
            # max_alert_threshold then no emails will be sent.
            known_entry = self._get_entry(blamed_entry, self.known_errors)
            if (known_entry.min_alert_threshold and err_info.error_count >= known_entry.min_alert_threshold
                    and not err_info.email_sent):
                send_email = True
            if (known_entry.alert_every_n_occurrences and
                    err_info.error_count % known_entry.alert_every_n_occurrences == 0):
                send_email = True
            if (known_entry.max_alert_threshold is not None
                    and err_info.error_count > known_entry.max_alert_threshold):
                send_email = False

        # Send email if applicable
        if send_email:
            self._send_error_email(request, key, err_info, blamed_entry, known_entry, email_recipients)

    def _send_error_email(self, request, err_key, err_info, blamed_entry, known_entry, email_recipients):
        email_body = []
        dev_email = self._get_email(err_info.developer_email)
        if dev_email:
            email_recipients.append(dev_email)

        # Add additional recipients that have registered for this error
        if blamed_entry.filename in self.watch_only_if_blamed:
            email_recipients.extend(self.watch_only_if_blamed[blamed_entry.filename])
        if known_entry:
            email_recipients.extend(known_entry.email_recipients or [])
            email_body.append(known_entry.email_header or "")

        email_body.append(self._format_traceback(err_info))
        email_body.append(
            "

View most recent occurrence" %
            (
                config.hostname,
                urlencode(
                    dict(filename=err_key.filename, function_name=err_key.function_name, text=err_key.text,
                         line_number=err_key.line_number, timestamp=self._epoch_ms() / 1000)
                )
            )
        )
        email_body.append("
View all errors assigned to you" % (config.hostname, dev_email))
        email_body.append(
            "
Add to whitelist" %
            (
                config.hostname,
                urlencode(
                    dict(filename=err_key.filename, function_name=err_key.function_name, code_fragment=err_key.text)
                )
            )
        )
        email_body.append("
What is Flawless?")

        if err_key in self.emailed_errors_cache:
            return  # Skip sending this email if we've already emailed it in the past 5 minutes
        self.emailed_errors_cache[err_key] = True

        # Send the email
        log.info("Sending email for %s to %s" % (str(err_key), ", ".join(email_recipients)))
        try:
            self._sendmail(
                to_addresses=email_recipients,
                subject="Error on %s in %s" % (request.hostname, err_key.filename),
                body="
".join([s for s in email_body if s]),
            )
            err_info.email_sent = True
            self.errors_seen[err_key] = err_info
        except smtplib.SMTPRecipientsRefused:
            log.error("Unable to send email to: %s for %s" % (str(err_key), ", ".join(email_recipients)))


#!/usr/bin/env python
#
# Copyright (c) 2011-2013, Shopkick Inc.
# All rights reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ---
# Author: John Egan <jwegan@gmail.com>

import os
import os.path
import re
from setuptools import setup, find_packages

def markdown_to_reST(text):
    '''This is not a general purpose converter. Only converts this readme'''
    # Convert parameters to italics and prepend a newline
    text = re.sub(pattern=r"\n       (\w+) - (.+)\n",
                  repl=r"\n\n       *\g<1>* - \g<2>\n",
                  string=text)

    # Parse [http://url](text), and just leave the url
    text = re.sub(pattern=r"\[([^\]]+)\]\([^)]+\)",
                  repl=r"\g<1>",
                  string=text)

    # Disable formatting of numbered lists
    text = re.sub(pattern=r"\n(\d+). ",
                  repl=r"\n\\\g<1>. ",
                  string=text)
    return text

setup(
    name='flawless',
    version='0.4.3',
    description='Python Error Monitoring and Reporting',
    long_description=markdown_to_reST(open("README.md").read()),
    license='MPL 2.0',
    author='John Egan',
    author_email='jwegan@gmail.com',
    url='http://github.com/shopkick/flawless',
    zip_safe=False,
    packages=find_packages(exclude=['ez_setup', 'examples', 'packages', 'tests*']),
    data_files=[('config', [os.path.join('config', f) for f in os.listdir('config')
                            if not f.startswith('.')])],
    entry_points="""
    [console_scripts]
    flawless = flawless.server.manage:main
    """,
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Framework :: Pylons",
        "Framework :: Django",
        "Framework :: Pyramid",
        "Framework :: Paste",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Topic :: Software Development :: Bug Tracking",
        "Topic :: Software Development :: Quality Assurance",
    ],
    install_requires=[
        'future',
        'thrift>=0.10.0',
    ],
)


#!/usr/bin/env python
#
# Copyright (c) 2011-2013, Shopkick Inc.
# All rights reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ---
# Author: John Egan <jwegan@gmail.com>

try:
    from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
except ImportError:
    from http.server import HTTPServer, BaseHTTPRequestHandler
import logging
import logging.handlers
import os
import os.path
try:
    from SocketServer import ThreadingMixIn
except ImportError:
    from socketserver import ThreadingMixIn
import signal
import sys
try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

from thrift.protocol import TBinaryProtocol
from thrift.server import TServer
from thrift.transport import TSocket
from thrift.transport import TTransport

import flawless.lib.config
from flawless.lib.storage import DiskStorage
from flawless.server.api import Flawless
from flawless.server.service import FlawlessThriftServiceHandler
from flawless.server.webapp import FlawlessWebServiceHandler

log = logging.getLogger(__name__)
config = flawless.lib.config.get()

class SimpleThreadedHTTPServer(ThreadingMixIn, HTTPServer):
    def attach_service(self, service):
        self.service = service

    def server_close(self):
        HTTPServer.server_close(self)

class SimpleRequestHTTPHandler(BaseHTTPRequestHandler):

    def do_GET(self):
        parts = urlparse.urlparse(self.path)
        kwargs = dict(urlparse.parse_qsl(parts.query))
        ret = None

        try:
            if hasattr(self.server.service, parts.path[1:] or "index"):
                ret = getattr(self.server.service, parts.path[1:] or "index")(**kwargs)
                self.send_response(200)
                self.send_header('Content-Length', len(ret or ""))
                self.send_header('Content-Type', 'text/html')
            else:
                self.send_response(404)
        except Exception as e:
            log.exception(e)
            self.send_response(500)
        finally:
            self.end_headers()
        if ret:
            self.wfile.write(bytes(ret.encode('utf8')))

    def do_POST(self):
        # Read in POST body
        parts = urlparse.urlparse(self.path)
        content_length = int(self.headers.get("Content-Length"))
        req_str = self.rfile.read(content_length).decode('utf8')

        ret = None
        try:
            ret = getattr(self.server.service, parts.path[1:])(req_str)
            self.send_response(200)
            if ret:
                self.send_header('Content-Length', len(ret))
                self.send_header('Content-Type', 'text/html')
        except Exception as e:
            log.exception(e)
            self.send_response(500)
        finally:
            self.end_headers()

        if ret:
            self.wfile.write(bytes(ret.encode('utf8')))

def serve(conf_path, storage_factory=None):
    """This method starts the server. There are two processes, one is an HTTP server that shows
    and admin interface and the second is a Thrift server that the client code calls.

    Arguments:
        `conf_path` - The path to your flawless.cfg file
        `storage_factory` - You can pass in your own storage class that implements StorageInterface. You must implement
                            storage_cls if you want Flawless to be horizontally scalable, since by default it will just
                            store everything on the local disk.
    """

    flawless.lib.config.init_config(conf_path)
    # Try and create datadir if it doesn't exist. For instance it might be in /tmp
    if not os.path.exists(config.data_dir_path):
        os.makedirs(config.data_dir_path)

    storage_factory = storage_factory or (lambda partition: DiskStorage(partition=partition))

    # Setup root logger
    root_logger = logging.getLogger()
    root_handler = logging.handlers.TimedRotatingFileHandler(
        filename=config.log_file, when='d', interval=1, backupCount=config.log_days_to_keep)
    root_logger.setLevel(getattr(logging, config.log_level))
    root_logger.addHandler(root_handler)

    child_pid = os.fork()
    if child_pid == 0:
        # Setup HTTP server
        handler = FlawlessWebServiceHandler(storage_factory=storage_factory)
        server = SimpleThreadedHTTPServer(('', config.http_port), SimpleRequestHTTPHandler)
        server.attach_service(handler)
        server.request_queue_size = 50

        try:
            server.serve_forever()
        except (KeyboardInterrupt, SystemExit):
            server.server_close()
    else:
        # Setup Thrift server
        handler = FlawlessThriftServiceHandler(storage_factory=storage_factory)
        processor = Flawless.Processor(handler)
        transport = TSocket.TServerSocket(port=config.port)
        tfactory = TTransport.TFramedTransportFactory()
        pfactory = TBinaryProtocol.TBinaryProtocolFactory()
        server = TServer.TThreadedServer(processor, transport, tfactory, pfactory)
        try:
            server.serve()
        except (KeyboardInterrupt, SystemExit):
            handler.errors_seen.sync()
            transport.close()
            os.kill(child_pid, signal.SIGINT)

def main():
    conf_path = flawless.lib.config.default_path
    if len(sys.argv) > 1:
        conf_path = sys.argv[1]
    serve(conf_path)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
#
# Copyright (c) 2011-2013, Shopkick Inc.
# All rights reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ---
# Author: John Egan <jwegan@gmail.com>

import functools
import hashlib
import linecache
import logging
import math
import os.path
import random
import re
import socket
import sys
import time
import traceback
import warnings

from future.utils import raise_
from thrift.Thrift import TException
from thrift.transport import TTransport
from thrift.transport import TSocket
from thrift.protocol import TBinaryProtocol

import flawless.lib.config
from flawless.lib.data_structures.lru_cache import ExpiringLRUCache
import flawless.server.api.ttypes as api_ttypes
from flawless.server.api import Flawless

log = logging.getLogger(__name__)
config = flawless.lib.config.get()

MAX_VARIABLE_REPR = 250
MAX_LOCALS = 100
NUM_FRAMES_TO_SAVE = 20

HOSTPORT_INFO = list()
SCRUBBED_VARIABLES_REGEX = None

CACHE_ERRORS_AFTER_N_OCCURRENCES = 10
REPORT_AFTER_N_MILLIS = 10 * 60 * 1000  # 10 minutes
LRU_CACHE_SIZE = 200
ERROR_CACHE = ExpiringLRUCache(size=LRU_CACHE_SIZE)

class HostportInfo(object):

    def __init__(self, hostport):
        host, port = hostport.split(":")
        self.host = host
        self.port = int(port)
        self.backoff_ms = 0
        self.consecutive_connection_errors = 0

    def increment_backoff(self):
        self.consecutive_connection_errors = max(12, self.consecutive_connection_errors + 1)
        backoff = 1000 * random.randint(1, 2 ** self.consecutive_connection_errors)
        self.backoff_ms = _get_epoch_ms() + backoff

    def decrement_backoff(self):
        self.consecutive_connection_errors = int(self.consecutive_connection_errors / 2)
        if self.consecutive_connection_errors > 0:
            backoff = 1000 * random.randint(1, 2 ** self.consecutive_connection_errors)
            self.backoff_ms = _get_epoch_ms() + backoff

class CachedErrorInfo(object):

    def __init__(self):
        self.last_report_ts = _get_epoch_ms()
        self.last_occur_ts = _get_epoch_ms()
        self.curr_count = 0
        self.last_report_count = 0

    @classmethod
    def get_hash_key(cls, stack_lines):
        m = hashlib.md5()
        for line in stack_lines:
            m.update(line.filename.encode('utf8'))
            m.update(str(line.line_number).encode('utf8'))
        return m.digest()

    def increment(self):
        self.last_occur_ts = _get_epoch_ms()
        self.curr_count += 1

    def mark_reported(self):
        self.last_report_ts = _get_epoch_ms()
        diff = self.curr_count - self.last_report_count
        self.last_report_count = self.curr_count
        return diff

    def should_report(self):
        report_conditions = list()
        report_conditions.append(self.curr_count <= CACHE_ERRORS_AFTER_N_OCCURRENCES)
        report_conditions.append(self.last_report_ts < (_get_epoch_ms() - REPORT_AFTER_N_MILLIS))
        log_count = math.log(self.curr_count, 2)
        report_conditions.append(int(log_count) == log_count)
        return any(report_conditions)

def _get_epoch_ms():
    return int(time.time() * 1000)

def set_hostports(hostports):
    if type(hostports) not in [tuple, list]:
        raise ValueError("hostports must be a list or tuple")
    global HOSTPORT_INFO
    HOSTPORT_INFO = [HostportInfo(hp) for hp in hostports]

def install_scrubbers(variables_regex):
    global SCRUBBED_VARIABLES_REGEX
    SCRUBBED_VARIABLES_REGEX = re.compile(variables_regex)

def _get_backend_host():
    if config.flawless_hostports and not HOSTPORT_INFO:
        set_hostports(config.flawless_hostports)
    return random.choice(HOSTPORT_INFO) if HOSTPORT_INFO else None

def _get_service():
    hostport_info = _get_backend_host()
    if not hostport_info:
        warnings.warn("Unable to record error: flawless server hostport not set", RuntimeWarning)
        return None, None, None

    tsocket = TSocket.TSocket(hostport_info.host, hostport_info.port)
    tsocket.setTimeout(2000)  # 2 second timeout
    transport = TTransport.TFramedTransport(tsocket)
    protocol = TBinaryProtocol.TBinaryProtocol(transport)
    client = Flawless.Client(protocol)
    return client, transport, hostport_info

def _send_request(req):
    # Try to send the request. If there are too many connection errors, then backoff
    client, transport, hostport_info = _get_service()
    try:
        if all([client, transport, hostport_info]) and _get_epoch_ms() >= hostport_info.backoff_ms:
            transport.open()
            client.record_error(req)
            hostport_info.decrement_backoff()
    except TException as e:
        hostport_info.increment_backoff()
        log.exception(e)
    finally:
        if transport and transport.isOpen():
            transport.close()

def _myrepr(var_name, value):
    try:
        if SCRUBBED_VARIABLES_REGEX and SCRUBBED_VARIABLES_REGEX.match(var_name):
            return '**scrubbed**'

        repr_str = repr(value)
        return repr_str[:MAX_VARIABLE_REPR] + "..." * int(len(repr_str) > MAX_VARIABLE_REPR)
    except:
        return "Exception executing repr for this field"

def record_error(hostname, exc_info, preceding_stack=None, error_threshold=None, additional_info=None):
    ''' Helper function to record errors to the flawless backend '''
    stack = []
    exc_type, exc_value, sys_traceback = exc_info

    while sys_traceback is not None:
        stack.append(sys_traceback)
        sys_traceback = sys_traceback.tb_next

    stack_lines = []
    for row in preceding_stack or []:
        stack_lines.append(
            api_ttypes.StackLine(filename=os.path.abspath(row[0]), line_number=row[1],
                                 function_name=row[2], text=row[3])
        )

    for index, tb in enumerate(stack):
        filename = tb.tb_frame.f_code.co_filename
        func_name = tb.tb_frame.f_code.co_name
        lineno = tb.tb_lineno
        line = linecache.getline(filename, lineno, tb.tb_frame.f_globals)
        frame_locals = None
        if index >= (len(stack) - NUM_FRAMES_TO_SAVE):
            # Include some limits on max string length & number of variables to keep things from getting
            # out of hand
            frame_locals = dict((k, _myrepr(k, v)) for k, v in
                                list(tb.tb_frame.f_locals.items())[:MAX_LOCALS] if k != "self")
            if "self" in tb.tb_frame.f_locals and hasattr(tb.tb_frame.f_locals["self"], "__dict__"):
                frame_locals.update(dict(("self." + k, _myrepr(k, v)) for k, v in
                                         list(tb.tb_frame.f_locals["self"].__dict__.items())[:MAX_LOCALS]
                                         if k != "self"))

        stack_lines.append(
            api_ttypes.StackLine(filename=os.path.abspath(filename), line_number=lineno,
                                 function_name=func_name, text=line, frame_locals=frame_locals)
        )

    # Check LRU cache & potentially do not send error report if this client has already reported this error
    # several times.
    key = CachedErrorInfo.get_hash_key(stack_lines)
    info = ERROR_CACHE.get(key) or CachedErrorInfo()
    info.increment()
    ERROR_CACHE[key] = info
    if info.should_report():
        error_count = info.mark_reported()
        _send_request(
            api_ttypes.RecordErrorRequest(
                traceback=stack_lines,
                exception_message=repr(exc_value),
                exception_type=exc_type.__module__ + "." + exc_type.__name__,
                hostname=hostname,
                error_threshold=error_threshold,
                additional_info=additional_info,
                error_count=error_count,
            )
        )

def _safe_wrap(func):
    safe_attrs = [attr for attr in functools.WRAPPER_ASSIGNMENTS if hasattr(func, attr)]
    return functools.wraps(func, safe_attrs)

def _wrap_function_with_error_decorator(func,
                                        save_current_stack_trace=True,
                                        reraise_exception=True,
                                        error_threshold=None):
    preceding_stack = []
    if save_current_stack_trace:
        preceding_stack = traceback.extract_stack()

    @_safe_wrap(func)
    def wrapped_func_with_error_reporting(*args, **kwargs):
        if not _get_backend_host():
            warnings.warn("flawless server hostport not set", RuntimeWarning, stacklevel=2)
        try:
            return func(*args, **kwargs)
        except:
            type, value, sys_traceback = sys.exc_info()

            # Check to try and prevent multiple reports of the same exception
            if hasattr(value, "_flawless_already_caught"):
                if reraise_exception:
                    raise_(value, None, sys_traceback)
                else:
                    return

            # Get trackback & report it
            hostname = socket.gethostname()
            record_error(
                hostname=hostname,
                exc_info=(type, value, sys_traceback),
                preceding_stack=preceding_stack,
                error_threshold=error_threshold,
            )

            # Reraise exception if so desired
            if reraise_exception:
                object.__setattr__(value, "_flawless_already_caught", True)
                raise_(value, None, sys_traceback)
    return wrapped_func_with_error_reporting


#!/usr/bin/env python
#
# Copyright (c) 2011-2013, Shopkick Inc.
# All rights reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ---
# Author: John Egan <jwegan@gmail.com>

import abc
import copy

from future.utils import with_metaclass

class StorageInterface(with_metaclass(abc.ABCMeta, object)):
    """By default Flawless stores everything on disk which means there can only be one centralized instance of
    Flawless. You can implement your own instance of StorageInterface that connects to a backend database and pass
    it into flawless.server.server.serve. Then it is possible to have the flawless server be horizontally scalable
    since the database serves as the centralized source of truth.

    It is worth noting is that the keys used in this interface are either python primitives (string, list, etc) or
    thrift objects. The values can also be python primitives (list, dictionary, etc) or thrift objects
    """

    def __init__(self, partition):
        """partition is a string used to partition keys by week. For instance, with disk storage, we create a new
        file for every unique partition. For a database you may want to consider prepending all keys with
        partition. For accessing config data, the partition is None"""
        self.partition = partition

    def open(self):
        """Called to create connection to storage"""
        pass

    def sync(self):
        """Called periodically to flush data"""
        pass

    def close(self):
        """Called to close connection to storage"""
        pass

    def migrate_thrift_obj(self, obj):
        """Helper function that can be called when serializing/deserializing thrift objects whose definitions
        have changed, we need to make sure we initialize the new attributes to their default value"""
        if not hasattr(obj, "thrift_spec"):
            return

        obj_key_set = set(obj.__dict__.keys())
        thrift_field_map = {t[2]: t[4] for t in obj.thrift_spec if t}
        obj.__dict__.update({f: copy.copy(thrift_field_map[f]) for f in set(thrift_field_map.keys()) - obj_key_set})
        for value in obj.__dict__.values():
            self.migrate_thrift_obj(value)

    @abc.abstractmethod
    def iteritems(self):
        """Should return iterator of tuples (key, value) for all entries for the given self.partition"""
        pass

    @abc.abstractmethod
    def __setitem__(self, key, item):
        pass

    @abc.abstractmethod
    def __getitem__(self, key):
        """Should return None if key does not exist"""
        pass

    @abc.abstractmethod
    def __contains__(self, key):
        pass


import requests
from cStringIO import StringIO
from PIL import Image

def file_to_image(image_file_name):
    """
    Convert a file into a Pillow Image object
    """
    with open(image_file_name, 'r') as f_image:
        return Image.open(f_image)

def url_to_image(url):
    """
    Fetch an image from url and convert it into a Pillow Image object
    """
    r = requests.get(url)
    image = StringIO(r.content)
    return image

def string_to_image(image_string):
    """
    Convert string datas into a Pillow Image object
    """
    image_filelike = StringIO(image_string)
    image = Image.open(image_filelike)
    return image