# -*- coding: utf-8 -*-
'''
    python-capture-agent
    ~~~~~~~~~~~~~~~~~~~~

    :copyright: 2014-2017, Lars Kiesow <lkiesow@uos.de>
    :license: LGPL – see license.lgpl for more details.
'''

from pyca.utils import http_request, configure_service, unix_ts, timestamp
from pyca.utils import set_service_status, terminate
from pyca.config import config
from pyca.db import get_session, UpcomingEvent, Service, ServiceStatus
from base64 import b64decode
from datetime import datetime
import dateutil.parser
import logging
import pycurl
import sdnotify
import sys
import time
import traceback
if sys.version_info[0] == 2:
    from urllib import urlencode
else:
    from urllib.parse import urlencode

logger = logging.getLogger(__name__)
notify = sdnotify.SystemdNotifier()

def parse_ical(vcal):
    '''Parse Opencast schedule iCalendar file and return events as dict
    '''
    vcal = vcal.replace('\r\n ', '').replace('\r\n\r\n', '\r\n')
    vevents = vcal.split('\r\nBEGIN:VEVENT\r\n')
    del(vevents[0])
    events = []
    for vevent in vevents:
        event = {}
        for line in vevent.split('\r\n'):
            line = line.split(':', 1)
            key = line[0].lower()
            if len(line) <= 1 or key == 'end':
                continue
            if key.startswith('dt'):
                event[key] = unix_ts(dateutil.parser.parse(line[1]))
                continue
            if not key.startswith('attach'):
                event[key] = line[1]
                continue
            # finally handle attachments
            event['attach'] = event.get('attach', [])
            attachment = {}
            for x in [x.split('=') for x in line[0].split(';')]:
                if x[0].lower() in ['fmttype', 'x-apple-filename']:
                    attachment[x[0].lower()] = x[1]
            attachment['data'] = b64decode(line[1]).decode('utf-8')
            event['attach'].append(attachment)
        events.append(event)
    return events

def get_schedule():
    '''Try to load schedule from the Matterhorn core. Returns a valid schedule
    or None on failure.
    '''
    params = {'agentid': config()['agent']['name'].encode('utf8')}
    lookahead = config()['agent']['cal_lookahead'] * 24 * 60 * 60
    if lookahead:
        params['cutoff'] = str((timestamp() + lookahead) * 1000)
    uri = '%s/calendars?%s' % (config()['service-scheduler'][0],
                               urlencode(params))
    try:
        vcal = http_request(uri)
    except pycurl.error as e:
        logger.error('Could not get schedule: %s' % e)
        return

    try:
        cal = parse_ical(vcal.decode('utf-8'))
    except Exception:
        logger.error('Could not parse ical')
        logger.error(traceback.format_exc())
        return
    db = get_session()
    db.query(UpcomingEvent).delete()
    for event in cal:
        # Ignore events that have already ended
        if event['dtend'] <= timestamp():
            continue
        e = UpcomingEvent()
        e.start = event['dtstart']
        e.end = event['dtend']
        e.uid = event.get('uid')
        e.title = event.get('summary')
        e.set_data(event)
        db.add(e)
    db.commit()

def control_loop():
    '''Main loop, retrieving the schedule.
    '''
    set_service_status(Service.SCHEDULE, ServiceStatus.BUSY)
    notify.notify('READY=1')
    while not terminate():
        notify.notify('WATCHDOG=1')
        # Try getting an updated schedule
        get_schedule()
        session = get_session()
        next_event = session.query(UpcomingEvent)\
                            .filter(UpcomingEvent.end > timestamp())\
                            .order_by(UpcomingEvent.start)\
                            .first()
        if next_event:
            logger.info('Next scheduled recording: %s',
                        datetime.fromtimestamp(next_event.start))
            notify.notify('STATUS=Next scheduled recording: %s' %
                          datetime.fromtimestamp(next_event.start))
        else:
            logger.info('No scheduled recording')
            notify.notify('STATUS=No scheduled recording')
        session.close()

        next_update = timestamp() + config()['agent']['update_frequency']
        while not terminate() and timestamp() < next_update:
            time.sleep(0.1)

    logger.info('Shutting down schedule service')
    set_service_status(Service.SCHEDULE, ServiceStatus.STOPPED)

def run():
    '''Start the capture agent.
    '''
    configure_service('scheduler')
    control_loop()


# -*- coding: utf-8 -*-
'''
    python-capture-agent
    ~~~~~~~~~~~~~~~~~~~~

    :copyright: 2014-2017, Lars Kiesow <lkiesow@uos.de>
    :license: LGPL – see license.lgpl for more details.
'''

from pyca.utils import set_service_status, update_agent_state, timestamp
from pyca.utils import terminate
from pyca.config import config
from pyca.db import Service, ServiceStatus
import logging
import sdnotify
import time

logger = logging.getLogger(__name__)
notify = sdnotify.SystemdNotifier()

def control_loop():
    '''Main loop, updating the capture agent state.
    '''
    set_service_status(Service.AGENTSTATE, ServiceStatus.BUSY)
    notify.notify('READY=1')
    notify.notify('STATUS=Running')
    while not terminate():
        notify.notify('WATCHDOG=1')
        update_agent_state()

        next_update = timestamp() + config()['agent']['update_frequency']
        while not terminate() and timestamp() < next_update:
            time.sleep(0.1)

    logger.info('Shutting down agentstate service')
    set_service_status(Service.AGENTSTATE, ServiceStatus.STOPPED)

def run():
    '''Start the capture agent state process.
    '''
    control_loop()


# -*- coding: utf-8 -*-
from flask import jsonify, make_response, request
from pyca.db import Service, ServiceStatus, UpcomingEvent, RecordedEvent
from pyca.db import get_session, Status
from pyca.ui import app
from pyca.ui.utils import requires_auth, jsonapi_mediatype
from pyca.utils import get_service_status, ensurelist
import logging
import shutil

logger = logging.getLogger(__name__)

def make_error_response(error, status=500):
    ''' Return a response with a jsonapi error object
    '''
    content = {
        'errors': [{
            'status': status,
            'title': error
        }]
    }
    return make_response(jsonify(content), status)

def make_data_response(data, status=200):
    ''' Return a response with a list of jsonapi data objects
    '''
    content = {'data': ensurelist(data)}
    return make_response(jsonify(content), status)

@app.route('/api/services')
@requires_auth
@jsonapi_mediatype
def internal_state():
    '''Serve a json representation of internal agentstate as meta data
    '''
    data = {'services': {
        'capture': ServiceStatus.str(get_service_status(Service.CAPTURE)),
        'ingest': ServiceStatus.str(get_service_status(Service.INGEST)),
        'schedule': ServiceStatus.str(get_service_status(Service.SCHEDULE)),
        'agentstate': ServiceStatus.str(get_service_status(Service.AGENTSTATE))
        }
    }
    return make_response(jsonify({'meta': data}))

@app.route('/api/events/')
@requires_auth
@jsonapi_mediatype
def events():
    '''Serve a JSON representation of events
    '''
    db = get_session()
    upcoming_events = db.query(UpcomingEvent)\
                        .order_by(UpcomingEvent.start)
    recorded_events = db.query(RecordedEvent)\
                        .order_by(RecordedEvent.start.desc())

    result = [event.serialize() for event in upcoming_events]
    result += [event.serialize() for event in recorded_events]
    return make_data_response(result)

@app.route('/api/events/<uid>')
@requires_auth
@jsonapi_mediatype
def event(uid):
    '''Return a specific events JSON
    '''
    db = get_session()
    event = db.query(RecordedEvent).filter(RecordedEvent.uid == uid).first() \
        or db.query(UpcomingEvent).filter(UpcomingEvent.uid == uid).first()

    if event:
        return make_data_response(event.serialize())
    return make_error_response('No event with specified uid', 404)

@app.route('/api/events/<uid>', methods=['DELETE'])
@requires_auth
@jsonapi_mediatype
def delete_event(uid):
    '''Delete a specific event identified by its uid. Note that only recorded
    events can be deleted. Events in the buffer for upcoming events are
    regularly replaced anyway and a manual removal could have unpredictable
    effects.

    Use ?hard=true parameter to delete the recorded files on disk as well.

    Returns 204 if the action was successful.
    Returns 404 if event does not exist
    '''
    logger.info('deleting event %s via api', uid)
    db = get_session()
    events = db.query(RecordedEvent).filter(RecordedEvent.uid == uid)
    if not events.count():
        return make_error_response('No event with specified uid', 404)
    hard_delete = request.args.get('hard', 'false')
    if hard_delete == 'true':
        logger.info('deleting recorded files at %s', events[0].directory())
        shutil.rmtree(events[0].directory())
    events.delete()
    db.commit()
    return make_response('', 204)

@app.route('/api/events/<uid>', methods=['PATCH'])
@requires_auth
@jsonapi_mediatype
def modify_event(uid):
    '''Modify an event specified by its uid. The modifications for the event
    are expected as JSON with the content type correctly set in the request.

    Note that this method works for recorded events only. Upcoming events part
    of the scheduler cache cannot be modified.
    '''
    try:
        data = request.get_json()['data'][0]
        if data['type'] != 'event' or data['id'] != uid:
            return make_error_response('Invalid data', 400)
        # Check attributes
        for key in data['attributes'].keys():
            if key not in ('status', 'start', 'end'):
                return make_error_response('Invalid data', 400)
        # Check new status
        new_status = data['attributes'].get('status')
        if new_status:
            new_status = new_status.upper().replace(' ', '_')
            data['attributes']['status'] = int(getattr(Status, new_status))
    except Exception:
        return make_error_response('Invalid data', 400)

    db = get_session()
    event = db.query(RecordedEvent).filter(RecordedEvent.uid == uid).first()
    if not event:
        return make_error_response('No event with specified uid', 404)
    event.start = data['attributes'].get('start', event.start)
    event.end = data['attributes'].get('end', event.end)
    event.status = data['attributes'].get('status', event.status)
    logger.debug('Updating event %s via api', uid)
    db.commit()
    return make_data_response(event.serialize())


# -*- coding: utf-8 -*-
'''
    python-capture-agent
    ~~~~~~~~~~~~~~~~~~~~

    :copyright: 2014-2017, Lars Kiesow <lkiesow@uos.de>
    :license: LGPL – see license.lgpl for more details.
'''

from pyca.config import config
from pyca.db import get_session, RecordedEvent, Status, Service, ServiceStatus
from pyca.utils import http_request, configure_service, set_service_status
from pyca.utils import set_service_status_immediate, recording_state
from pyca.utils import update_event_status, terminate
from random import randrange
import logging
import pycurl
import sdnotify
import time
import traceback

logger = logging.getLogger(__name__)
notify = sdnotify.SystemdNotifier()

def get_config_params(properties):
    '''Extract the set of configuration parameters from the properties attached
    to the schedule
    '''
    param = []
    wdef = ''
    for prop in properties.split('\n'):
        if prop.startswith('org.opencastproject.workflow.config'):
            key, val = prop.split('=', 1)
            key = key.split('.')[-1]
            param.append((key, val))
        elif prop.startswith('org.opencastproject.workflow.definition'):
            wdef = prop.split('=', 1)[-1]
    return wdef, param

def ingest(event):
    '''Ingest a finished recording to the Opencast server.
    '''
    # Update status
    set_service_status(Service.INGEST, ServiceStatus.BUSY)
    notify.notify('STATUS=Uploading')
    recording_state(event.uid, 'uploading')
    update_event_status(event, Status.UPLOADING)

    # Select ingest service
    # The ingest service to use is selected at random from the available
    # ingest services to ensure that not every capture agent uses the same
    # service at the same time
    service = config('service-ingest')
    service = service[randrange(0, len(service))]
    logger.info('Selecting ingest service to use: ' + service)

    # create mediapackage
    logger.info('Creating new mediapackage')
    mediapackage = http_request(service + '/createMediaPackage')

    # extract workflow_def, workflow_config and add DC catalogs
    prop = 'org.opencastproject.capture.agent.properties'
    dcns = 'http://www.opencastproject.org/xsd/1.0/dublincore/'
    for attachment in event.get_data().get('attach'):
        data = attachment.get('data')
        if attachment.get('x-apple-filename') == prop:
            workflow_def, workflow_config = get_config_params(data)

        # Check for dublincore catalogs
        elif attachment.get('fmttype') == 'application/xml' and dcns in data:
            name = attachment.get('x-apple-filename', '').rsplit('.', 1)[0]
            logger.info('Adding %s DC catalog' % name)
            fields = [('mediaPackage', mediapackage),
                      ('flavor', 'dublincore/%s' % name),
                      ('dublinCore', data.encode('utf-8'))]
            mediapackage = http_request(service + '/addDCCatalog', fields)

    # add track
    for (flavor, track) in event.get_tracks():
        logger.info('Adding track ({0} -> {1})'.format(flavor, track))
        track = track.encode('ascii', 'ignore')
        fields = [('mediaPackage', mediapackage), ('flavor', flavor),
                  ('BODY1', (pycurl.FORM_FILE, track))]
        mediapackage = http_request(service + '/addTrack', fields)

    # ingest
    logger.info('Ingest recording')
    fields = [('mediaPackage', mediapackage)]
    if workflow_def:
        fields.append(('workflowDefinitionId', workflow_def))
    if event.uid:
        fields.append(('workflowInstanceId',
                       event.uid.encode('ascii', 'ignore')))
    fields += workflow_config
    mediapackage = http_request(service + '/ingest', fields)

    # Update status
    recording_state(event.uid, 'upload_finished')
    update_event_status(event, Status.FINISHED_UPLOADING)
    notify.notify('STATUS=Running')
    set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)

    logger.info('Finished ingest')

def safe_start_ingest(event):
    '''Start a capture process but make sure to catch any errors during this
    process, log them but otherwise ignore them.
    '''
    try:
        ingest(event)
    except Exception:
        logger.error('Something went wrong during the upload')
        logger.error(traceback.format_exc())
        # Update state if something went wrong
        recording_state(event.uid, 'upload_error')
        update_event_status(event, Status.FAILED_UPLOADING)
        set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)

def control_loop():
    '''Main loop of the capture agent, retrieving and checking the schedule as
    well as starting the capture process if necessry.
    '''
    set_service_status(Service.INGEST, ServiceStatus.IDLE)
    notify.notify('READY=1')
    notify.notify('STATUS=Running')
    while not terminate():
        notify.notify('WATCHDOG=1')
        # Get next recording
        event = get_session().query(RecordedEvent)\
                             .filter(RecordedEvent.status ==
                                     Status.FINISHED_RECORDING).first()
        if event:
            safe_start_ingest(event)
        time.sleep(1.0)
    logger.info('Shutting down ingest service')
    set_service_status(Service.INGEST, ServiceStatus.STOPPED)

def run():
    '''Start the capture agent.
    '''
    # If we are a backup CA, we don't want to actually upload anything. So
    # let's just quit here and do not run the ingest service at all.
    if config('agent')['backup_mode']:
        return

    configure_service('ingest')
    configure_service('capture.admin')
    control_loop()


# -*- coding: utf-8 -*-
'''
    python-capture-agent
    ~~~~~~~~~~~~~~~~~~~~

    :copyright: 2014-2017, Lars Kiesow <lkiesow@uos.de>
    :license: LGPL – see license.lgpl for more details.
'''

from pyca.utils import timestamp, try_mkdir, configure_service, terminate
from pyca.utils import set_service_status, set_service_status_immediate
from pyca.utils import recording_state, update_event_status
from pyca.config import config
from pyca.db import get_session, RecordedEvent, UpcomingEvent, Status,\
                    Service, ServiceStatus
import logging
import os
import os.path
import sdnotify
import shlex
import signal
import subprocess
import sys
import time
import traceback

logger = logging.getLogger(__name__)
notify = sdnotify.SystemdNotifier()
captureproc = None

def sigterm_handler(signum, frame):
    '''Intercept sigterm and terminate all processes.
    '''
    if captureproc and captureproc.poll() is None:
        captureproc.terminate()
    terminate(True)
    sys.exit(0)

def start_capture(upcoming_event):
    '''Start the capture process, creating all necessary files and directories
    as well as ingesting the captured files if no backup mode is configured.
    '''
    logger.info('Start recording')

    # First move event to recording_event table
    db = get_session()
    event = db.query(RecordedEvent)\
              .filter(RecordedEvent.uid == upcoming_event.uid)\
              .filter(RecordedEvent.start == upcoming_event.start)\
              .first()
    if not event:
        event = RecordedEvent(upcoming_event)
        db.add(event)
        db.commit()

    try_mkdir(config()['capture']['directory'])
    os.mkdir(event.directory())

    # Set state
    update_event_status(event, Status.RECORDING)
    recording_state(event.uid, 'capturing')
    set_service_status_immediate(Service.CAPTURE, ServiceStatus.BUSY)

    # Recording
    tracks = recording_command(event)
    event.set_tracks(tracks)
    db.commit()

    # Set status
    update_event_status(event, Status.FINISHED_RECORDING)
    recording_state(event.uid, 'capture_finished')
    set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)

    logger.info('Finished recording')

def safe_start_capture(event):
    '''Start a capture process but make sure to catch any errors during this
    process, log them but otherwise ignore them.
    '''
    try:
        start_capture(event)
    except Exception:
        logger.error('Recording failed')
        logger.error(traceback.format_exc())
        # Update state
        recording_state(event.uid, 'capture_error')
        update_event_status(event, Status.FAILED_RECORDING)
        set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)

def recording_command(event):
    '''Run the actual command to record the a/v material.
    '''
    conf = config('capture')
    # Prepare command line
    cmd = conf['command']
    cmd = cmd.replace('{{time}}', str(event.remaining_duration(timestamp())))
    cmd = cmd.replace('{{dir}}', event.directory())
    cmd = cmd.replace('{{name}}', event.name())
    cmd = cmd.replace('{{previewdir}}', conf['preview_dir'])

    # Signal configuration
    sigterm_time = conf['sigterm_time']
    sigkill_time = conf['sigkill_time']
    sigcustom_time = conf['sigcustom_time']
    sigcustom_time = 0 if sigcustom_time < 0 else event.end + sigcustom_time
    sigterm_time = 0 if sigterm_time < 0 else event.end + sigterm_time
    sigkill_time = 0 if sigkill_time < 0 else event.end + sigkill_time

    # Launch capture command
    logger.info(cmd)
    args = shlex.split(cmd)
    DEVNULL = getattr(subprocess, 'DEVNULL', os.open(os.devnull, os.O_RDWR))
    captureproc = subprocess.Popen(args, stdin=DEVNULL)
    hasattr(subprocess, 'DEVNULL') or os.close(DEVNULL)

    # Set systemd status
    notify.notify('STATUS=Capturing')

    # Check process
    while captureproc.poll() is None:
        notify.notify('WATCHDOG=1')
        if sigcustom_time and timestamp() > sigcustom_time:
            logger.info("Sending custom signal to capture process")
            captureproc.send_signal(conf['sigcustom'])
            sigcustom_time = 0  # send only once
        if sigterm_time and timestamp() > sigterm_time:
            logger.info("Terminating capture process")
            captureproc.terminate()
            sigterm_time = 0  # send only once
        elif sigkill_time and timestamp() > sigkill_time:
            logger.warning("Killing capture process")
            captureproc.kill()
            sigkill_time = 0  # send only once
        time.sleep(0.1)

    # Remove preview files:
    for preview in conf['preview']:
        try:
            os.remove(preview.replace('{{previewdir}}', conf['preview_dir']))
        except OSError:
            logger.warning('Could not remove preview files')
            logger.warning(traceback.format_exc())

    # Check process for errors
    exitcode = config()['capture']['exit_code']
    if captureproc.poll() > 0 and captureproc.returncode != exitcode:
        raise RuntimeError('Recording failed (%i)' % captureproc.returncode)

    # Reset systemd status
    notify.notify('STATUS=Waiting')

    # Return [(flavor,path),…]
    files = (f.replace('{{dir}}', event.directory()) for f in conf['files'])
    files = (f.replace('{{name}}', event.name()) for f in files)
    return list(zip(conf['flavors'], files))

def control_loop():
    '''Main loop of the capture agent, retrieving and checking the schedule as
    well as starting the capture process if necessry.
    '''
    set_service_status(Service.CAPTURE, ServiceStatus.IDLE)
    notify.notify('READY=1')
    notify.notify('STATUS=Waiting')
    while not terminate():
        notify.notify('WATCHDOG=1')
        # Get next recording
        event = get_session().query(UpcomingEvent)\
                             .filter(UpcomingEvent.start <= timestamp())\
                             .filter(UpcomingEvent.end > timestamp())\
                             .first()
        if event:
            safe_start_capture(event)
        time.sleep(1.0)
    logger.info('Shutting down capture service')
    set_service_status(Service.CAPTURE, ServiceStatus.STOPPED)

def run():
    '''Start the capture agent.
    '''
    signal.signal(signal.SIGTERM, sigterm_handler)
    configure_service('capture.admin')
    control_loop()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Example fragment view.
"""

from __future__ import absolute_import, unicode_literals

from web_fragments.fragment import Fragment
from web_fragments.test_utils import TEST_CSS, TEST_HTML, TEST_JS
from web_fragments.views import FragmentView

EXAMPLE_FRAGMENT_VIEW_NAME = 'example_fragment_view'

class ExampleFragmentView(FragmentView):
    """
    Simple fragment view for testing.
    """

    def render_to_fragment(self, request, **kwargs):
        """
        Returns a simple fragment
        """
        fragment = Fragment(TEST_HTML)
        fragment.add_javascript(TEST_JS)
        fragment.add_css(TEST_CSS)
        return fragment


"""
Python representation of a web fragment.
"""

from __future__ import absolute_import, division, print_function, unicode_literals

from collections import namedtuple

import six

FragmentResource = namedtuple("FragmentResource", "kind, data, mimetype, placement")  # pylint: disable=C0103

JS_API_VERSION = 1

class Fragment(object):
    """
    A fragment of a web page to be included on another page.

    A fragment consists of HTML for the body of the page, and a series of
    resources needed by the body. Resources are specified with a MIME type
    (such as "application/javascript" or "text/css") that determines how they
    are inserted into the page.  The resource is provided either as literal
    text, or as a URL.  Text will be included on the page, wrapped
    appropriately for the MIME type.  URLs will be used as-is on the page.

    Resources are only inserted into the page once, even if many Fragments
    in the page ask for them.  Determining duplicates is done by simple text
    matching.
    """
    def __init__(self, content=None):
        #: The html content for this Fragment
        self.content = u""

        self._resources = []
        self.js_init_fn = None
        self.js_init_version = None
        self.json_init_args = None

        if content is not None:
            self.add_content(content)

    @property
    def resources(self):
        """
        Returns list of unique `FragmentResource`s by order of first appearance.
        """
        seen = set()
        # seen.add always returns None, so 'not seen.add(x)' is always True,
        # but will only be called if the value is not already in seen (because
        # 'and' short-circuits)
        return [x for x in self._resources if x not in seen and not seen.add(x)]

    def to_dict(self):
        """
        Returns the fragment in a dictionary representation.
        """
        return {
            'content': self.content,
            'resources': [r._asdict() for r in self.resources],  # pylint: disable=W0212
            'js_init_fn': self.js_init_fn,
            'js_init_version': self.js_init_version,
            'json_init_args': self.json_init_args
        }

    @classmethod
    def from_dict(cls, pods):
        """
        Returns a new Fragment from a dictionary representation.
        """
        frag = cls()
        frag.content = pods['content']
        frag._resources = [FragmentResource(**d) for d in pods['resources']]  # pylint: disable=protected-access
        frag.js_init_fn = pods['js_init_fn']
        frag.js_init_version = pods['js_init_version']
        frag.json_init_args = pods['json_init_args']
        return frag

    def add_content(self, content):
        """
        Add content to this fragment.

        `content` is a Unicode string, HTML to append to the body of the
        fragment.  It must not contain a ``<body>`` tag, or otherwise assume
        that it is the only content on the page.
        """
        assert isinstance(content, six.text_type)
        self.content += content

    def _default_placement(self, mimetype):
        """
        Decide where a resource will go, if the user didn't say.
        """
        if mimetype == 'application/javascript':
            return 'foot'
        return 'head'

    def add_resource(self, text, mimetype, placement=None):
        """
        Add a resource needed by this Fragment.

        Other helpers, such as :func:`add_css` or :func:`add_javascript` are
        more convenient for those common types of resource.

        `text`: the actual text of this resource, as a unicode string.

        `mimetype`: the MIME type of the resource.

        `placement`: where on the page the resource should be placed:

            None: let the Fragment choose based on the MIME type.

            "head": put this resource in the ``<head>`` of the page.

            "foot": put this resource at the end of the ``<body>`` of the
            page.

        """
        if not placement:
            placement = self._default_placement(mimetype)
        res = FragmentResource('text', text, mimetype, placement)
        self._resources.append(res)

    def add_resource_url(self, url, mimetype, placement=None):
        """
        Add a resource by URL needed by this Fragment.

        Other helpers, such as :func:`add_css_url` or
        :func:`add_javascript_url` are more convenent for those common types of
        resource.

        `url`: the URL to the resource.

        Other parameters are as defined for :func:`add_resource`.
        """
        if not placement:
            placement = self._default_placement(mimetype)
        self._resources.append(FragmentResource('url', url, mimetype, placement))

    def add_css(self, text):
        """
        Add literal CSS to the Fragment.
        """
        self.add_resource(text, 'text/css')

    def add_css_url(self, url):
        """
        Add a CSS URL to the Fragment.
        """
        self.add_resource_url(url, 'text/css')

    def add_javascript(self, text):
        """
        Add literal Javascript to the Fragment.
        """
        self.add_resource(text, 'application/javascript')

    def add_javascript_url(self, url):
        """
        Add a Javascript URL to the Fragment.
        """
        self.add_resource_url(url, 'application/javascript')

    def add_fragment_resources(self, fragment):
        """
        Add all the resources from a single fragment to my resources.

        This is used to aggregate resources from another fragment that
        should be considered part of the current fragment.

        The content from the Fragment is ignored.  The caller must collect
        together the content into this Fragment's content.
        """
        self._resources.extend(fragment.resources)

    def add_resources(self, fragments):
        """
        Add all the resources from `fragments` to my resources.

        This is used to aggregate resources from a sequence of fragments that
        should be considered part of the current fragment.

        The content from the Fragments is ignored.  The caller must collect
        together the content into this Fragment's content.
        """
        for fragment in fragments:
            self.add_fragment_resources(fragment)

    def initialize_js(self, js_func, json_args=None):
        """
        Register a Javascript function to initialize the Javascript resources.

        `js_func` is the name of a Javascript function defined by one of the
        Javascript resources.  As part of setting up the browser's runtime
        environment, the function will be invoked, passing a runtime object
        and a DOM element.
        """
        self.js_init_fn = js_func
        self.js_init_version = JS_API_VERSION
        if json_args:
            self.json_init_args = json_args

    def body_html(self):
        """
        Get the body HTML for this Fragment.

        Returns a Unicode string, the HTML content for the ``<body>`` section
        of the page.
        """
        return self.content

    def head_html(self):
        """
        Get the head HTML for this Fragment.

        Returns a Unicode string, the HTML content for the ``<head>`` section
        of the page.
        """
        return self.resources_to_html("head")

    def foot_html(self):
        """
        Get the foot HTML for this Fragment.

        Returns a Unicode string, the HTML content for the end of the
        ``<body>`` section of the page.
        """
        return self.resources_to_html("foot")

    def resources_to_html(self, placement):
        """
        Get some resource HTML for this Fragment.

        `placement` is "head" or "foot".

        Returns a unicode string, the HTML for the head or foot of the page.
        """
        # - non url js could be wrapped in an anonymous function
        # - non url css could be rewritten to match the wrapper tag

        return '\n'.join(
            self.resource_to_html(resource)
            for resource in self.resources
            if resource.placement == placement
        )

    @staticmethod
    def resource_to_html(resource):
        """
        Returns `resource` wrapped in the appropriate html tag for it's mimetype.
        """
        if resource.mimetype == "text/css":
            if resource.kind == "text":
                return u"<style type='text/css'>\n%s\n</style>" % resource.data
            elif resource.kind == "url":
                return u"<link rel='stylesheet' href='%s' type='text/css'>" % resource.data
            else:
                raise Exception("Unrecognized resource kind %r" % resource.kind)

        elif resource.mimetype == "application/javascript":
            if resource.kind == "text":
                return u"<script>\n%s\n</script>" % resource.data
            elif resource.kind == "url":
                return u"<script src='%s' type='application/javascript'></script>" % resource.data
            else:
                raise Exception("Unrecognized resource kind %r" % resource.kind)

        elif resource.mimetype == "text/html":
            assert resource.kind == "text"
            return resource.data

        else:
            raise Exception("Unrecognized mimetype %r" % resource.mimetype)


"""
Django view implementation of web fragments.
"""

from abc import ABCMeta, abstractmethod

from django.http import HttpResponse, JsonResponse
from django.template.loader import get_template
from django.views.generic import View

WEB_FRAGMENT_RESPONSE_TYPE = 'application/web-fragment'
STANDALONE_TEMPLATE_NAME = 'web_fragments/standalone_fragment.html'

class FragmentView(View):
    """
    Base class for Django web fragment views.
    """
    __metaclass__ = ABCMeta

    def get(self, request, *args, **kwargs):
        """
        Render a fragment to HTML or return JSON describing it, based on the request.
        """
        fragment = self.render_to_fragment(request, **kwargs)
        response_format = request.GET.get('format') or request.POST.get('format') or 'html'
        if response_format == 'json' or WEB_FRAGMENT_RESPONSE_TYPE in request.META.get('HTTP_ACCEPT', 'text/html'):
            return JsonResponse(fragment.to_dict())
        else:
            return self.render_standalone_response(request, fragment, **kwargs)

    def render_standalone_response(self, request, fragment, **kwargs):  # pylint: disable=unused-argument
        """
        Renders a standalone page as a response for the specified fragment.
        """
        if fragment is None:
            return HttpResponse(status=204)

        html = self.render_to_standalone_html(request, fragment, **kwargs)
        return HttpResponse(html)

    def render_to_standalone_html(self, request, fragment, **kwargs):  # pylint: disable=unused-argument
        """
        Render the specified fragment to HTML for a standalone page.
        """
        template = get_template(STANDALONE_TEMPLATE_NAME)
        context = {
            'head_html': fragment.head_html(),
            'body_html': fragment.body_html(),
            'foot_html': fragment.foot_html(),
        }
        return template.render(context)

    @abstractmethod
    def render_to_fragment(self, request, **kwargs):  # pylint: disable=unused-argument
        """
        Render this view to a fragment.
        """
        raise NotImplementedError()


# encoding: utf-8
from __future__ import print_function

import math

import numpy as np
import pandas as pd

"""
We compare the implementaion of FDR in pyprophet to the one from Johan Telemann.

"""

p_values = np.loadtxt("p_values.txt")
p_values = np.sort((p_values))[::-1]

from pyprophet.stats import get_error_table_from_pvalues_new

def calc(pvalues, lamb):
    """ meaning pvalues presorted i descending order"""

    m = len(pvalues)
    pi0 = (pvalues > lamb).sum() / ((1 - lamb)*m)

    pFDR = np.ones(m)
    print("pFDR    y        Pr     fastPow")
    for i in range(m):
        y = pvalues[i]
        Pr = max(1, m - i) / float(m)
        pFDR[i] = (pi0 * y) / (Pr * (1 - math.pow(1-y, m)))
        print(i, pFDR[i], y, Pr, 1.0 - math.pow(1-y, m))

    num_null = pi0*m
    num_alt = m - num_null
    num_negs = np.array(range(m))
    num_pos = m - num_negs
    pp = num_pos / float(m)

    qvalues = np.ones(m)
    qvalues[0] = pFDR[0]
    for i in range(m-1):
        qvalues[i+1] = min(qvalues[i], pFDR[i+1])

    sens = ((1.0 - qvalues) * num_pos) / num_alt
    sens[sens > 1.0] = 1.0

    df = pd.DataFrame(dict(
        pvalue=pvalues,
        qvalue=qvalues,
        FDR=pFDR,
        percentile_positive=pp,
        sens=sens
    ))

    df["svalue"] = df.sens[::-1].cummax()[::-1]

    return df, num_null, m

errstat = get_error_table_from_pvalues_new(p_values, 0.4, True)
fdr_pyprophet = errstat.df["FDR"]
df, __, __ = calc(p_values, 0.4)
fdr_storey = df["FDR"]
fdrs = pd.DataFrame(dict(fdr_pp=fdr_pyprophet, fdr_storey=fdr_storey))
print(fdrs[:34])
print(fdrs[:])


from __future__ import division

import pandas as pd
import numpy as np
import multiprocessing
import sys
import time
import click
import operator

from .stats import (lookup_values_from_error_table, error_statistics,
                   mean_and_std_dev, final_err_table, summary_err_table,
                   posterior_chromatogram_hypotheses_fast)
from .data_handling import (prepare_data_table, Experiment)
from .classifiers import (LDALearner, XGBLearner)
from .semi_supervised import (AbstractSemiSupervisedLearner, StandardSemiSupervisedLearner)
from collections import namedtuple
from contextlib import contextmanager

try:
    profile
except NameError:
    def profile(fun):
        return fun

@contextmanager
def timer(name=""):
    start_at = time.time()

    yield

    needed = time.time() - start_at
    hours = int(needed / 3600)
    needed -= hours * 3600

    minutes = int(needed / 60)
    needed -= minutes * 60

    if name:
        click.echo("Info: Time needed for %s: %02d:%02d:%.1f" % (name, hours, minutes, needed))
    else:
        click.echo("Info: Time needed: %02d:%02d:%.1f" % (hours, minutes, needed))

Result = namedtuple("Result", "summary_statistics final_statistics scored_tables")

def unwrap_self_for_multiprocessing(arg):
    """ You can not call methods with multiprocessing, but free functions,
        If you want to call  inst.method(arg0, arg1),

            unwrap_self_for_multiprocessing(inst, "method", (arg0, arg1))

        does the trick.
    """
    (inst, method_name, args) = arg
    return getattr(inst, method_name)(*args)

@profile
def calculate_params_for_d_score(classifier, experiment):
    score = classifier.score(experiment, True)
    experiment.set_and_rerank("classifier_score", score)

    td_scores = experiment.get_top_decoy_peaks()["classifier_score"]

    mu, nu = mean_and_std_dev(td_scores)
    return mu, nu

class Scorer(object):

    def __init__(self, classifier, score_columns, experiment, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, tric_chromprob):

        self.classifier = classifier
        self.score_columns = score_columns
        self.mu, self.nu = calculate_params_for_d_score(classifier, experiment)
        final_score = classifier.score(experiment, True)
        experiment["r_score"] = final_score
        experiment["d_score"] = (final_score - self.mu) / self.nu

        self.group_id = group_id
        self.parametric = parametric
        self.pfdr = pfdr
        self.pi0_lambda = pi0_lambda
        self.pi0_method = pi0_method
        self.pi0_smooth_df = pi0_smooth_df
        self.pi0_smooth_log_pi0 = pi0_smooth_log_pi0
        self.lfdr_truncate = lfdr_truncate
        self.lfdr_monotone = lfdr_monotone
        self.lfdr_transformation = lfdr_transformation
        self.lfdr_adj = lfdr_adj
        self.lfdr_eps = lfdr_eps
        self.tric_chromprob = tric_chromprob

        target_scores = experiment.get_top_target_peaks()["d_score"]
        decoy_scores = experiment.get_top_decoy_peaks()["d_score"]

        self.error_stat, self.pi0 = error_statistics(target_scores,
                                                     decoy_scores,
                                                     self.parametric,
                                                     self.pfdr,
                                                     self.pi0_lambda,
                                                     self.pi0_method,
                                                     self.pi0_smooth_df,
                                                     self.pi0_smooth_log_pi0,
                                                     True, # compute_lfdr
                                                     self.lfdr_truncate,
                                                     self.lfdr_monotone,
                                                     self.lfdr_transformation,
                                                     self.lfdr_adj,
                                                     self.lfdr_eps)

        self.number_target_pg = len(experiment.df[experiment.df.is_decoy.eq(False)])
        self.number_target_peaks = len(experiment.get_top_target_peaks().df)
        self.dvals = experiment.df.loc[(experiment.df.is_decoy.eq(True)), "d_score"]
        self.target_scores = experiment.get_top_target_peaks().df["d_score"]
        self.decoy_scores = experiment.get_top_decoy_peaks().df["d_score"]

    def score(self, table):

        prepared_table, __ = prepare_data_table(table, tg_id_name=self.group_id, score_columns=self.score_columns)
        texp = Experiment(prepared_table)
        score = self.classifier.score(texp, True)
        texp["r_score"] = score
        texp["d_score"] = (score - self.mu) / self.nu

        p_values, s_values, peps, q_values = lookup_values_from_error_table(texp["d_score"].values,
                                                                    self.error_stat)

        texp["pep"] = peps
        texp["q_value"] = q_values
        texp["s_value"] = s_values
        texp["p_value"] = p_values
        click.echo("Info: Mean qvalue = %e, std_dev qvalue = %e" % (np.mean(q_values),
                                                                  np.std(q_values, ddof=1)))
        click.echo("Info: Mean svalue = %e, std_dev svalue = %e" % (np.mean(s_values),
                                                                  np.std(s_values, ddof=1)))
        texp.add_peak_group_rank()

        df = table.join(texp[["r_score", "d_score", "p_value", "q_value", "pep", "peak_group_rank"]])

        if self.tric_chromprob:
            df = self.add_chromatogram_probabilities(df, texp)

        return df

    def add_chromatogram_probabilities(self, scored_table, texp):
        allhypothesis, h0 = posterior_chromatogram_hypotheses_fast(texp, self.pi0['pi0'])
        texp.df["h_score"] = allhypothesis
        texp.df["h0_score"] = h0
        scored_table = scored_table.join(texp[["h_score", "h0_score"]])

        return scored_table

    def get_error_stats(self):
        return final_err_table(self.error_stat), summary_err_table(self.error_stat)

    def minimal_error_stat(self):
        minimal_err_stat = ErrorStatistics(self.error_stat.df.loc[:, ["svalue", "qvalue", "pvalue", "pep", "cutoff"]],
                                           self.error_stat.num_null, self.error_stat.num_total)
        return minimal_err_stat

    def __getstate__(self):
        """when pickling"""
        data = vars(self)
        data["error_stat"] = self.minimal_error_stat()
        return data

    def __setstate__(self, data):
        """when unpickling"""
        self.__dict__.update(data)

class HolyGostQuery(object):

    """ HolyGhostQuery assembles the unsupervised methods.
        See below how PyProphet parameterises this class.
    """

    def __init__(self, semi_supervised_learner, classifier, ss_num_iter, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, tric_chromprob, threads, test):
        assert isinstance(semi_supervised_learner,
                          AbstractSemiSupervisedLearner)
        self.semi_supervised_learner = semi_supervised_learner
        self.classifier = classifier
        self.ss_num_iter = ss_num_iter
        self.group_id = group_id
        self.parametric = parametric
        self.pfdr = pfdr
        self.pi0_lambda = pi0_lambda
        self.pi0_method = pi0_method
        self.pi0_smooth_df = pi0_smooth_df
        self.pi0_smooth_log_pi0 = pi0_smooth_log_pi0
        self.lfdr_truncate = lfdr_truncate
        self.lfdr_monotone = lfdr_monotone
        self.lfdr_transformation = lfdr_transformation
        self.lfdr_adj = lfdr_adj
        self.lfdr_eps = lfdr_eps
        self.tric_chromprob = tric_chromprob
        self.threads = threads
        self.test = test

    def _setup_experiment(self, table):
        prepared_table, score_columns = prepare_data_table(table, tg_id_name=self.group_id)
        experiment = Experiment(prepared_table)
        experiment.log_summary()
        return experiment, score_columns

    def apply_weights(self, table, loaded_weights):
        with timer():
            click.echo("Info: Applying weights.")
            result, scorer, trained_weights = self._apply_weights(table, loaded_weights)
            click.echo("Info: Finished processing of input data.")
        return result, scorer, trained_weights

    def _apply_weights(self, table, loaded_weights):

        experiment, score_columns = self._setup_experiment(table)

        if self.classifier == "LDA":
            if np.all(score_columns == loaded_weights['score'].values):
                weights = loaded_weights['weight'].values
            else:
                raise click.ClickException("Scores in weights file do not match data.")
        elif self.classifier == "XGBoost":
            weights = loaded_weights

        final_classifier = self._apply_weights_on_exp(experiment, weights)

        return self._build_result(table, final_classifier, score_columns, experiment)

    def _apply_weights_on_exp(self, experiment, loaded_weights):

        learner = self.semi_supervised_learner

        click.echo("Info: Start application of pretrained weights.")
        clf_scores = learner.score(experiment, loaded_weights)
        experiment.set_and_rerank("classifier_score", clf_scores)

        click.echo("Info: Finished pretrained scoring.")

        if self.classifier == "LDA":
            ws = [loaded_weights.flatten()]
            final_classifier = self.semi_supervised_learner.averaged_learner(ws)
        elif self.classifier == "XGBoost":
            final_classifier = self.semi_supervised_learner.set_learner(loaded_weights)

        return final_classifier

    @profile
    def learn_and_apply(self, table):
        with timer():

            click.echo("Info: Learn and apply classifier from input data.")
            result, scorer, trained_weights = self._learn_and_apply(table)
            click.echo("Info: Processing input data finished.")

        return result, scorer, trained_weights

    def _learn_and_apply(self, table):

        experiment, score_columns = self._setup_experiment(table)
        final_classifier = self._learn(experiment)

        return self._build_result(table, final_classifier, score_columns, experiment)

    def _learn(self, experiment):
        if self.test:  # for reliable results
            experiment.df.sort_values("tg_id", ascending=True, inplace=True)

        learner = self.semi_supervised_learner

        ws = [] # weights/models
        ttt = [] # top test targets
        ttd = [] # top test decoys

        neval = self.ss_num_iter

        click.echo("Info: Semi-supervised learning of weights:")
        click.echo("Info: Start learning on %d folds using %d processes." % (neval, self.threads))

        if self.threads == 1:
            for k in range(neval):
                (ttt_scores, ttd_scores, w) = learner.learn_randomized(experiment)
                ttt.append(ttt_scores)
                ttd.append(ttd_scores)
                ws.append(w)
        else:
            pool = multiprocessing.Pool(processes=self.threads)
            while neval:
                remaining = max(0, neval - self.threads)
                todo = neval - remaining
                neval -= todo
                args = ((learner, "learn_randomized", (experiment, )), ) * todo
                res = pool.map(unwrap_self_for_multiprocessing, args)
                ttt_scores = [r[0] for r in res]
                ttd_scores = [r[1] for r in res]
                ttt.extend(ttt_scores)
                ttd.extend(ttd_scores)
                ws.extend([r[2] for r in res])
        click.echo("Info: Finished learning.")

        if self.classifier == "LDA":
            final_classifier = self.semi_supervised_learner.averaged_learner(ws)
        elif self.classifier == "XGBoost":
            # Generate average scores over all folds
            ttt_avg = pd.concat(ttt, axis=1).mean(axis=1)
            ttd_avg = pd.concat(ttd, axis=1).mean(axis=1)
            integrated_scores = pd.concat([ttt_avg, ttd_avg], axis=0)

            experiment.set_and_rerank("classifier_score", integrated_scores)

            # Learn final model
            model = learner.learn_final(experiment)
            final_classifier = learner.set_learner(model)

        return final_classifier

    def _build_result(self, table, final_classifier, score_columns, experiment):

        if self.classifier == "LDA":
            weights = final_classifier.get_parameters()
            classifier_table = pd.DataFrame({'score': score_columns, 'weight': weights})
        elif self.classifier == "XGBoost":
            classifier_table = final_classifier.get_parameters()

            mapper = {'f{0}'.format(i): v for i, v in enumerate(score_columns)}
            mapped = {mapper[k]: v for k, v in final_classifier.importance.items()}

            for key, value in reversed(sorted(mapped.items(), key=operator.itemgetter(1))):
                click.echo("Info: Importance of %s: %s" % (key, value))

        scorer = Scorer(final_classifier, score_columns, experiment, self.group_id, self.parametric, self.pfdr, self.pi0_lambda, self.pi0_method, self.pi0_smooth_df, self.pi0_smooth_log_pi0, self.lfdr_truncate, self.lfdr_monotone, self.lfdr_transformation, self.lfdr_adj, self.lfdr_eps, self.tric_chromprob)

        scored_table = scorer.score(table)

        final_statistics, summary_statistics = scorer.get_error_stats()

        result = Result(summary_statistics, final_statistics, scored_table)

        click.echo("Info: Finished scoring and estimation statistics.")
        return result, scorer, classifier_table

@profile
def PyProphet(classifier, xgb_hyperparams, xgb_params, xgb_params_space, xeval_fraction, xeval_num_iter, ss_initial_fdr, ss_iteration_fdr, ss_num_iter, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, tric_chromprob, threads, test):
    if classifier == "LDA":
        return HolyGostQuery(StandardSemiSupervisedLearner(LDALearner(), xeval_fraction, xeval_num_iter, ss_initial_fdr, ss_iteration_fdr, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, test), classifier, ss_num_iter, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, tric_chromprob, threads, test)
    elif classifier == "XGBoost":
        return HolyGostQuery(StandardSemiSupervisedLearner(XGBLearner(xgb_hyperparams, xgb_params, xgb_params_space, threads), xeval_fraction, xeval_num_iter, ss_initial_fdr, ss_iteration_fdr, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, test), classifier, ss_num_iter, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, tric_chromprob, threads, test)
    else:
        raise click.ClickException("Classifier not supported.")


from __future__ import division

import numpy as np
import scipy as sp
import pandas as pd
import math
import scipy.stats
import scipy.special
import multiprocessing
import click

from .optimized import (find_nearest_matches as _find_nearest_matches,
                       count_num_positives, single_chromatogram_hypothesis_fast)
from statsmodels.nonparametric.kde import KDEUnivariate
from collections import namedtuple
# from .config import CONFIG

try:
    profile
except NameError:
    profile = lambda x: x

def _ff(a):
    return _find_nearest_matches(*a)

def find_nearest_matches(x, y):
    # num_processes = CONFIG.get("num_processes")
    # if num_processes > 1:
    #     pool = multiprocessing.Pool(processes=num_processes)
    #     batch_size = int(math.ceil(len(y) / num_processes))
    #     parts = [(x, y[i:i + batch_size]) for i in range(0, len(y),
    #                                                      batch_size)]
    #     res = pool.map(_ff, parts)
    #     res_par = np.hstack(res)
    #     return res_par
    return _find_nearest_matches(x, y)

def to_one_dim_array(values, as_type=None):
    """ Converts list or flattens n-dim array to 1-dim array if possible """

    if isinstance(values, (list, tuple)):
        values = np.array(values, dtype=np.float32)
    elif isinstance(values, pd.Series):
        values = values.values
    values = values.flatten()
    assert values.ndim == 1, "values has wrong dimension"
    if as_type is not None:
        return values.astype(as_type)
    return values

@profile
def lookup_values_from_error_table(scores, err_df):
    """ Find matching q-value for each score in 'scores' """
    ix = find_nearest_matches(np.float32(err_df.cutoff.values), np.float32(scores))
    return err_df.pvalue.iloc[ix].values, err_df.svalue.iloc[ix].values, err_df.pep.iloc[ix].values, err_df.qvalue.iloc[ix].values

def posterior_chromatogram_hypotheses_fast(experiment, prior_chrom_null):
    """ Compute posterior probabilities for each chromatogram

    For each chromatogram (each group_id / peptide precursor), all hypothesis of all peaks
    being correct (and all others false) as well as the h0 (all peaks are
    false) are computed.

    The prior probability that the  are given in the function

    This assumes that the input data is sorted by tg_num_id

        Args:
            experiment(:class:`data_handling.Multipeptide`): the data of one experiment
            prior_chrom_null(float): the prior probability that any precursor
                is absent (all peaks are false)

        Returns:
            tuple(hypothesis, h0): two vectors that contain for each entry in
            the input dataframe the probabilities for the hypothesis that the
            peak is correct and the probability for the h0
    """

    tg_ids = experiment.df.tg_num_id.values
    pp_values = 1-experiment.df["pep"].values

    current_tg_id = tg_ids[0]
    scores = []
    final_result = []
    final_result_h0 = []
    for i in range(tg_ids.shape[0]):

        id_ = tg_ids[i]
        if id_ != current_tg_id:

            # Actual computation for a single transition group (chromatogram)
            prior_pg_true = (1.0 - prior_chrom_null) / len(scores)
            rr = single_chromatogram_hypothesis_fast(
                np.array(scores), prior_chrom_null, prior_pg_true)
            final_result.extend(rr[1:])
            final_result_h0.extend(rr[0] for i in range(len(scores)))

            # Reset for next cycle
            scores = []
            current_tg_id = id_

        scores.append(1.0 - pp_values[i])

    # Last cycle
    prior_pg_true = (1.0 - prior_chrom_null) / len(scores)
    rr = single_chromatogram_hypothesis_fast(np.array(scores), prior_chrom_null, prior_pg_true)
    final_result.extend(rr[1:])
    final_result_h0.extend([rr[0]] * len(scores))

    return final_result, final_result_h0

def mean_and_std_dev(values):
    return np.mean(values), np.std(values, ddof=1)

def pnorm(stat, stat0):
    """ [P(X>pi, mu, sigma) for pi in pvalues] for normal distributed stat with
    expectation value mu and std deviation sigma """

    mu, sigma = mean_and_std_dev(stat0)

    stat = to_one_dim_array(stat, np.float64)
    args = (stat - mu) / sigma
    return 1-(0.5 * (1.0 + scipy.special.erf(args / np.sqrt(2.0))))

def pemp(stat, stat0):
    """ Computes empirical values identically to bioconductor/qvalue empPvals """

    assert len(stat0) > 0
    assert len(stat) > 0

    stat = np.array(stat)
    stat0 = np.array(stat0)

    m = len(stat)
    m0 = len(stat0)

    statc = np.concatenate((stat, stat0))
    v = np.array([True] * m + [False] * m0)
    perm = np.argsort(-statc, kind="mergesort")  # reversed sort, mergesort is stable
    v = v[perm]

    u = np.where(v)[0]
    p = (u - np.arange(m)) / float(m0)

    # ranks can be fractional, we round down to the next integer, ranking returns values starting
    # with 1, not 0:
    ranks = np.floor(scipy.stats.rankdata(-stat)).astype(int) - 1
    p = p[ranks]
    p[p <= 1.0 / m0] = 1.0 / m0

    return p

@profile
def pi0est(p_values, lambda_ = np.arange(0.05,1.0,0.05), pi0_method = "smoother", smooth_df = 3, smooth_log_pi0 = False):
    """ Estimate pi0 according to bioconductor/qvalue """

    # Compare to bioconductor/qvalue reference implementation
    # import rpy2
    # import rpy2.robjects as robjects
    # from rpy2.robjects import pandas2ri
    # pandas2ri.activate()

    # smoothspline=robjects.r('smooth.spline')
    # predict=robjects.r('predict')

    p = np.array(p_values)

    rm_na = np.isfinite(p)
    p = p[rm_na]
    m = len(p)
    ll = 1
    if isinstance(lambda_, np.ndarray ):
        ll = len(lambda_)
        lambda_ = np.sort(lambda_)

    if (min(p) < 0 or max(p) > 1):
        raise click.ClickException("p-values not in valid range [0,1].")
    elif (ll > 1 and ll < 4):
        raise click.ClickException("If lambda_ is not predefined (one value), at least four data points are required.")
    elif (np.min(lambda_) < 0 or np.max(lambda_) >= 1):
        raise click.ClickException("Lambda must be within [0,1)")

    if (ll == 1):
        pi0 = np.mean(p >= lambda_)/(1 - lambda_)
        pi0_lambda = pi0
        pi0 = np.minimum(pi0, 1)
        pi0Smooth = False
    else:
        pi0 = []
        for l in lambda_:
            pi0.append(np.mean(p >= l)/(1 - l))
        pi0_lambda = pi0

        if (pi0_method == "smoother"):
            if smooth_log_pi0:
                pi0 = np.log(pi0)
                spi0 = sp.interpolate.UnivariateSpline(lambda_, pi0, k=smooth_df)
                pi0Smooth = np.exp(spi0(lambda_))
                # spi0 = smoothspline(lambda_, pi0, df = smooth_df) # R reference function
                # pi0Smooth = np.exp(predict(spi0, x = lambda_).rx2('y')) # R reference function
            else:
                spi0 = sp.interpolate.UnivariateSpline(lambda_, pi0, k=smooth_df)
                pi0Smooth = spi0(lambda_)
                # spi0 = smoothspline(lambda_, pi0, df = smooth_df) # R reference function
                # pi0Smooth = predict(spi0, x = lambda_).rx2('y')  # R reference function
            pi0 = np.minimum(pi0Smooth[ll-1],1)
        elif (pi0_method == "bootstrap"):
            minpi0 = np.percentile(pi0,0.1)
            W = []
            for l in lambda_:
                W.append(np.sum(p >= l))
            mse = (np.array(W) / (np.power(m,2) * np.power((1 - lambda_),2))) * (1 - np.array(W) / m) + np.power((pi0 - minpi0),2)
            pi0 = np.minimum(pi0[np.argmin(mse)],1)
            pi0Smooth = False
        else:
            raise click.ClickException("pi0_method must be one of 'smoother' or 'bootstrap'.")
    if (pi0<=0):
        raise click.ClickException("The estimated pi0 <= 0. Check that you have valid p-values or use a different range of lambda.")

    return {'pi0': pi0, 'pi0_lambda': pi0_lambda, 'lambda_': lambda_, 'pi0_smooth': pi0Smooth}

@profile
def qvalue(p_values, pi0, pfdr = False):
    p = np.array(p_values)

    qvals_out = p
    rm_na = np.isfinite(p)
    p = p[rm_na]

    if (min(p) < 0 or max(p) > 1):
        raise click.ClickException("p-values not in valid range [0,1].")
    elif (pi0 < 0 or pi0 > 1):
        raise click.ClickException("pi0 not in valid range [0,1].")

    m = len(p)
    u = np.argsort(p)
    v = scipy.stats.rankdata(p,"max")

    if pfdr:
        qvals = (pi0 * m * p) / (v * (1 - np.power((1 - p), m)))
    else:
        qvals = (pi0 * m * p) / v

    qvals[u[m-1]] = np.minimum(qvals[u[m-1]], 1)
    for i in list(reversed(range(0,m-2,1))):
        qvals[u[i]] = np.minimum(qvals[u[i]], qvals[u[i + 1]])

    qvals_out[rm_na] = qvals
    return qvals_out

def bw_nrd0(x):
    if len(x) < 2:
        raise click.ClickException("bandwidth estimation requires at least two data points.")

    hi = np.std(x, ddof=1)
    q75, q25 = np.percentile(x, [75 ,25])
    iqr = q75 - q25
    lo = min(hi, iqr/1.34)
    lo = lo or hi or abs(x[0]) or 1

    return 0.9 * lo *len(x)**-0.2

@profile
def lfdr(p_values, pi0, trunc = True, monotone = True, transf = "probit", adj = 1.5, eps = np.power(10.0,-8)):
    """ Estimate local FDR / posterior error probability from p-values according to bioconductor/qvalue """
    p = np.array(p_values)

    # Compare to bioconductor/qvalue reference implementation
    # import rpy2
    # import rpy2.robjects as robjects
    # from rpy2.robjects import pandas2ri
    # pandas2ri.activate()

    # density=robjects.r('density')
    # smoothspline=robjects.r('smooth.spline')
    # predict=robjects.r('predict')

    # Check inputs
    lfdr_out = p
    rm_na = np.isfinite(p)
    p = p[rm_na]

    if (min(p) < 0 or max(p) > 1):
        raise click.ClickException("p-values not in valid range [0,1].")
    elif (pi0 < 0 or pi0 > 1):
        raise click.ClickException("pi0 not in valid range [0,1].")

    # Local FDR method for both probit and logit transformations
    if (transf == "probit"):
        p = np.maximum(p, eps)
        p = np.minimum(p, 1-eps)
        x = scipy.stats.norm.ppf(p, loc=0, scale=1)

        # R-like implementation
        bw = bw_nrd0(x)
        myd = KDEUnivariate(x)
        myd.fit(bw=adj*bw, gridsize = 512)
        splinefit = sp.interpolate.splrep(myd.support, myd.density)
        y = sp.interpolate.splev(x, splinefit)
        # myd = density(x, adjust = 1.5) # R reference function
        # mys = smoothspline(x = myd.rx2('x'), y = myd.rx2('y')) # R reference function
        # y = predict(mys, x).rx2('y') # R reference function

        lfdr = pi0 * scipy.stats.norm.pdf(x) / y
    elif (transf == "logit"):
        x = np.log((p + eps) / (1 - p + eps))

        # R-like implementation
        bw = bw_nrd0(x)
        myd = KDEUnivariate(x)
        myd.fit(bw=adj*bw, gridsize = 512)

        splinefit = sp.interpolate.splrep(myd.support, myd.density)
        y = sp.interpolate.splev(x, splinefit)
        # myd = density(x, adjust = 1.5) # R reference function
        # mys = smoothspline(x = myd.rx2('x'), y = myd.rx2('y')) # R reference function
        # y = predict(mys, x).rx2('y') # R reference function

        dx = np.exp(x) / np.power((1 + np.exp(x)),2)
        lfdr = (pi0 * dx) / y
    else:
        raise click.ClickException("Invalid local FDR method.")

    if (trunc):
        lfdr[lfdr > 1] = 1
    if (monotone):
        lfdr = lfdr[p.ravel().argsort()]
        for i in range(1,len(x)):
            if (lfdr[i] < lfdr[i - 1]):
                lfdr[i] = lfdr[i - 1]
        lfdr = lfdr[scipy.stats.rankdata(p,"min")-1]

    lfdr_out[rm_na] = lfdr
    return lfdr_out

@profile
def stat_metrics(p_values, pi0, pfdr):
    num_total = len(p_values)
    num_positives = count_num_positives(p_values)
    num_negatives = num_total - num_positives
    num_null = pi0 * num_total
    tp = num_positives - num_null * p_values
    fp = num_null * p_values
    tn = num_null * (1.0 - p_values)
    fn = num_negatives - num_null * (1.0 - p_values)

    fpr = fp / num_null

    # fdr = fp / num_positives # produces divide by zero warnining
    fdr = np.divide(fp, num_positives, out=np.zeros_like(fp), where=num_positives!=0)

    # fnr = fn / num_negatives # produces divide by zero warnining
    fnr = np.divide(fn, num_negatives, out=np.zeros_like(fn), where=num_negatives!=0)

    if pfdr:
        fdr /= (1.0 - (1.0 - p_values) ** num_total)
        fdr[p_values == 0] = 1.0 / num_total

        fnr /= 1.0 - p_values ** num_total
        fnr[p_values == 0] = 1.0 / num_total

    sens = tp / (num_total - num_null)

    sens[sens < 0.0] = 0.0
    sens[sens > 1.0] = 1.0

    fdr[fdr < 0.0] = 0.0
    fdr[fdr > 1.0] = 1.0
    fdr[num_positives == 0] = 0.0

    fnr[fnr < 0.0] = 0.0
    fnr[fnr > 1.0] = 1.0
    fnr[num_positives == 0] = 0.0

    svalues = pd.Series(sens)[::-1].cummax()[::-1]

    return pd.DataFrame({'tp': tp, 'fp': fp, 'tn': tn, 'fn': fn, 'fpr': fpr, 'fdr': fdr, 'fnr': fnr, 'svalue': svalues})

@profile
def final_err_table(df, num_cut_offs=51):
    """ Create artificial cutoff sample points from given range of cutoff
    values in df, number of sample points is 'num_cut_offs' """

    cutoffs = df.cutoff.values
    min_ = min(cutoffs)
    max_ = max(cutoffs)
    # extend max_ and min_ by 5 % of full range
    margin = (max_ - min_) * 0.05
    sampled_cutoffs = np.linspace(min_ - margin, max_ + margin, num_cut_offs, dtype=np.float32)

    # find best matching row index for each sampled cut off:
    ix = find_nearest_matches(np.float32(df.cutoff.values), sampled_cutoffs)

    # create sub dataframe:
    sampled_df = df.iloc[ix].copy()
    sampled_df.cutoff = sampled_cutoffs
    # remove 'old' index from input df:
    sampled_df.reset_index(inplace=True, drop=True)

    return sampled_df

@profile
def summary_err_table(df, qvalues=[0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5]):
    """ Summary error table for some typical q-values """

    qvalues = to_one_dim_array(qvalues)
    # find best matching fows in df for given qvalues:
    ix = find_nearest_matches(np.float32(df.qvalue.values), qvalues)
    # extract sub table
    df_sub = df.iloc[ix].copy()
    # remove duplicate hits, mark them with None / NAN:
    for i_sub, (i0, i1) in enumerate(zip(ix, ix[1:])):
        if i1 == i0:
            df_sub.iloc[i_sub + 1, :] = None
    # attach q values column
    df_sub.qvalue = qvalues
    # remove old index from original df:
    df_sub.reset_index(inplace=True, drop=True)
    return df_sub[['qvalue','pvalue','svalue','pep','fdr','fnr','fpr','tp','tn','fp','fn','cutoff']]

@profile
def error_statistics(target_scores, decoy_scores, parametric, pfdr, pi0_lambda, pi0_method = "smoother", pi0_smooth_df = 3, pi0_smooth_log_pi0 = False, compute_lfdr = False, lfdr_trunc = True, lfdr_monotone = True, lfdr_transf = "probit", lfdr_adj = 1.5, lfdr_eps = np.power(10.0,-8)):
    """ Takes list of decoy and target scores and creates error statistics for target values """

    target_scores = to_one_dim_array(target_scores)
    target_scores = np.sort(target_scores[~np.isnan(target_scores)])

    decoy_scores = to_one_dim_array(decoy_scores)
    decoy_scores = np.sort(decoy_scores[~np.isnan(decoy_scores)])

    # compute p-values using decoy scores
    if parametric:
        # parametric
        target_pvalues = pnorm(target_scores, decoy_scores)
    else:
        # non-parametric
        target_pvalues = pemp(target_scores, decoy_scores)

    # estimate pi0
    pi0 = pi0est(target_pvalues, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0)

    # compute q-value
    target_qvalues = qvalue(target_pvalues, pi0['pi0'], pfdr)

    # compute other metrics
    metrics = stat_metrics(target_pvalues, pi0['pi0'], pfdr)

    # generate main statistics table
    error_stat = pd.DataFrame({'cutoff': target_scores, 'pvalue': target_pvalues, 'qvalue': target_qvalues, 'svalue': metrics['svalue'], 'tp': metrics['tp'], 'fp': metrics['fp'], 'tn': metrics['tn'], 'fn': metrics['fn'], 'fpr': metrics['fpr'], 'fdr': metrics['fdr'], 'fnr': metrics['fnr']})

    # compute lfdr / PEP
    if compute_lfdr:
        error_stat['pep'] = lfdr(target_pvalues, pi0['pi0'], lfdr_trunc, lfdr_monotone, lfdr_transf, lfdr_adj, lfdr_eps)

    return error_stat, pi0

def find_cutoff(tt_scores, td_scores, cutoff_fdr, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0):
    """ Finds cut off target score for specified false discovery rate fdr """

    error_stat, pi0 = error_statistics(tt_scores, td_scores, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, False)
    if not len(error_stat):
        raise click.ClickException("Too little data for calculating error statistcs.")
    i0 = (error_stat.qvalue - cutoff_fdr).abs().idxmin()
    cutoff = error_stat.iloc[i0]["cutoff"]
    return cutoff


import pandas as pd
import numpy as np
import click
import sys

from .runner import PyProphetLearner, PyProphetWeightApplier
from .ipf import infer_peptidoforms
from .levels_contexts import infer_peptides, infer_proteins, subsample_osw, reduce_osw, merge_osw, backpropagate_oswr
from .export import export_tsv, export_score_plots
from .export_compound import export_compound_tsv, export_score_plots
from .filter import filter_sqmass
from .data_handling import (transform_pi0_lambda, transform_threads, transform_subsample_ratio)
from functools import update_wrapper
import sqlite3

from hyperopt import hp

try:
    profile
except NameError:
    def profile(fun):
        return fun

@click.group(chain=True)
@click.version_option()
def cli():
    """
    PyProphet: Semi-supervised learning and scoring of OpenSWATH results.

    Visit http://openswath.org for usage instructions and help.
    """

# PyProphet semi-supervised learning and scoring
@cli.command()
# # File handling
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='PyProphet input file.')
@click.option('--out', 'outfile', type=click.Path(exists=False), help='PyProphet output file.')
# Semi-supervised learning
@click.option('--classifier', default='LDA', show_default=True, type=click.Choice(['LDA', 'XGBoost']), help='Either a "LDA" or "XGBoost" classifier is used for semi-supervised learning.')
@click.option('--xgb_autotune/--no-xgb_autotune', default=False, show_default=True, help='XGBoost: Autotune hyperparameters.')

@click.option('--apply_weights', type=click.Path(exists=True), help='Apply PyProphet score weights file instead of semi-supervised learning.')
@click.option('--xeval_fraction', default=0.5, show_default=True, type=float, help='Data fraction used for cross-validation of semi-supervised learning step.')
@click.option('--xeval_num_iter', default=10, show_default=True, type=int, help='Number of iterations for cross-validation of semi-supervised learning step.')
@click.option('--ss_initial_fdr', default=0.15, show_default=True, type=float, help='Initial FDR cutoff for best scoring targets.')
@click.option('--ss_iteration_fdr', default=0.05, show_default=True, type=float, help='Iteration FDR cutoff for best scoring targets.')
@click.option('--ss_num_iter', default=10, show_default=True, type=int, help='Number of iterations for semi-supervised learning step.')
@click.option('--ss_main_score', default="var_xcorr_shape", show_default=True, type=str, help='Main score to start semi-supervised-learning.')
# Statistics
@click.option('--group_id', default="group_id", show_default=True, type=str, help='Group identifier for calculation of statistics.')
@click.option('--parametric/--no-parametric', default=False, show_default=True, help='Do parametric estimation of p-values.')
@click.option('--pfdr/--no-pfdr', default=False, show_default=True, help='Compute positive false discovery rate (pFDR) instead of FDR.')
@click.option('--pi0_lambda', default=[0.1,0.5,0.05], show_default=True, type=(float, float, float), help='Use non-parametric estimation of p-values. Either use <START END STEPS>, e.g. 0.1, 1.0, 0.1 or set to fixed value, e.g. 0.4, 0, 0.', callback=transform_pi0_lambda)
@click.option('--pi0_method', default='bootstrap', show_default=True, type=click.Choice(['smoother', 'bootstrap']), help='Either "smoother" or "bootstrap"; the method for automatically choosing tuning parameter in the estimation of pi_0, the proportion of true null hypotheses.')
@click.option('--pi0_smooth_df', default=3, show_default=True, type=int, help='Number of degrees-of-freedom to use when estimating pi_0 with a smoother.')
@click.option('--pi0_smooth_log_pi0/--no-pi0_smooth_log_pi0', default=False, show_default=True, help='If True and pi0_method = "smoother", pi0 will be estimated by applying a smoother to a scatterplot of log(pi0) estimates against the tuning parameter lambda.')
@click.option('--lfdr_truncate/--no-lfdr_truncate', show_default=True, default=True, help='If True, local FDR values >1 are set to 1.')
@click.option('--lfdr_monotone/--no-lfdr_monotone', show_default=True, default=True, help='If True, local FDR values are non-decreasing with increasing p-values.')
@click.option('--lfdr_transformation', default='probit', show_default=True, type=click.Choice(['probit', 'logit']), help='Either a "probit" or "logit" transformation is applied to the p-values so that a local FDR estimate can be formed that does not involve edge effects of the [0,1] interval in which the p-values lie.')
@click.option('--lfdr_adj', default=1.5, show_default=True, type=float, help='Numeric value that is applied as a multiple of the smoothing bandwidth used in the density estimation.')
@click.option('--lfdr_eps', default=np.power(10.0,-8), show_default=True, type=float, help='Numeric value that is threshold for the tails of the empirical p-value distribution.')
# OpenSWATH options
@click.option('--level', default='ms2', show_default=True, type=click.Choice(['ms1', 'ms2', 'ms1ms2', 'transition']), help='Either "ms1", "ms2", "ms1ms2" or "transition"; the data level selected for scoring. "ms1ms2 integrates both MS1- and MS2-level scores and can be used instead of "ms2"-level results."')
# IPF options
@click.option('--ipf_max_peakgroup_rank', default=1, show_default=True, type=int, help='Assess transitions only for candidate peak groups until maximum peak group rank.')
@click.option('--ipf_max_peakgroup_pep', default=0.7, show_default=True, type=float, help='Assess transitions only for candidate peak groups until maximum posterior error probability.')
@click.option('--ipf_max_transition_isotope_overlap', default=0.5, show_default=True, type=float, help='Maximum isotope overlap to consider transitions in IPF.')
@click.option('--ipf_min_transition_sn', default=0, show_default=True, type=float, help='Minimum log signal-to-noise level to consider transitions in IPF. Set -1 to disable this filter.')
# TRIC
@click.option('--tric_chromprob/--no-tric_chromprob', default=False, show_default=True, help='Whether chromatogram probabilities for TRIC should be computed.')
# Processing
@click.option('--threads', default=1, show_default=True, type=int, help='Number of threads used for semi-supervised learning. -1 means all available CPUs.', callback=transform_threads)
@click.option('--test/--no-test', default=False, show_default=True, help='Run in test mode with fixed seed.')
def score(infile, outfile, classifier, xgb_autotune, apply_weights, xeval_fraction, xeval_num_iter, ss_initial_fdr, ss_iteration_fdr, ss_num_iter, ss_main_score, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, level, ipf_max_peakgroup_rank, ipf_max_peakgroup_pep, ipf_max_transition_isotope_overlap, ipf_min_transition_sn, tric_chromprob, threads, test):
    """
    Conduct semi-supervised learning and error-rate estimation for MS1, MS2 and transition-level data.
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    # Prepare XGBoost-specific parameters
    xgb_hyperparams = {'autotune': xgb_autotune, 'autotune_num_rounds': 10, 'num_boost_round': 100, 'early_stopping_rounds': 10, 'test_size': 0.33}

    xgb_params = {'eta': 0.3, 'gamma': 0, 'max_depth': 6, 'min_child_weight': 1, 'subsample': 1, 'colsample_bytree': 1, 'colsample_bylevel': 1, 'colsample_bynode': 1, 'lambda': 1, 'alpha': 0, 'scale_pos_weight': 1, 'silent': 1, 'objective': 'binary:logitraw', 'nthread': 1, 'eval_metric': 'auc'}

    xgb_params_space = {'eta': hp.uniform('eta', 0.0, 0.3), 'gamma': hp.uniform('gamma', 0.0, 0.5), 'max_depth': hp.quniform('max_depth', 2, 8, 1), 'min_child_weight': hp.quniform('min_child_weight', 1, 5, 1), 'subsample': 1, 'colsample_bytree': 1, 'colsample_bylevel': 1, 'colsample_bynode': 1, 'lambda': hp.uniform('lambda', 0.0, 1.0), 'alpha': hp.uniform('alpha', 0.0, 1.0), 'scale_pos_weight': 1.0, 'silent': 1, 'objective': 'binary:logitraw', 'nthread': 1, 'eval_metric': 'auc'}

    if not apply_weights:
        PyProphetLearner(infile, outfile, classifier, xgb_hyperparams, xgb_params, xgb_params_space, xeval_fraction, xeval_num_iter, ss_initial_fdr, ss_iteration_fdr, ss_num_iter, ss_main_score, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, level, ipf_max_peakgroup_rank, ipf_max_peakgroup_pep, ipf_max_transition_isotope_overlap, ipf_min_transition_sn, tric_chromprob, threads, test).run()
    else:
        PyProphetWeightApplier(infile, outfile, classifier, xgb_hyperparams, xgb_params, xgb_params_space, xeval_fraction, xeval_num_iter, ss_initial_fdr, ss_iteration_fdr, ss_num_iter, ss_main_score, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, level, ipf_max_peakgroup_rank, ipf_max_peakgroup_pep, ipf_max_transition_isotope_overlap, ipf_min_transition_sn, tric_chromprob, threads, test, apply_weights).run()

# IPF
@cli.command()
# File handling
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='PyProphet input file.')
@click.option('--out', 'outfile', type=click.Path(exists=False), help='PyProphet output file.')
# IPF parameters
@click.option('--ipf_ms1_scoring/--no-ipf_ms1_scoring', default=True, show_default=True, help='Use MS1 precursor data for IPF.')
@click.option('--ipf_ms2_scoring/--no-ipf_ms2_scoring', default=True, show_default=True, help='Use MS2 precursor data for IPF.')
@click.option('--ipf_h0/--no-ipf_h0', default=True, show_default=True, help='Include possibility that peak groups are not covered by peptidoform space.')
@click.option('--ipf_grouped_fdr/--no-ipf_grouped_fdr', default=False, show_default=True, help='[Experimental] Compute grouped FDR instead of pooled FDR to better support data where peak groups are evaluated to originate from very heterogeneous numbers of peptidoforms.')
@click.option('--ipf_max_precursor_pep', default=0.7, show_default=True, type=float, help='Maximum PEP to consider scored precursors in IPF.')
@click.option('--ipf_max_peakgroup_pep', default=0.7, show_default=True, type=float, help='Maximum PEP to consider scored peak groups in IPF.')
@click.option('--ipf_max_precursor_peakgroup_pep', default=0.4, show_default=True, type=float, help='Maximum BHM layer 1 integrated precursor peakgroup PEP to consider in IPF.')
@click.option('--ipf_max_transition_pep', default=0.6, show_default=True, type=float, help='Maximum PEP to consider scored transitions in IPF.')
def ipf(infile, outfile, ipf_ms1_scoring, ipf_ms2_scoring, ipf_h0, ipf_grouped_fdr, ipf_max_precursor_pep, ipf_max_peakgroup_pep, ipf_max_precursor_peakgroup_pep, ipf_max_transition_pep):
    """
    Infer peptidoforms after scoring of MS1, MS2 and transition-level data.
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    infer_peptidoforms(infile, outfile, ipf_ms1_scoring, ipf_ms2_scoring, ipf_h0, ipf_grouped_fdr, ipf_max_precursor_pep, ipf_max_peakgroup_pep, ipf_max_precursor_peakgroup_pep, ipf_max_transition_pep)

# Peptide-level inference
@cli.command()
# File handling
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='PyProphet input file.')
@click.option('--out', 'outfile', type=click.Path(exists=False), help='PyProphet output file.')
# Context
@click.option('--context', default='run-specific', show_default=True, type=click.Choice(['run-specific', 'experiment-wide', 'global']), help='Context to estimate protein-level FDR control.')
# Statistics
@click.option('--parametric/--no-parametric', default=False, show_default=True, help='Do parametric estimation of p-values.')
@click.option('--pfdr/--no-pfdr', default=False, show_default=True, help='Compute positive false discovery rate (pFDR) instead of FDR.')
@click.option('--pi0_lambda', default=[0.1,0.5,0.05], show_default=True, type=(float, float, float), help='Use non-parametric estimation of p-values. Either use <START END STEPS>, e.g. 0.1, 1.0, 0.1 or set to fixed value, e.g. 0.4, 0, 0.', callback=transform_pi0_lambda)
@click.option('--pi0_method', default='bootstrap', show_default=True, type=click.Choice(['smoother', 'bootstrap']), help='Either "smoother" or "bootstrap"; the method for automatically choosing tuning parameter in the estimation of pi_0, the proportion of true null hypotheses.')
@click.option('--pi0_smooth_df', default=3, show_default=True, type=int, help='Number of degrees-of-freedom to use when estimating pi_0 with a smoother.')
@click.option('--pi0_smooth_log_pi0/--no-pi0_smooth_log_pi0', default=False, show_default=True, help='If True and pi0_method = "smoother", pi0 will be estimated by applying a smoother to a scatterplot of log(pi0) estimates against the tuning parameter lambda.')
@click.option('--lfdr_truncate/--no-lfdr_truncate', show_default=True, default=True, help='If True, local FDR values >1 are set to 1.')
@click.option('--lfdr_monotone/--no-lfdr_monotone', show_default=True, default=True, help='If True, local FDR values are non-decreasing with increasing p-values.')
@click.option('--lfdr_transformation', default='probit', show_default=True, type=click.Choice(['probit', 'logit']), help='Either a "probit" or "logit" transformation is applied to the p-values so that a local FDR estimate can be formed that does not involve edge effects of the [0,1] interval in which the p-values lie.')
@click.option('--lfdr_adj', default=1.5, show_default=True, type=float, help='Numeric value that is applied as a multiple of the smoothing bandwidth used in the density estimation.')
@click.option('--lfdr_eps', default=np.power(10.0,-8), show_default=True, type=float, help='Numeric value that is threshold for the tails of the empirical p-value distribution.')
def peptide(infile, outfile, context, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps):
    """
    Infer peptides and conduct error-rate estimation in different contexts.
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    infer_peptides(infile, outfile, context, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps)

# Protein-level inference
@cli.command()
# File handling
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='PyProphet input file.')
@click.option('--out', 'outfile', type=click.Path(exists=False), help='PyProphet output file.')
# Context
@click.option('--context', default='run-specific', show_default=True, type=click.Choice(['run-specific', 'experiment-wide', 'global']), help='Context to estimate protein-level FDR control.')
# Statistics
@click.option('--parametric/--no-parametric', default=False, show_default=True, help='Do parametric estimation of p-values.')
@click.option('--pfdr/--no-pfdr', default=False, show_default=True, help='Compute positive false discovery rate (pFDR) instead of FDR.')
@click.option('--pi0_lambda', default=[0.1,0.5,0.05], show_default=True, type=(float, float, float), help='Use non-parametric estimation of p-values. Either use <START END STEPS>, e.g. 0.1, 1.0, 0.1 or set to fixed value, e.g. 0.4, 0, 0.', callback=transform_pi0_lambda)
@click.option('--pi0_method', default='bootstrap', show_default=True, type=click.Choice(['smoother', 'bootstrap']), help='Either "smoother" or "bootstrap"; the method for automatically choosing tuning parameter in the estimation of pi_0, the proportion of true null hypotheses.')
@click.option('--pi0_smooth_df', default=3, show_default=True, type=int, help='Number of degrees-of-freedom to use when estimating pi_0 with a smoother.')
@click.option('--pi0_smooth_log_pi0/--no-pi0_smooth_log_pi0', default=False, show_default=True, help='If True and pi0_method = "smoother", pi0 will be estimated by applying a smoother to a scatterplot of log(pi0) estimates against the tuning parameter lambda.')
@click.option('--lfdr_truncate/--no-lfdr_truncate', show_default=True, default=True, help='If True, local FDR values >1 are set to 1.')
@click.option('--lfdr_monotone/--no-lfdr_monotone', show_default=True, default=True, help='If True, local FDR values are non-decreasing with increasing p-values.')
@click.option('--lfdr_transformation', default='probit', show_default=True, type=click.Choice(['probit', 'logit']), help='Either a "probit" or "logit" transformation is applied to the p-values so that a local FDR estimate can be formed that does not involve edge effects of the [0,1] interval in which the p-values lie.')
@click.option('--lfdr_adj', default=1.5, show_default=True, type=float, help='Numeric value that is applied as a multiple of the smoothing bandwidth used in the density estimation.')
@click.option('--lfdr_eps', default=np.power(10.0,-8), show_default=True, type=float, help='Numeric value that is threshold for the tails of the empirical p-value distribution.')
def protein(infile, outfile, context, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps):
    """
    Infer proteins and conduct error-rate estimation in different contexts.
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    infer_proteins(infile, outfile, context, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps)

# Subsample OpenSWATH file to minimum for integrated scoring
@cli.command()
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='OpenSWATH input file.')
@click.option('--out','outfile', type=click.Path(exists=False), help='Subsampled OSWS output file.')
@click.option('--subsample_ratio', default=1, show_default=True, type=float, help='Subsample ratio used per input file.', callback=transform_subsample_ratio)
@click.option('--test/--no-test', default=False, show_default=True, help='Run in test mode with fixed seed.')
def subsample(infile, outfile, subsample_ratio, test):
    """
    Subsample OpenSWATH file to minimum for integrated scoring
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    subsample_osw(infile, outfile, subsample_ratio, test)

# Reduce scored PyProphet file to minimum for global scoring
@cli.command()
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='Scored PyProphet input file.')
@click.option('--out','outfile', type=click.Path(exists=False), help='Reduced OSWR output file.')
def reduce(infile, outfile):
    """
    Reduce scored PyProphet file to minimum for global scoring
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    reduce_osw(infile, outfile)

# Merging of multiple runs
@cli.command()
@click.argument('infiles', nargs=-1, type=click.Path(exists=True))
@click.option('--out','outfile', required=True, type=click.Path(exists=False), help='Merged OSW output file.')
@click.option('--same_run/--no-same_run', default=False, help='Assume input files are from same run (deletes run information).')
@click.option('--template','templatefile', required=True, type=click.Path(exists=False), help='Template OSW file.')
def merge(infiles, outfile, same_run, templatefile):
    """
    Merge multiple OSW files and (for large experiments, it is recommended to subsample first).
    """

    if len(infiles) < 1:
        raise click.ClickException("At least one PyProphet input file needs to be provided.")

    merge_osw(infiles, outfile, templatefile, same_run)

# Backpropagate multi-run peptide and protein scores to single files
@cli.command()
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='Single run PyProphet input file.')
@click.option('--out','outfile', type=click.Path(exists=False), help='Single run (with multi-run scores) PyProphet output file.')
@click.option('--apply_scores', required=True, type=click.Path(exists=True), help='PyProphet multi-run scores file to apply.')
def backpropagate(infile, outfile, apply_scores):
    """
    Backpropagate multi-run peptide and protein scores to single files
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    backpropagate_oswr(infile, outfile, apply_scores)

# Export TSV
@cli.command()
# File handling
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='PyProphet input file.')
@click.option('--out', 'outfile', type=click.Path(exists=False), help='Output TSV/CSV (matrix, legacy_split, legacy_merged) file.')
@click.option('--format', default='legacy_split', show_default=True, type=click.Choice(['matrix', 'legacy_split', 'legacy_merged','score_plots']), help='Export format, either matrix, legacy_split/legacy_merged (mProphet/PyProphet) or score_plots format.')
@click.option('--csv/--no-csv', 'outcsv', default=False, show_default=True, help='Export CSV instead of TSV file.')
# Context
@click.option('--transition_quantification/--no-transition_quantification', default=True, show_default=True, help='[format: legacy] Report aggregated transition-level quantification.')
@click.option('--max_transition_pep', default=0.7, show_default=True, type=float, help='[format: legacy] Maximum PEP to retain scored transitions for quantification (requires transition-level scoring).')
@click.option('--ipf', default='peptidoform', show_default=True, type=click.Choice(['peptidoform','augmented','disable']), help='[format: matrix/legacy] Should IPF results be reported if present? "peptidoform": Report results on peptidoform-level, "augmented": Augment OpenSWATH results with IPF scores, "disable": Ignore IPF results')
@click.option('--ipf_max_peptidoform_pep', default=0.4, show_default=True, type=float, help='[format: matrix/legacy] IPF: Filter results to maximum run-specific peptidoform-level PEP.')
@click.option('--max_rs_peakgroup_qvalue', default=0.05, show_default=True, type=float, help='[format: matrix/legacy] Filter results to maximum run-specific peak group-level q-value.')
@click.option('--peptide/--no-peptide', default=True, show_default=True, help='Append peptide-level error-rate estimates if available.')
@click.option('--max_global_peptide_qvalue', default=0.01, show_default=True, type=float, help='[format: matrix/legacy] Filter results to maximum global peptide-level q-value.')
@click.option('--protein/--no-protein', default=True, show_default=True, help='Append protein-level error-rate estimates if available.')
@click.option('--max_global_protein_qvalue', default=0.01, show_default=True, type=float, help='[format: matrix/legacy] Filter results to maximum global protein-level q-value.')
def export(infile, outfile, format, outcsv, transition_quantification, max_transition_pep, ipf, ipf_max_peptidoform_pep, max_rs_peakgroup_qvalue, peptide, max_global_peptide_qvalue, protein, max_global_protein_qvalue):
    """
    Export TSV/CSV tables
    """
    if format == "score_plots":
        export_score_plots(infile)
    else:
        if outfile is None:
            if outcsv:
                outfile = infile.split(".osw")[0] + ".csv"
            else:
                outfile = infile.split(".osw")[0] + ".tsv"
        else:
            outfile = outfile

        export_tsv(infile, outfile, format, outcsv, transition_quantification, max_transition_pep, ipf, ipf_max_peptidoform_pep, max_rs_peakgroup_qvalue, peptide, max_global_peptide_qvalue, protein, max_global_protein_qvalue)

# Export Compound TSV
@cli.command()
#File handling
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='PyProphet input file.')
@click.option('--out', 'outfile', type=click.Path(exists=False), help='Output TSV/CSV (matrix, legacy_merged) file.')
@click.option('--format', default='legacy_merged', show_default=True, type=click.Choice(['matrix', 'legacy_merged','score_plots']), help='Export format, either matrix, legacy_merged (PyProphet) or score_plots format.')
@click.option('--csv/--no-csv', 'outcsv', default=False, show_default=True, help='Export CSV instead of TSV file.')
# Context
@click.option('--max_rs_peakgroup_qvalue', default=0.05, show_default=True, type=float, help='[format: matrix/legacy] Filter results to maximum run-specific peak group-level q-value.')
def export_compound(infile, outfile, format, outcsv, max_rs_peakgroup_qvalue):
    """
    Export Compound TSV/CSV tables
    """
    if format == "score_plots":
        export_score_plots(infile)
    else:
        if outfile is None:
            if outcsv:
                outfile = infile.split(".osw")[0] + ".csv"
            else:
                outfile = infile.split(".osw")[0] + ".tsv"
        else:
            outfile = outfile

        export_compound_tsv(infile, outfile, format, outcsv, max_rs_peakgroup_qvalue)

# Filter sqMass files
@cli.command()
# File handling
@click.argument('sqmassfiles', nargs=-1, type=click.Path(exists=True))
@click.option('--in', 'infile', required=True, type=click.Path(exists=True), help='PyProphet input file.')
@click.option('--max_precursor_pep', default=0.7, show_default=True, type=float, help='Maximum PEP to retain scored precursors in sqMass.')
@click.option('--max_peakgroup_pep', default=0.7, show_default=True, type=float, help='Maximum PEP to retain scored peak groups in sqMass.')
@click.option('--max_transition_pep', default=0.7, show_default=True, type=float, help='Maximum PEP to retain scored transitions in sqMass.')
def filter(sqmassfiles, infile, max_precursor_pep, max_peakgroup_pep, max_transition_pep):
    """
    Filter sqMass files
    """

    filter_sqmass(sqmassfiles, infile, max_precursor_pep, max_peakgroup_pep, max_transition_pep)