def eplus_version(self):
        """
        if _eplus_version is defined => _eplus_version
        else most recent eplus available version
        """
        # check energy plus is installed
        if len(self.eplus_available_versions) == 0:
            raise RuntimeError("Energy plus is not install, can't use oplus package.")

        # see if version is defined
        if self._eplus_version is not None:
            return self._eplus_version

        # return most recent version
        return sorted(self.eplus_available_versions.keys(), reverse=True)[0]

if _eplus_version is defined = > _eplus_version else most recent eplus available version


def _file_refs(self):
        """
        Defined here so that we can use the class variables, in order to subclass in oplusplus
        """
        if self._prepared_file_refs is None:
            self._prepared_file_refs = {
                FILE_REFS.idf: FileInfo(
                    constructor=lambda path: self._epm_cls.from_idf(path, idd_or_buffer_or_path=self._idd),
                    get_path=lambda: get_input_file_path(self.dir_path, FILE_REFS.idf)
                ),
                FILE_REFS.epw: FileInfo(
                    constructor=lambda path: self._weather_data_cls.from_epw(path),
                    get_path=lambda: get_input_file_path(self.dir_path, FILE_REFS.epw)
                ),
                FILE_REFS.eio: FileInfo(
                    constructor=lambda path: self._eio_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.eio)
                ),
                FILE_REFS.eso: FileInfo(
                    constructor=lambda path: self._standard_output_cls(path),
                    get_path=lambda: get_output_file_path(
                        self.dir_path,
                        FILE_REFS.eso
                    )
                ),
                FILE_REFS.mtr: FileInfo(
                    constructor=lambda path: self._standard_output_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.mtr)
                ),
                FILE_REFS.mtd: FileInfo(
                    constructor=lambda path: self._mtd_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.mtd)
                ),
                FILE_REFS.mdd: FileInfo(
                    constructor=lambda path: open(path).read(),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.mdd)

                ),
                FILE_REFS.err: FileInfo(
                    constructor=lambda path: self._err_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.err)
                ),
                FILE_REFS.summary_table: FileInfo(
                    constructor=lambda path: self._summary_table_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.summary_table)
                )
            }
        return self._prepared_file_refs

def _file_refs ( self ) : if self . _prepared_file_refs is None : self . _prepared_file_refs = { FILE_REFS . idf : FileInfo ( constructor = lambda path : self . _epm_cls . from_idf ( path , idd_or_buffer_or_path = self . _idd ) , get_path = lambda : get_input_file_path ( self . dir_path , FILE_REFS . idf ) ) , FILE_REFS . epw : FileInfo ( constructor = lambda path : self . _weather_data_cls . from_epw ( path ) , get_path = lambda : get_input_file_path ( self . dir_path , FILE_REFS . epw ) ) , FILE_REFS . eio : FileInfo ( constructor = lambda path : self . _eio_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . eio ) ) , FILE_REFS . eso : FileInfo ( constructor = lambda path : self . _standard_output_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . eso ) ) , FILE_REFS . mtr : FileInfo ( constructor = lambda path : self . _standard_output_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . mtr ) ) , FILE_REFS . mtd : FileInfo ( constructor = lambda path : self . _mtd_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . mtd ) ) , FILE_REFS . mdd : FileInfo ( constructor = lambda path : open ( path ) . read ( ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . mdd ) ) , FILE_REFS . err : FileInfo ( constructor = lambda path : self . _err_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . err ) ) , FILE_REFS . summary_table : FileInfo ( constructor = lambda path : self . _summary_table_cls ( path ) , get_path = lambda : get_output_file_path ( self . dir_path , FILE_REFS . summary_table ) ) } return self . _prepared_file_refs

Defined here so that we can use the class variables in order to subclass in oplusplus


def _dev_populate_from_json_data(self, json_data):
        """
        !! Must only be called once, when empty !!
        """

        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. add inert
        #     * data is checked
        #     * old links are unregistered
        #     * record is stored in table (=> pk uniqueness is checked)
        # 2. activate: hooks, links, external files

        # manage comment if any
        comment = json_data.pop("_comment", None)
        if comment is not None:
            self._comment = comment

        # populate external files
        external_files_data = json_data.pop("_external_files", dict())
        self._dev_external_files_manager.populate_from_json_data(external_files_data)

        # manage records
        added_records = []
        for table_ref, json_data_records in json_data.items():
            # find table
            table = getattr(self, table_ref)

            # create record (inert)
            records = table._dev_add_inert(json_data_records)

            # add records (inert)
            added_records.extend(records)

        # activate hooks
        for r in added_records:
            r._dev_activate_hooks()

        # activate links and external files
        for r in added_records:
            r._dev_activate_links()
            r._dev_activate_external_files()

!! Must only be called once when empty !!


def get_external_files(self):
        """
        An external file manages file paths.
        """
        external_files = []
        for table in self._tables.values():
            for r in table:
                external_files.extend([ef for ef in r.get_external_files()])
        return external_files

def get_external_files ( self ) : external_files = [ ] for table in self . _tables . values ( ) : for r in table : external_files . extend ( [ ef for ef in r . get_external_files ( ) ] ) return external_files


def set_defaults(self):
        """
        All fields of Epm with a default value and that are null will be set to their default value.
        """
        for table in self._tables.values():
            for r in table:
                r.set_defaults()

def set_defaults ( self ) : for table in self . _tables . values ( ) : for r in table : r . set_defaults ( )

All fields of Epm with a default value and that are null will be set to their default value .


def prepare_extensible(self):
        """
        This function finishes initialization, must be called once all field descriptors and tag have been filled.
        """
        # see if extensible and store cycle len
        for k in self._tags:
            if "extensible" in k:
                cycle_len = int(k.split(":")[1])
                break
        else:
            # not extensible
            return

        # find cycle start and prepare patterns
        cycle_start = None
        cycle_patterns = []
        for i, field_descriptor in enumerate(self._field_descriptors):
            # quit if finished
            if (cycle_start is not None) and (i >= (cycle_start + cycle_len)):
                break

            # set cycle start if not set yet
            if (cycle_start is None) and ("begin-extensible" in field_descriptor.tags):
                cycle_start = i

            # leave if cycle start not reached yet
            if cycle_start is None:
                continue

            # store pattern
            cycle_patterns.append(field_descriptor.ref.replace("1", r"(\d+)"))
        else:
            raise RuntimeError("cycle start not found")

        # detach unnecessary field descriptors
        self._field_descriptors = self._field_descriptors[:cycle_start + cycle_len]

        # store cycle info
        self.extensible_info = (cycle_start, cycle_len, tuple(cycle_patterns))

        # set field descriptor cycle_start index (for error messages while serialization)
        for i, fd in enumerate(self._field_descriptors[cycle_start:]):
            fd.set_extensible_info(cycle_start, cycle_len, cycle_patterns[i])

This function finishes initialization must be called once all field descriptors and tag have been filled .


def get_extended_name(self, index):
        """
        manages extensible names
        """
        field_descriptor = self.get_field_descriptor(index)
        if self.extensible_info is None:
            return field_descriptor.name
        cycle_start, cycle_len, _ = self.extensible_info
        cycle_num = (index - cycle_start) // cycle_len
        return None if field_descriptor.name is None else field_descriptor.name.replace("1", str(cycle_num))

def get_extended_name ( self , index ) : field_descriptor = self . get_field_descriptor ( index ) if self . extensible_info is None : return field_descriptor . name cycle_start , cycle_len , _ = self . extensible_info cycle_num = ( index - cycle_start ) // cycle_len return None if field_descriptor . name is None else field_descriptor . name . replace ( "1" , str ( cycle_num ) )


def short_refs(self):
        """
        we calculate on the fly to avoid managing registrations and un-registrations

        Returns
        -------
        {ref: short_ref, ...
        """
        naive_short_refs_d = dict()  # naive_short_ref: {refs, ...}
        for ef in self._external_files:
            if ef.naive_short_ref not in naive_short_refs_d:
                naive_short_refs_d[ef.naive_short_ref] = set()
            naive_short_refs_d[ef.naive_short_ref].add(ef.ref)

        short_refs = dict()
        for naive_short_ref, refs in naive_short_refs_d.items():
            if len(refs) == 1:
                short_refs[refs.pop()] = naive_short_ref
                continue
            base, ext = os.path.splitext(naive_short_ref)
            for i, ref in enumerate(sorted(refs)):
                short_refs[ref] = f"{base}-{i}.{ext}"

        return short_refs

def
short_refs
(
self
)
:
naive_short_refs_d
=
dict
(
)
# naive_short_ref: {refs, ...}
for
ef
in
self
.
_external_files
:
if
ef
.
naive_short_ref
not
in
naive_short_refs_d
:
naive_short_refs_d
[
ef
.
naive_short_ref
]
=
set
(
)
naive_short_refs_d
[
ef
.
naive_short_ref
]
.
add
(
ef
.
ref
)
short_refs
=
dict
(
)
for
naive_short_ref
,
refs
in
naive_short_refs_d
.
items
(
)
:
if
len
(
refs
)
==
1
:
short_refs
[
refs
.
pop
(
)
]
=
naive_short_ref
continue
base
,
ext
=
os
.
path
.
splitext
(
naive_short_ref
)
for
i
,
ref
in
enumerate
(
sorted
(
refs
)
)
:
short_refs
[
ref
]
=
f"{base}-{i}.{ext}"
return
short_refs

we calculate on the fly to avoid managing registrations and un - registrations


def get_value(self, column_name_or_i, filter_column_name_or_i, filter_criterion):
        """
        Returns first occurrence of value of filter column matching filter criterion.
        """
        # find column indexes
        column_i = self._get_column_index(column_name_or_i)
        filter_column_i = self._get_column_index(filter_column_name_or_i)

        filter_fct = {
            float: lambda x: float(x) == filter_criterion,
            int: lambda x: int(x) == filter_criterion,
            str: lambda x: x.lower() == filter_criterion.lower()
        }[type(filter_criterion)]

        for row_i, row in enumerate(self._data):
            if filter_fct(row[filter_column_i]):
                break
        else:
            raise ValueError("Filter did not return any values.")

        return self._data[row_i][column_i]

def
get_value
(
self
,
column_name_or_i
,
filter_column_name_or_i
,
filter_criterion
)
:
# find column indexes
column_i
=
self
.
_get_column_index
(
column_name_or_i
)
filter_column_i
=
self
.
_get_column_index
(
filter_column_name_or_i
)
filter_fct
=
{
float
:
lambda
x
:
float
(
x
)
==
filter_criterion
,
int
:
lambda
x
:
int
(
x
)
==
filter_criterion
,
str
:
lambda
x
:
x
.
lower
(
)
==
filter_criterion
.
lower
(
)
}
[
type
(
filter_criterion
)
]
for
row_i
,
row
in
enumerate
(
self
.
_data
)
:
if
filter_fct
(
row
[
filter_column_i
]
)
:
break
else
:
raise
ValueError
(
"Filter did not return any values."
)
return
self
.
_data
[
row_i
]
[
column_i
]

Returns first occurrence of value of filter column matching filter criterion .


def _update_value_inert(self, index, value):
        """
        is only called by _update_inert
        """
        # get field descriptor
        field_descriptor = self._table._dev_descriptor.get_field_descriptor(index)

        # prepare value
        value = field_descriptor.deserialize(value, index)

        # unregister previous link if relevant
        if isinstance(value, Link):
            # de-activate current link if any
            current_link = self._data.get(index)
            if current_link is not None:
                current_link.unregister()

        # unregister previous hook if relevant
        if isinstance(value, RecordHook):
            current_record_hook = self._data.get(index)
            if current_record_hook is not None:
                current_record_hook.unregister()

        # unregister previous external file if relevant
        if isinstance(value, ExternalFile):
            current_external_file = self._data.get(index)
            if current_external_file is not None:
                current_external_file._dev_unregister()

        # if None remove and leave
        if value in (None, NONE_RECORD_HOOK, NONE_LINK, NONE_EXTERNAL_FILE):
            # we don't check required, because this method is called by _update_inert which does the job
            self._dev_set_none_without_unregistering(index, check_not_required=False)
            return

        # if relevant, store current pk to signal table
        old_hook = None
        if index == 0 and not self._table._dev_auto_pk:
            old_hook = self._data.get(0)  # we use get, because record may not have a pk yet if it is being created

        # set value
        self._data[index] = value

        # signal pk update if relevant
        if old_hook is not None:
            self._table._dev_record_pk_was_updated(old_hook.target_value)


def update(self, data=None, **or_data):
        """
        Updates simultaneously all given fields.

        Parameters
        ----------
        data: dictionary containing field lowercase names or index as keys, and field values as values (dict syntax)
        or_data: keyword arguments containing field names as keys (kwargs syntax)
        """

        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. add inert
        #     * data is checked
        #     * old links are unregistered
        #     * record is stored in table (=> pk uniqueness is checked)
        # 2. activate: hooks, links, external files

        data = or_data if data is None else data

        self._update_inert(data)

        self._dev_activate_hooks()
        self._dev_activate_links()
        self._dev_activate_external_files()

Updates simultaneously all given fields .


def set_defaults(self):
        """
        sets all empty fields for which a default value is defined to default value
        """
        defaults = {}
        for i in range(len(self)):
            if i in self._data:
                continue
            default = self.get_field_descriptor(i).tags.get("default", [None])[0]
            if default is not None:
                defaults[i] = default

        self.update(defaults)

def set_defaults ( self ) : defaults = { } for i in range ( len ( self ) ) : if i in self . _data : continue default = self . get_field_descriptor ( i ) . tags . get ( "default" , [ None ] ) [ 0 ] if default is not None : defaults [ i ] = default self . update ( defaults )


def add_fields(self, *args):
        """
        This method only works for extensible fields. It allows to add values without precising their fields' names
        or indexes.

        Parameters
        ----------
        args: field values
        """
        if not self.is_extensible():
            raise TypeError("Can't use add_fields on a non extensible record.")

        # prepare update data
        self_len = len(self)
        data = dict([(self_len + i, args[i]) for i in range(len(args))])

        # update
        self.update(data)

This method only works for extensible fields . It allows to add values without precising their fields names or indexes .


def pop(self, index=None):
        """
        This method only works for extensible fields. It allows to remove a value and shift all other values to fill
        the gap.

        Parameters
        ----------
        index: int, default None
            index of field to remove.

        Returns
        -------
        serialize value of popped field
        """
        # prepare index (will check for extensible)
        index = self._prepare_pop_insert_index(index=index)

        # get extensible info
        cycle_start, cycle_len, patterns = self.get_extensible_info()

        # remove extensible fields
        fields = self.clear_extensible_fields()

        # pop
        serialized_value = fields.pop(index-cycle_start)

        # add remaining
        self.add_fields(*fields)

        return serialized_value

This method only works for extensible fields . It allows to remove a value and shift all other values to fill the gap .


def insert(self, index, value):
        """
        This method only works for extensible fields. It allows to insert a value, and shifts all other following
        values.

        Parameters
        ----------
        index: position of insertion
        value: value to insert
        """
        # prepare index (will check for extensible)
        index = self._prepare_pop_insert_index(index=index)

        # remove extensible fields
        fields = self.clear_extensible_fields()

        # insert
        fields.insert(index, value)

        # add new list
        self.add_fields(*fields)

This method only works for extensible fields . It allows to insert a value and shifts all other following values .


def delete(self):
        """
        Deletes record, and removes it from database.
        """
        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. unregister: links, hooks and external files
        # 3. remove from table without unregistering

        # unregister links
        self._unregister_links()

        # unregister hooks
        self._unregister_hooks()

        # unregister external files
        self._unregister_external_files()

        # tell table to remove without unregistering
        self.get_table()._dev_remove_record_without_unregistering(self)

        # make stale
        self._table = None
        self._data = None

Deletes record and removes it from database .


def register_record_hook(self, hook):
        """
        target record must have been set
        """
        for key in hook.keys:
            if key in self._record_hooks:
                field_descriptor = hook.target_record.get_field_descriptor(hook.target_index)
                raise FieldValidationError(
                    f"Reference key already exists, can't create: {key}. "
                    f"{field_descriptor.get_error_location_message(hook.target_value, hook.target_index)}"
                )
            self._record_hooks[key] = hook


def register_link(self, link):
        """
        source record and index must have been set
        """
        keys = tuple((ref, link.initial_hook_value) for ref in link.hook_references)

        # look for a record hook
        for k in keys:
            if k in self._record_hooks:
                # set link target
                link.set_target(target_record=self._record_hooks[k].target_record)
                break
        else:
            # look for a table hook
            for k in keys:
                if k in self._table_hooks:
                    # set link target
                    link.set_target(target_table=self._table_hooks[k])
                    break
            else:
                field_descriptor = link.source_record.get_field_descriptor(link.source_index)
                raise FieldValidationError(
                    f"No object found with any of given references : {keys}. "
                    f"{field_descriptor.get_error_location_message(link.initial_hook_value)}"
                )

        # store by source
        if link.source_record not in self._links_by_source:
            self._links_by_source[link.source_record] = set()
        self._links_by_source[link.source_record].add(link)

        # store by target
        if link.target not in self._links_by_target:
            self._links_by_target[link.target] = set()
        self._links_by_target[link.target].add(link)


def _create_regex(self, line, intent_name):
        """ Create regex and return. If error occurs returns None. """
        try:
            return re.compile(self._create_intent_pattern(line, intent_name),
                              re.IGNORECASE)
        except sre_constants.error as e:
            LOG.warning('Failed to parse the line "{}" '
                        'for {}'.format(line, intent_name))
            return None

def
_create_regex
(
self
,
line
,
intent_name
)
:
try
:
return
re
.
compile
(
self
.
_create_intent_pattern
(
line
,
intent_name
)
,
re
.
IGNORECASE
)
except
sre_constants
.
error
as
e
:
LOG
.
warning
(
'Failed to parse the line "{}" '
'for {}'
.
format
(
line
,
intent_name
)
)
return

Create regex and return . If error occurs returns None .


def remaining_duration(self, time):
        '''Returns the remaining duration for a recording.
        '''
        return max(0, self.end - max(self.start, time))

def remaining_duration ( self , time ) : return max ( 0 , self . end - max ( self . start , time ) )

Returns the remaining duration for a recording .


def serialize(self):
        '''Serialize this object as dictionary usable for conversion to JSON.

        :return: Dictionary representing this object.
        '''
        return {
            'type': 'event',
            'id': self.uid,
            'attributes': {
                'start': self.start,
                'end': self.end,
                'uid': self.uid,
                'title': self.title,
                'data': self.get_data(),
                'status': Status.str(self.status)
                }
            }

def serialize ( self ) : return { 'type' : 'event' , 'id' : self . uid , 'attributes' : { 'start' : self . start , 'end' : self . end , 'uid' : self . uid , 'title' : self . title , 'data' : self . get_data ( ) , 'status' : Status . str ( self . status ) } }

Serialize this object as dictionary usable for conversion to JSON .


def http_request(url, post_data=None):
    '''Make an HTTP request to a given URL with optional parameters.
    '''
    logger.debug('Requesting URL: %s' % url)
    buf = bio()
    curl = pycurl.Curl()
    curl.setopt(curl.URL, url.encode('ascii', 'ignore'))

    # Disable HTTPS verification methods if insecure is set
    if config()['server']['insecure']:
        curl.setopt(curl.SSL_VERIFYPEER, 0)
        curl.setopt(curl.SSL_VERIFYHOST, 0)

    if config()['server']['certificate']:
        # Make sure verification methods are turned on
        curl.setopt(curl.SSL_VERIFYPEER, 1)
        curl.setopt(curl.SSL_VERIFYHOST, 2)
        # Import your certificates
        curl.setopt(pycurl.CAINFO, config()['server']['certificate'])

    if post_data:
        curl.setopt(curl.HTTPPOST, post_data)
    curl.setopt(curl.WRITEFUNCTION, buf.write)
    curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
    curl.setopt(pycurl.USERPWD, "%s:%s" % (config()['server']['username'],
                                           config()['server']['password']))
    curl.setopt(curl.HTTPHEADER, ['X-Requested-Auth: Digest'])
    curl.setopt(curl.FAILONERROR, True)
    curl.setopt(curl.FOLLOWLOCATION, True)
    curl.perform()
    curl.close()
    result = buf.getvalue()
    buf.close()
    return result

Make an HTTP request to a given URL with optional parameters .


def get_service(service_type):
    '''Get available service endpoints for a given service type from the
    Opencast ServiceRegistry.
    '''
    endpoint = '/services/available.json?serviceType=' + str(service_type)
    url = '%s%s' % (config()['server']['url'], endpoint)
    response = http_request(url).decode('utf-8')
    services = (json.loads(response).get('services') or {}).get('service', [])
    services = ensurelist(services)
    endpoints = [service['host'] + service['path'] for service in services
                 if service['online'] and service['active']]
    for endpoint in endpoints:
        logger.info(u'Endpoint for %s: %s', service_type, endpoint)
    return endpoints

def
get_service
(
service_type
)
:
endpoint
=
'/services/available.json?serviceType='
+
str
(
service_type
)
url
=
'%s%s'
%
(
config
(
)
[
'server'
]
[
'url'
]
,
endpoint
)
response
=
http_request
(
url
)
.
decode
(
'utf-8'
)
services
=
(
json
.
loads
(
response
)
.
get
(
'services'
)
or
{
}
)
.
get
(
'service'
,
[
]
)
services
=
ensurelist
(
services
)
endpoints
=
[
service
[
'host'
]
+
service
[
'path'
]
for
service
in
services
if
service
[
'online'
]
and
service
[
'active'
]
]
for
endpoint
in
endpoints
:
logger
.
info
(
u'Endpoint for %s: %s'
,
service_type
,
endpoint
)
return
endpoints

Get available service endpoints for a given service type from the Opencast ServiceRegistry .


def try_mkdir(directory):
    '''Try to create a directory. Pass without error if it already exists.
    '''
    try:
        os.mkdir(directory)
    except OSError as err:
        if err.errno != errno.EEXIST:
            raise err

def try_mkdir ( directory ) : try : os . mkdir ( directory ) except OSError as err : if err . errno != errno . EEXIST : raise err

Try to create a directory . Pass without error if it already exists .


def configure_service(service):
    '''Get the location of a given service from Opencast and add it to the
    current configuration.
    '''
    while not config().get('service-' + service) and not terminate():
        try:
            config()['service-' + service] = \
                get_service('org.opencastproject.' + service)
        except pycurl.error as e:
            logger.error('Could not get %s endpoint: %s. Retrying in 5s' %
                         (service, e))
            time.sleep(5.0)

Get the location of a given service from Opencast and add it to the current configuration .


def register_ca(status='idle'):
    '''Register this capture agent at the Matterhorn admin server so that it
    shows up in the admin interface.

    :param address: Address of the capture agent web ui
    :param status: Current status of the capture agent
    '''
    # If this is a backup CA we don't tell the Matterhorn core that we are
    # here.  We will just run silently in the background:
    if config()['agent']['backup_mode']:
        return
    params = [('address', config()['ui']['url']), ('state', status)]
    name = urlquote(config()['agent']['name'].encode('utf-8'), safe='')
    url = '%s/agents/%s' % (config()['service-capture.admin'][0], name)
    try:
        response = http_request(url, params).decode('utf-8')
        if response:
            logger.info(response)
    except pycurl.error as e:
        logger.warning('Could not set agent state to %s: %s' % (status, e))

Register this capture agent at the Matterhorn admin server so that it shows up in the admin interface .


def recording_state(recording_id, status):
    '''Send the state of the current recording to the Matterhorn core.

    :param recording_id: ID of the current recording
    :param status: Status of the recording
    '''
    # If this is a backup CA we do not update the recording state since the
    # actual CA does that and we want to interfere.  We will just run silently
    # in the background:
    if config()['agent']['backup_mode']:
        return
    params = [('state', status)]
    url = config()['service-capture.admin'][0]
    url += '/recordings/%s' % recording_id
    try:
        result = http_request(url, params)
        logger.info(result)
    except pycurl.error as e:
        logger.warning('Could not set recording state to %s: %s' % (status, e))

Send the state of the current recording to the Matterhorn core .


def update_event_status(event, status):
    '''Update the status of a particular event in the database.
    '''
    dbs = db.get_session()
    dbs.query(db.RecordedEvent).filter(db.RecordedEvent.start == event.start)\
                               .update({'status': status})
    event.status = status
    dbs.commit()

def update_event_status ( event , status ) : dbs = db . get_session ( ) dbs . query ( db . RecordedEvent ) . filter ( db . RecordedEvent . start == event . start ) . update ( { 'status' : status } ) event . status = status dbs . commit ( )

Update the status of a particular event in the database .


def update_agent_state():
    '''Update the current agent state in opencast.
    '''
    configure_service('capture.admin')
    status = 'idle'

    # Determine reported agent state with priority list
    if get_service_status(db.Service.SCHEDULE) == db.ServiceStatus.STOPPED:
        status = 'offline'
    elif get_service_status(db.Service.CAPTURE) == db.ServiceStatus.BUSY:
        status = 'capturing'
    elif get_service_status(db.Service.INGEST) == db.ServiceStatus.BUSY:
        status = 'uploading'

    register_ca(status=status)

Update the current agent state in opencast .


def configuration_file(cfgfile):
    '''Find the best match for the configuration file.
    '''
    if cfgfile is not None:
        return cfgfile
    # If no file is explicitely specified, probe for the configuration file
    # location.
    cfg = './etc/pyca.conf'
    if not os.path.isfile(cfg):
        return '/etc/pyca.conf'
    return cfg

Find the best match for the configuration file .


def update_configuration(cfgfile=None):
    '''Update configuration from file.

    :param cfgfile: Configuration file to load.
    '''
    configobj.DEFAULT_INTERPOLATION = 'template'
    cfgfile = configuration_file(cfgfile)
    cfg = configobj.ConfigObj(cfgfile, configspec=cfgspec, encoding='utf-8')
    validator = Validator()
    val = cfg.validate(validator)
    if val is not True:
        raise ValueError('Invalid configuration: %s' % val)
    if len(cfg['capture']['files']) != len(cfg['capture']['flavors']):
        raise ValueError('List of files and flavors do not match')
    globals()['__config'] = cfg
    logger_init()
    if cfg['server'].get('url', '').endswith('/'):
        logger.warning('Base URL ends with /. This is most likely a '
                       'configuration error. The URL should contain nothing '
                       'of the service paths.')
    logger.info('Configuration loaded from %s' % cfgfile)
    check()
    return cfg


def check():
    '''Check configuration for sanity.
    '''
    if config('server')['insecure']:
        logger.warning('HTTPS CHECKS ARE TURNED OFF. A SECURE CONNECTION IS '
                       'NOT GUARANTEED')
    if config('server')['certificate']:
        # Ensure certificate exists and is readable
        open(config('server')['certificate'], 'rb').close()
    if config('agent')['backup_mode']:
        logger.info('Agent runs in backup mode. No data will be sent to '
                    'Opencast')


def logger_init():
    '''Initialize logger based on configuration
    '''
    handlers = []
    logconf = config('logging')
    if logconf['syslog']:
        handlers.append(logging.handlers.SysLogHandler(address='/dev/log'))
    if logconf['stderr']:
        handlers.append(logging.StreamHandler(sys.stderr))
    if logconf['file']:
        handlers.append(logging.handlers.WatchedFileHandler(logconf['file']))
    for handler in handlers:
        handler.setFormatter(logging.Formatter(logconf['format']))
        logging.root.addHandler(handler)

    logging.root.setLevel(logconf['level'].upper())
    logger.info('Log level set to %s' % logconf['level'])

def
logger_init
(
)
:
handlers
=
[
]
logconf
=
config
(
'logging'
)
if
logconf
[
'syslog'
]
:
handlers
.
append
(
logging
.
handlers
.
SysLogHandler
(
address
=
'/dev/log'
)
)
if
logconf
[
'stderr'
]
:
handlers
.
append
(
logging
.
StreamHandler
(
sys
.
stderr
)
)
if
logconf
[
'file'
]
:
handlers
.
append
(
logging
.
handlers
.
WatchedFileHandler
(
logconf
[
'file'
]
)
)
for
handler
in
handlers
:
handler
.
setFormatter
(
logging
.
Formatter
(
logconf
[
'format'
]
)
)
logging
.
root
.
addHandler
(
handler
)
logging
.
root
.
setLevel
(
logconf
[
'level'
]
.
upper
(
)
)
logger
.
info
(
'Log level set to %s'
%
logconf
[
'level'
]
)


def home():
    '''Serve the status page of the capture agent.
    '''
    # Get IDs of existing preview images
    preview = config()['capture']['preview']
    previewdir = config()['capture']['preview_dir']
    preview = [p.replace('{{previewdir}}', previewdir) for p in preview]
    preview = zip(preview, range(len(preview)))
    preview = [p[1] for p in preview if os.path.isfile(p[0])]

    # Get limits for recording table
    try:
        limit_upcoming = int(request.args.get('limit_upcoming', 5))
        limit_processed = int(request.args.get('limit_processed', 15))
    except ValueError:
        limit_upcoming = 5
        limit_processed = 15

    db = get_session()
    upcoming_events = db.query(UpcomingEvent)\
                        .order_by(UpcomingEvent.start)\
                        .limit(limit_upcoming)
    recorded_events = db.query(RecordedEvent)\
                        .order_by(RecordedEvent.start.desc())\
                        .limit(limit_processed)
    recording = get_service_status(Service.CAPTURE) == ServiceStatus.BUSY
    uploading = get_service_status(Service.INGEST) == ServiceStatus.BUSY
    processed = db.query(RecordedEvent).count()
    upcoming = db.query(UpcomingEvent).count()

    return render_template('home.html', preview=preview, config=config(),
                           recorded_events=recorded_events,
                           upcoming_events=upcoming_events,
                           recording=recording, uploading=uploading,
                           processed=processed, upcoming=upcoming,
                           limit_upcoming=limit_upcoming,
                           limit_processed=limit_processed,
                           dtfmt=dtfmt)

def
home
(
)
:
# Get IDs of existing preview images
preview
=
config
(
)
[
'capture'
]
[
'preview'
]
previewdir
=
config
(
)
[
'capture'
]
[
'preview_dir'
]
preview
=
[
p
.
replace
(
'{{previewdir}}'
,
previewdir
)
for
p
in
preview
]
preview
=
zip
(
preview
,
range
(
len
(
preview
)
)
)
preview
=
[
p
[
1
]
for
p
in
preview
if
os
.
path
.
isfile
(
p
[
0
]
)
]
# Get limits for recording table
try
:
limit_upcoming
=
int
(
request
.
args
.
get
(
'limit_upcoming'
,
5
)
)
limit_processed
=
int
(
request
.
args
.
get
(
'limit_processed'
,
15
)
)
except
ValueError
:
limit_upcoming
=
5
limit_processed
=
15
db
=
get_session
(
)
upcoming_events
=
db
.
query
(
UpcomingEvent
)
.
order_by
(
UpcomingEvent
.
start
)
.
limit
(
limit_upcoming
)
recorded_events
=
db
.
query
(
RecordedEvent
)
.
order_by
(
RecordedEvent
.
start
.
desc
(
)
)
.
limit
(
limit_processed
)
recording
=
get_service_status
(
Service
.
CAPTURE
)
==
ServiceStatus
.
BUSY
uploading
=
get_service_status
(
Service
.
INGEST
)
==
ServiceStatus
.
BUSY
processed
=
db
.
query
(
RecordedEvent
)
.
count
(
)
upcoming
=
db
.
query
(
UpcomingEvent
)
.
count
(
)
return
render_template
(
'home.html'
,
preview
=
preview
,
config
=
config
(
)
,
recorded_events
=
recorded_events
,
upcoming_events
=
upcoming_events
,
recording
=
recording
,
uploading
=
uploading
,
processed
=
processed
,
upcoming
=
upcoming
,
limit_upcoming
=
limit_upcoming
,
limit_processed
=
limit_processed
,
dtfmt
=
dtfmt
)

Serve the status page of the capture agent .


def serve_image(image_id):
    '''Serve the preview image with the given id
    '''
    try:
        preview_dir = config()['capture']['preview_dir']
        filepath = config()['capture']['preview'][image_id]
        filepath = filepath.replace('{{previewdir}}', preview_dir)
        filepath = os.path.abspath(filepath)
        if os.path.isfile(filepath):
            directory, filename = filepath.rsplit('/', 1)
            return send_from_directory(directory, filename)
    except (IndexError, KeyError):
        pass
    return '', 404

def serve_image ( image_id ) : try : preview_dir = config ( ) [ 'capture' ] [ 'preview_dir' ] filepath = config ( ) [ 'capture' ] [ 'preview' ] [ image_id ] filepath = filepath . replace ( '{{previewdir}}' , preview_dir ) filepath = os . path . abspath ( filepath ) if os . path . isfile ( filepath ) : directory , filename = filepath . rsplit ( '/' , 1 ) return send_from_directory ( directory , filename ) except ( IndexError , KeyError ) : pass return '' , 404


def run_all(*modules):
    '''Start all services.
    '''
    processes = [multiprocessing.Process(target=mod.run) for mod in modules]
    for p in processes:
        p.start()
    for p in processes:
        p.join()

def run_all ( * modules ) : processes = [ multiprocessing . Process ( target = mod . run ) for mod in modules ] for p in processes : p . start ( ) for p in processes : p . join ( )


def parse_ical(vcal):
    '''Parse Opencast schedule iCalendar file and return events as dict
    '''
    vcal = vcal.replace('\r\n ', '').replace('\r\n\r\n', '\r\n')
    vevents = vcal.split('\r\nBEGIN:VEVENT\r\n')
    del(vevents[0])
    events = []
    for vevent in vevents:
        event = {}
        for line in vevent.split('\r\n'):
            line = line.split(':', 1)
            key = line[0].lower()
            if len(line) <= 1 or key == 'end':
                continue
            if key.startswith('dt'):
                event[key] = unix_ts(dateutil.parser.parse(line[1]))
                continue
            if not key.startswith('attach'):
                event[key] = line[1]
                continue
            # finally handle attachments
            event['attach'] = event.get('attach', [])
            attachment = {}
            for x in [x.split('=') for x in line[0].split(';')]:
                if x[0].lower() in ['fmttype', 'x-apple-filename']:
                    attachment[x[0].lower()] = x[1]
            attachment['data'] = b64decode(line[1]).decode('utf-8')
            event['attach'].append(attachment)
        events.append(event)
    return events

def
parse_ical
(
vcal
)
:
vcal
=
vcal
.
replace
(
'\r\n '
,
''
)
.
replace
(
'\r\n\r\n'
,
'\r\n'
)
vevents
=
vcal
.
split
(
'\r\nBEGIN:VEVENT\r\n'
)
del
(
vevents
[
0
]
)
events
=
[
]
for
vevent
in
vevents
:
event
=
{
}
for
line
in
vevent
.
split
(
'\r\n'
)
:
line
=
line
.
split
(
':'
,
1
)
key
=
line
[
0
]
.
lower
(
)
if
len
(
line
)
<=
1
or
key
==
'end'
:
continue
if
key
.
startswith
(
'dt'
)
:
event
[
key
]
=
unix_ts
(
dateutil
.
parser
.
parse
(
line
[
1
]
)
)
continue
if
not
key
.
startswith
(
'attach'
)
:
event
[
key
]
=
line
[
1
]
continue
# finally handle attachments
event
[
'attach'
]
=
event
.
get
(
'attach'
,
[
]
)
attachment
=
{
}
for
x
in
[
x
.
split
(
'='
)
for
x
in
line
[
0
]
.
split
(
';'
)
]
:
if
x
[
0
]
.
lower
(
)
in
[
'fmttype'
,
'x-apple-filename'
]
:
attachment
[
x
[
0
]
.
lower
(
)
]
=
x
[
1
]
attachment
[
'data'
]
=
b64decode
(
line
[
1
]
)
.
decode
(
'utf-8'
)
event
[
'attach'
]
.
append
(
attachment
)
events
.
append
(
event
)
return
events


def get_schedule():
    '''Try to load schedule from the Matterhorn core. Returns a valid schedule
    or None on failure.
    '''
    params = {'agentid': config()['agent']['name'].encode('utf8')}
    lookahead = config()['agent']['cal_lookahead'] * 24 * 60 * 60
    if lookahead:
        params['cutoff'] = str((timestamp() + lookahead) * 1000)
    uri = '%s/calendars?%s' % (config()['service-scheduler'][0],
                               urlencode(params))
    try:
        vcal = http_request(uri)
    except pycurl.error as e:
        logger.error('Could not get schedule: %s' % e)
        return

    try:
        cal = parse_ical(vcal.decode('utf-8'))
    except Exception:
        logger.error('Could not parse ical')
        logger.error(traceback.format_exc())
        return
    db = get_session()
    db.query(UpcomingEvent).delete()
    for event in cal:
        # Ignore events that have already ended
        if event['dtend'] <= timestamp():
            continue
        e = UpcomingEvent()
        e.start = event['dtstart']
        e.end = event['dtend']
        e.uid = event.get('uid')
        e.title = event.get('summary')
        e.set_data(event)
        db.add(e)
    db.commit()

def
get_schedule
(
)
:
params
=
{
'agentid'
:
config
(
)
[
'agent'
]
[
'name'
]
.
encode
(
'utf8'
)
}
lookahead
=
config
(
)
[
'agent'
]
[
'cal_lookahead'
]
*
24
*
60
*
60
if
lookahead
:
params
[
'cutoff'
]
=
str
(
(
timestamp
(
)
+
lookahead
)
*
1000
)
uri
=
'%s/calendars?%s'
%
(
config
(
)
[
'service-scheduler'
]
[
0
]
,
urlencode
(
params
)
)
try
:
vcal
=
http_request
(
uri
)
except
pycurl
.
error
as
e
:
logger
.
error
(
'Could not get schedule: %s'
%
e
)
return
try
:
cal
=
parse_ical
(
vcal
.
decode
(
'utf-8'
)
)
except
Exception
:
logger
.
error
(
'Could not parse ical'
)
logger
.
error
(
traceback
.
format_exc
(
)
)
return
db
=
get_session
(
)
db
.
query
(
UpcomingEvent
)
.
delete
(
)
for
event
in
cal
:
# Ignore events that have already ended
if
event
[
'dtend'
]
<=
timestamp
(
)
:
continue
e
=
UpcomingEvent
(
)
e
.
start
=
event
[
'dtstart'
]
e
.
end
=
event
[
'dtend'
]
e
.
uid
=
event
.
get
(
'uid'
)
e
.
title
=
event
.
get
(
'summary'
)
e
.
set_data
(
event
)
db
.
add
(
e
)
db
.
commit
(
)

Try to load schedule from the Matterhorn core . Returns a valid schedule or None on failure .


def control_loop():
    '''Main loop, retrieving the schedule.
    '''
    set_service_status(Service.SCHEDULE, ServiceStatus.BUSY)
    notify.notify('READY=1')
    while not terminate():
        notify.notify('WATCHDOG=1')
        # Try getting an updated schedule
        get_schedule()
        session = get_session()
        next_event = session.query(UpcomingEvent)\
                            .filter(UpcomingEvent.end > timestamp())\
                            .order_by(UpcomingEvent.start)\
                            .first()
        if next_event:
            logger.info('Next scheduled recording: %s',
                        datetime.fromtimestamp(next_event.start))
            notify.notify('STATUS=Next scheduled recording: %s' %
                          datetime.fromtimestamp(next_event.start))
        else:
            logger.info('No scheduled recording')
            notify.notify('STATUS=No scheduled recording')
        session.close()

        next_update = timestamp() + config()['agent']['update_frequency']
        while not terminate() and timestamp() < next_update:
            time.sleep(0.1)

    logger.info('Shutting down schedule service')
    set_service_status(Service.SCHEDULE, ServiceStatus.STOPPED)

def
control_loop
(
)
:
set_service_status
(
Service
.
SCHEDULE
,
ServiceStatus
.
BUSY
)
notify
.
notify
(
'READY=1'
)
while
not
terminate
(
)
:
notify
.
notify
(
'WATCHDOG=1'
)
# Try getting an updated schedule
get_schedule
(
)
session
=
get_session
(
)
next_event
=
session
.
query
(
UpcomingEvent
)
.
filter
(
UpcomingEvent
.
end
>
timestamp
(
)
)
.
order_by
(
UpcomingEvent
.
start
)
.
first
(
)
if
next_event
:
logger
.
info
(
'Next scheduled recording: %s'
,
datetime
.
fromtimestamp
(
next_event
.
start
)
)
notify
.
notify
(
'STATUS=Next scheduled recording: %s'
%
datetime
.
fromtimestamp
(
next_event
.
start
)
)
else
:
logger
.
info
(
'No scheduled recording'
)
notify
.
notify
(
'STATUS=No scheduled recording'
)
session
.
close
(
)
next_update
=
timestamp
(
)
+
config
(
)
[
'agent'
]
[
'update_frequency'
]
while
not
terminate
(
)
and
timestamp
(
)
<
next_update
:
time
.
sleep
(
0.1
)
logger
.
info
(
'Shutting down schedule service'
)
set_service_status
(
Service
.
SCHEDULE
,
ServiceStatus
.
STOPPED
)


def control_loop():
    '''Main loop, updating the capture agent state.
    '''
    set_service_status(Service.AGENTSTATE, ServiceStatus.BUSY)
    notify.notify('READY=1')
    notify.notify('STATUS=Running')
    while not terminate():
        notify.notify('WATCHDOG=1')
        update_agent_state()

        next_update = timestamp() + config()['agent']['update_frequency']
        while not terminate() and timestamp() < next_update:
            time.sleep(0.1)

    logger.info('Shutting down agentstate service')
    set_service_status(Service.AGENTSTATE, ServiceStatus.STOPPED)

def
control_loop
(
)
:
set_service_status
(
Service
.
AGENTSTATE
,
ServiceStatus
.
BUSY
)
notify
.
notify
(
'READY=1'
)
notify
.
notify
(
'STATUS=Running'
)
while
not
terminate
(
)
:
notify
.
notify
(
'WATCHDOG=1'
)
update_agent_state
(
)
next_update
=
timestamp
(
)
+
config
(
)
[
'agent'
]
[
'update_frequency'
]
while
not
terminate
(
)
and
timestamp
(
)
<
next_update
:
time
.
sleep
(
0.1
)
logger
.
info
(
'Shutting down agentstate service'
)
set_service_status
(
Service
.
AGENTSTATE
,
ServiceStatus
.
STOPPED
)

Main loop updating the capture agent state .


def make_error_response(error, status=500):
    ''' Return a response with a jsonapi error object
    '''
    content = {
        'errors': [{
            'status': status,
            'title': error
        }]
    }
    return make_response(jsonify(content), status)

def make_error_response ( error , status = 500 ) : content = { 'errors' : [ { 'status' : status , 'title' : error } ] } return make_response ( jsonify ( content ) , status )


def make_data_response(data, status=200):
    ''' Return a response with a list of jsonapi data objects
    '''
    content = {'data': ensurelist(data)}
    return make_response(jsonify(content), status)

def make_data_response ( data , status = 200 ) : content = { 'data' : ensurelist ( data ) } return make_response ( jsonify ( content ) , status )


def internal_state():
    '''Serve a json representation of internal agentstate as meta data
    '''
    data = {'services': {
        'capture': ServiceStatus.str(get_service_status(Service.CAPTURE)),
        'ingest': ServiceStatus.str(get_service_status(Service.INGEST)),
        'schedule': ServiceStatus.str(get_service_status(Service.SCHEDULE)),
        'agentstate': ServiceStatus.str(get_service_status(Service.AGENTSTATE))
        }
    }
    return make_response(jsonify({'meta': data}))

def internal_state ( ) : data = { 'services' : { 'capture' : ServiceStatus . str ( get_service_status ( Service . CAPTURE ) ) , 'ingest' : ServiceStatus . str ( get_service_status ( Service . INGEST ) ) , 'schedule' : ServiceStatus . str ( get_service_status ( Service . SCHEDULE ) ) , 'agentstate' : ServiceStatus . str ( get_service_status ( Service . AGENTSTATE ) ) } } return make_response ( jsonify ( { 'meta' : data } ) )


def events():
    '''Serve a JSON representation of events
    '''
    db = get_session()
    upcoming_events = db.query(UpcomingEvent)\
                        .order_by(UpcomingEvent.start)
    recorded_events = db.query(RecordedEvent)\
                        .order_by(RecordedEvent.start.desc())

    result = [event.serialize() for event in upcoming_events]
    result += [event.serialize() for event in recorded_events]
    return make_data_response(result)

def events ( ) : db = get_session ( ) upcoming_events = db . query ( UpcomingEvent ) . order_by ( UpcomingEvent . start ) recorded_events = db . query ( RecordedEvent ) . order_by ( RecordedEvent . start . desc ( ) ) result = [ event . serialize ( ) for event in upcoming_events ] result += [ event . serialize ( ) for event in recorded_events ] return make_data_response ( result )


def event(uid):
    '''Return a specific events JSON
    '''
    db = get_session()
    event = db.query(RecordedEvent).filter(RecordedEvent.uid == uid).first() \
        or db.query(UpcomingEvent).filter(UpcomingEvent.uid == uid).first()

    if event:
        return make_data_response(event.serialize())
    return make_error_response('No event with specified uid', 404)

def
event
(
uid
)
:
db
=
get_session
(
)
event
=
db
.
query
(
RecordedEvent
)
.
filter
(
RecordedEvent
.
uid
==
uid
)
.
first
(
)
or
db
.
query
(
UpcomingEvent
)
.
filter
(
UpcomingEvent
.
uid
==
uid
)
.
first
(
)
if
event
:
return
make_data_response
(
event
.
serialize
(
)
)
return
make_error_response
(
'No event with specified uid'
,
404
)


def delete_event(uid):
    '''Delete a specific event identified by its uid. Note that only recorded
    events can be deleted. Events in the buffer for upcoming events are
    regularly replaced anyway and a manual removal could have unpredictable
    effects.

    Use ?hard=true parameter to delete the recorded files on disk as well.

    Returns 204 if the action was successful.
    Returns 404 if event does not exist
    '''
    logger.info('deleting event %s via api', uid)
    db = get_session()
    events = db.query(RecordedEvent).filter(RecordedEvent.uid == uid)
    if not events.count():
        return make_error_response('No event with specified uid', 404)
    hard_delete = request.args.get('hard', 'false')
    if hard_delete == 'true':
        logger.info('deleting recorded files at %s', events[0].directory())
        shutil.rmtree(events[0].directory())
    events.delete()
    db.commit()
    return make_response('', 204)

def
delete_event
(
uid
)
:
logger
.
info
(
'deleting event %s via api'
,
uid
)
db
=
get_session
(
)
events
=
db
.
query
(
RecordedEvent
)
.
filter
(
RecordedEvent
.
uid
==
uid
)
if
not
events
.
count
(
)
:
return
make_error_response
(
'No event with specified uid'
,
404
)
hard_delete
=
request
.
args
.
get
(
'hard'
,
'false'
)
if
hard_delete
==
'true'
:
logger
.
info
(
'deleting recorded files at %s'
,
events
[
0
]
.
directory
(
)
)
shutil
.
rmtree
(
events
[
0
]
.
directory
(
)
)
events
.
delete
(
)
db
.
commit
(
)
return
make_response
(
''
,
204
)

Delete a specific event identified by its uid . Note that only recorded events can be deleted . Events in the buffer for upcoming events are regularly replaced anyway and a manual removal could have unpredictable effects .


def modify_event(uid):
    '''Modify an event specified by its uid. The modifications for the event
    are expected as JSON with the content type correctly set in the request.

    Note that this method works for recorded events only. Upcoming events part
    of the scheduler cache cannot be modified.
    '''
    try:
        data = request.get_json()['data'][0]
        if data['type'] != 'event' or data['id'] != uid:
            return make_error_response('Invalid data', 400)
        # Check attributes
        for key in data['attributes'].keys():
            if key not in ('status', 'start', 'end'):
                return make_error_response('Invalid data', 400)
        # Check new status
        new_status = data['attributes'].get('status')
        if new_status:
            new_status = new_status.upper().replace(' ', '_')
            data['attributes']['status'] = int(getattr(Status, new_status))
    except Exception:
        return make_error_response('Invalid data', 400)

    db = get_session()
    event = db.query(RecordedEvent).filter(RecordedEvent.uid == uid).first()
    if not event:
        return make_error_response('No event with specified uid', 404)
    event.start = data['attributes'].get('start', event.start)
    event.end = data['attributes'].get('end', event.end)
    event.status = data['attributes'].get('status', event.status)
    logger.debug('Updating event %s via api', uid)
    db.commit()
    return make_data_response(event.serialize())

def
modify_event
(
uid
)
:
try
:
data
=
request
.
get_json
(
)
[
'data'
]
[
0
]
if
data
[
'type'
]
!=
'event'
or
data
[
'id'
]
!=
uid
:
return
make_error_response
(
'Invalid data'
,
400
)
# Check attributes
for
key
in
data
[
'attributes'
]
.
keys
(
)
:
if
key
not
in
(
'status'
,
'start'
,
'end'
)
:
return
make_error_response
(
'Invalid data'
,
400
)
# Check new status
new_status
=
data
[
'attributes'
]
.
get
(
'status'
)
if
new_status
:
new_status
=
new_status
.
upper
(
)
.
replace
(
' '
,
'_'
)
data
[
'attributes'
]
[
'status'
]
=
int
(
getattr
(
Status
,
new_status
)
)
except
Exception
:
return
make_error_response
(
'Invalid data'
,
400
)
db
=
get_session
(
)
event
=
db
.
query
(
RecordedEvent
)
.
filter
(
RecordedEvent
.
uid
==
uid
)
.
first
(
)
if
not
event
:
return
make_error_response
(
'No event with specified uid'
,
404
)
event
.
start
=
data
[
'attributes'
]
.
get
(
'start'
,
event
.
start
)
event
.
end
=
data
[
'attributes'
]
.
get
(
'end'
,
event
.
end
)
event
.
status
=
data
[
'attributes'
]
.
get
(
'status'
,
event
.
status
)
logger
.
debug
(
'Updating event %s via api'
,
uid
)
db
.
commit
(
)
return
make_data_response
(
event
.
serialize
(
)
)

Modify an event specified by its uid . The modifications for the event are expected as JSON with the content type correctly set in the request .


def get_config_params(properties):
    '''Extract the set of configuration parameters from the properties attached
    to the schedule
    '''
    param = []
    wdef = ''
    for prop in properties.split('\n'):
        if prop.startswith('org.opencastproject.workflow.config'):
            key, val = prop.split('=', 1)
            key = key.split('.')[-1]
            param.append((key, val))
        elif prop.startswith('org.opencastproject.workflow.definition'):
            wdef = prop.split('=', 1)[-1]
    return wdef, param


def ingest(event):
    '''Ingest a finished recording to the Opencast server.
    '''
    # Update status
    set_service_status(Service.INGEST, ServiceStatus.BUSY)
    notify.notify('STATUS=Uploading')
    recording_state(event.uid, 'uploading')
    update_event_status(event, Status.UPLOADING)

    # Select ingest service
    # The ingest service to use is selected at random from the available
    # ingest services to ensure that not every capture agent uses the same
    # service at the same time
    service = config('service-ingest')
    service = service[randrange(0, len(service))]
    logger.info('Selecting ingest service to use: ' + service)

    # create mediapackage
    logger.info('Creating new mediapackage')
    mediapackage = http_request(service + '/createMediaPackage')

    # extract workflow_def, workflow_config and add DC catalogs
    prop = 'org.opencastproject.capture.agent.properties'
    dcns = 'http://www.opencastproject.org/xsd/1.0/dublincore/'
    for attachment in event.get_data().get('attach'):
        data = attachment.get('data')
        if attachment.get('x-apple-filename') == prop:
            workflow_def, workflow_config = get_config_params(data)

        # Check for dublincore catalogs
        elif attachment.get('fmttype') == 'application/xml' and dcns in data:
            name = attachment.get('x-apple-filename', '').rsplit('.', 1)[0]
            logger.info('Adding %s DC catalog' % name)
            fields = [('mediaPackage', mediapackage),
                      ('flavor', 'dublincore/%s' % name),
                      ('dublinCore', data.encode('utf-8'))]
            mediapackage = http_request(service + '/addDCCatalog', fields)

    # add track
    for (flavor, track) in event.get_tracks():
        logger.info('Adding track ({0} -> {1})'.format(flavor, track))
        track = track.encode('ascii', 'ignore')
        fields = [('mediaPackage', mediapackage), ('flavor', flavor),
                  ('BODY1', (pycurl.FORM_FILE, track))]
        mediapackage = http_request(service + '/addTrack', fields)

    # ingest
    logger.info('Ingest recording')
    fields = [('mediaPackage', mediapackage)]
    if workflow_def:
        fields.append(('workflowDefinitionId', workflow_def))
    if event.uid:
        fields.append(('workflowInstanceId',
                       event.uid.encode('ascii', 'ignore')))
    fields += workflow_config
    mediapackage = http_request(service + '/ingest', fields)

    # Update status
    recording_state(event.uid, 'upload_finished')
    update_event_status(event, Status.FINISHED_UPLOADING)
    notify.notify('STATUS=Running')
    set_service_status_immediate(Service.INGEST, ServiceStatus.IDLE)

    logger.info('Finished ingest')

Ingest a finished recording to the Opencast server .


def start_capture(upcoming_event):
    '''Start the capture process, creating all necessary files and directories
    as well as ingesting the captured files if no backup mode is configured.
    '''
    logger.info('Start recording')

    # First move event to recording_event table
    db = get_session()
    event = db.query(RecordedEvent)\
              .filter(RecordedEvent.uid == upcoming_event.uid)\
              .filter(RecordedEvent.start == upcoming_event.start)\
              .first()
    if not event:
        event = RecordedEvent(upcoming_event)
        db.add(event)
        db.commit()

    try_mkdir(config()['capture']['directory'])
    os.mkdir(event.directory())

    # Set state
    update_event_status(event, Status.RECORDING)
    recording_state(event.uid, 'capturing')
    set_service_status_immediate(Service.CAPTURE, ServiceStatus.BUSY)

    # Recording
    tracks = recording_command(event)
    event.set_tracks(tracks)
    db.commit()

    # Set status
    update_event_status(event, Status.FINISHED_RECORDING)
    recording_state(event.uid, 'capture_finished')
    set_service_status_immediate(Service.CAPTURE, ServiceStatus.IDLE)

    logger.info('Finished recording')

Start the capture process creating all necessary files and directories as well as ingesting the captured files if no backup mode is configured .


def render_to_fragment(self, request, **kwargs):
        """
        Returns a simple fragment
        """
        fragment = Fragment(TEST_HTML)
        fragment.add_javascript(TEST_JS)
        fragment.add_css(TEST_CSS)
        return fragment

def render_to_fragment ( self , request , * * kwargs ) : fragment = Fragment ( TEST_HTML ) fragment . add_javascript ( TEST_JS ) fragment . add_css ( TEST_CSS ) return fragment


def resources(self):
        """
        Returns list of unique `FragmentResource`s by order of first appearance.
        """
        seen = set()
        # seen.add always returns None, so 'not seen.add(x)' is always True,
        # but will only be called if the value is not already in seen (because
        # 'and' short-circuits)
        return [x for x in self._resources if x not in seen and not seen.add(x)]

Returns list of unique FragmentResource s by order of first appearance .


def to_dict(self):
        """
        Returns the fragment in a dictionary representation.
        """
        return {
            'content': self.content,
            'resources': [r._asdict() for r in self.resources],  # pylint: disable=W0212
            'js_init_fn': self.js_init_fn,
            'js_init_version': self.js_init_version,
            'json_init_args': self.json_init_args
        }

def
to_dict
(
self
)
:
return
{
'content'
:
self
.
content
,
'resources'
:
[
r
.
_asdict
(
)
for
r
in
self
.
resources
]
,
# pylint: disable=W0212
'js_init_fn'
:
self
.
js_init_fn
,
'js_init_version'
:
self
.
js_init_version
,
'json_init_args'
:
self
.
json_init_args
}

Returns the fragment in a dictionary representation .


def from_dict(cls, pods):
        """
        Returns a new Fragment from a dictionary representation.
        """
        frag = cls()
        frag.content = pods['content']
        frag._resources = [FragmentResource(**d) for d in pods['resources']]  # pylint: disable=protected-access
        frag.js_init_fn = pods['js_init_fn']
        frag.js_init_version = pods['js_init_version']
        frag.json_init_args = pods['json_init_args']
        return frag

def
from_dict
(
cls
,
pods
)
:
frag
=
cls
(
)
frag
.
content
=
pods
[
'content'
]
frag
.
_resources
=
[
FragmentResource
(
*
*
d
)
for
d
in
pods
[
'resources'
]
]
# pylint: disable=protected-access
frag
.
js_init_fn
=
pods
[
'js_init_fn'
]
frag
.
js_init_version
=
pods
[
'js_init_version'
]
frag
.
json_init_args
=
pods
[
'json_init_args'
]
return
frag

Returns a new Fragment from a dictionary representation .


def add_content(self, content):
        """
        Add content to this fragment.

        `content` is a Unicode string, HTML to append to the body of the
        fragment.  It must not contain a ``<body>`` tag, or otherwise assume
        that it is the only content on the page.
        """
        assert isinstance(content, six.text_type)
        self.content += content

def add_content ( self , content ) : assert isinstance ( content , six . text_type ) self . content += content


def add_resource(self, text, mimetype, placement=None):
        """
        Add a resource needed by this Fragment.

        Other helpers, such as :func:`add_css` or :func:`add_javascript` are
        more convenient for those common types of resource.

        `text`: the actual text of this resource, as a unicode string.

        `mimetype`: the MIME type of the resource.

        `placement`: where on the page the resource should be placed:

            None: let the Fragment choose based on the MIME type.

            "head": put this resource in the ``<head>`` of the page.

            "foot": put this resource at the end of the ``<body>`` of the
            page.

        """
        if not placement:
            placement = self._default_placement(mimetype)
        res = FragmentResource('text', text, mimetype, placement)
        self._resources.append(res)

def add_resource ( self , text , mimetype , placement = None ) : if not placement : placement = self . _default_placement ( mimetype ) res = FragmentResource ( 'text' , text , mimetype , placement ) self . _resources . append ( res )

Add a resource needed by this Fragment .


def add_resource_url(self, url, mimetype, placement=None):
        """
        Add a resource by URL needed by this Fragment.

        Other helpers, such as :func:`add_css_url` or
        :func:`add_javascript_url` are more convenent for those common types of
        resource.

        `url`: the URL to the resource.

        Other parameters are as defined for :func:`add_resource`.
        """
        if not placement:
            placement = self._default_placement(mimetype)
        self._resources.append(FragmentResource('url', url, mimetype, placement))

def add_resource_url ( self , url , mimetype , placement = None ) : if not placement : placement = self . _default_placement ( mimetype ) self . _resources . append ( FragmentResource ( 'url' , url , mimetype , placement ) )

Add a resource by URL needed by this Fragment .


def initialize_js(self, js_func, json_args=None):
        """
        Register a Javascript function to initialize the Javascript resources.

        `js_func` is the name of a Javascript function defined by one of the
        Javascript resources.  As part of setting up the browser's runtime
        environment, the function will be invoked, passing a runtime object
        and a DOM element.
        """
        self.js_init_fn = js_func
        self.js_init_version = JS_API_VERSION
        if json_args:
            self.json_init_args = json_args

Register a Javascript function to initialize the Javascript resources .


def resources_to_html(self, placement):
        """
        Get some resource HTML for this Fragment.

        `placement` is "head" or "foot".

        Returns a unicode string, the HTML for the head or foot of the page.
        """
        # - non url js could be wrapped in an anonymous function
        # - non url css could be rewritten to match the wrapper tag

        return '\n'.join(
            self.resource_to_html(resource)
            for resource in self.resources
            if resource.placement == placement
        )

Get some resource HTML for this Fragment .


def resource_to_html(resource):
        """
        Returns `resource` wrapped in the appropriate html tag for it's mimetype.
        """
        if resource.mimetype == "text/css":
            if resource.kind == "text":
                return u"<style type='text/css'>\n%s\n</style>" % resource.data
            elif resource.kind == "url":
                return u"<link rel='stylesheet' href='%s' type='text/css'>" % resource.data
            else:
                raise Exception("Unrecognized resource kind %r" % resource.kind)

        elif resource.mimetype == "application/javascript":
            if resource.kind == "text":
                return u"<script>\n%s\n</script>" % resource.data
            elif resource.kind == "url":
                return u"<script src='%s' type='application/javascript'></script>" % resource.data
            else:
                raise Exception("Unrecognized resource kind %r" % resource.kind)

        elif resource.mimetype == "text/html":
            assert resource.kind == "text"
            return resource.data

        else:
            raise Exception("Unrecognized mimetype %r" % resource.mimetype)

Returns resource wrapped in the appropriate html tag for it s mimetype .


def get(self, request, *args, **kwargs):
        """
        Render a fragment to HTML or return JSON describing it, based on the request.
        """
        fragment = self.render_to_fragment(request, **kwargs)
        response_format = request.GET.get('format') or request.POST.get('format') or 'html'
        if response_format == 'json' or WEB_FRAGMENT_RESPONSE_TYPE in request.META.get('HTTP_ACCEPT', 'text/html'):
            return JsonResponse(fragment.to_dict())
        else:
            return self.render_standalone_response(request, fragment, **kwargs)

def get ( self , request , * args , * * kwargs ) : fragment = self . render_to_fragment ( request , * * kwargs ) response_format = request . GET . get ( 'format' ) or request . POST . get ( 'format' ) or 'html' if response_format == 'json' or WEB_FRAGMENT_RESPONSE_TYPE in request . META . get ( 'HTTP_ACCEPT' , 'text/html' ) : return JsonResponse ( fragment . to_dict ( ) ) else : return self . render_standalone_response ( request , fragment , * * kwargs )

Render a fragment to HTML or return JSON describing it based on the request .


def render_standalone_response(self, request, fragment, **kwargs):  # pylint: disable=unused-argument
        """
        Renders a standalone page as a response for the specified fragment.
        """
        if fragment is None:
            return HttpResponse(status=204)

        html = self.render_to_standalone_html(request, fragment, **kwargs)
        return HttpResponse(html)

def
render_standalone_response
(
self
,
request
,
fragment
,
*
*
kwargs
)
:
# pylint: disable=unused-argument
if
fragment
is
:
return
HttpResponse
(
status
=
204
)
html
=
self
.
render_to_standalone_html
(
request
,
fragment
,
*
*
kwargs
)
return
HttpResponse
(
html
)

Renders a standalone page as a response for the specified fragment .


def render_to_standalone_html(self, request, fragment, **kwargs):  # pylint: disable=unused-argument
        """
        Render the specified fragment to HTML for a standalone page.
        """
        template = get_template(STANDALONE_TEMPLATE_NAME)
        context = {
            'head_html': fragment.head_html(),
            'body_html': fragment.body_html(),
            'foot_html': fragment.foot_html(),
        }
        return template.render(context)

def
render_to_standalone_html
(
self
,
request
,
fragment
,
*
*
kwargs
)
:
# pylint: disable=unused-argument
template
=
get_template
(
STANDALONE_TEMPLATE_NAME
)
context
=
{
'head_html'
:
fragment
.
head_html
(
)
,
'body_html'
:
fragment
.
body_html
(
)
,
'foot_html'
:
fragment
.
foot_html
(
)
,
}
return
template
.
render
(
context
)

Render the specified fragment to HTML for a standalone page .


def calc(pvalues, lamb):
    """ meaning pvalues presorted i descending order"""

    m = len(pvalues)
    pi0 = (pvalues > lamb).sum() / ((1 - lamb)*m)

    pFDR = np.ones(m)
    print("pFDR    y        Pr     fastPow")
    for i in range(m):
        y = pvalues[i]
        Pr = max(1, m - i) / float(m)
        pFDR[i] = (pi0 * y) / (Pr * (1 - math.pow(1-y, m)))
        print(i, pFDR[i], y, Pr, 1.0 - math.pow(1-y, m))

    num_null = pi0*m
    num_alt = m - num_null
    num_negs = np.array(range(m))
    num_pos = m - num_negs
    pp = num_pos / float(m)

    qvalues = np.ones(m)
    qvalues[0] = pFDR[0]
    for i in range(m-1):
        qvalues[i+1] = min(qvalues[i], pFDR[i+1])

    sens = ((1.0 - qvalues) * num_pos) / num_alt
    sens[sens > 1.0] = 1.0

    df = pd.DataFrame(dict(
        pvalue=pvalues,
        qvalue=qvalues,
        FDR=pFDR,
        percentile_positive=pp,
        sens=sens
    ))

    df["svalue"] = df.sens[::-1].cummax()[::-1]

    return df, num_null, m

def
calc
(
pvalues
,
lamb
)
:
m
=
len
(
pvalues
)
pi0
=
(
pvalues
>
lamb
)
.
sum
(
)
/
(
(
1
lamb
)
*
m
)
pFDR
=
np
.
ones
(
m
)
print
(
"pFDR    y        Pr     fastPow"
)
for
i
in
range
(
m
)
:
y
=
pvalues
[
i
]
Pr
=
max
(
1
,
m
i
)
/
float
(
m
)
pFDR
[
i
]
=
(
pi0
*
y
)
/
(
Pr
*
(
1
math
.
pow
(
1
y
,
m
)
)
)
print
(
i
,
pFDR
[
i
]
,
y
,
Pr
,
1.0
math
.
pow
(
1
y
,
m
)
)
num_null
=
pi0
*
m
num_alt
=
m
num_null
num_negs
=
np
.
array
(
range
(
m
)
)
num_pos
=
m
num_negs
pp
=
num_pos
/
float
(
m
)
qvalues
=
np
.
ones
(
m
)
qvalues
[
0
]
=
pFDR
[
0
]
for
i
in
range
(
m
1
)
:
qvalues
[
i
+
1
]
=
min
(
qvalues
[
i
]
,
pFDR
[
i
+
1
]
)
sens
=
(
(
1.0
qvalues
)
*
num_pos
)
/
num_alt
sens
[
sens
>
1.0
]
=
1.0
df
=
pd
.
DataFrame
(
dict
(
pvalue
=
pvalues
,
qvalue
=
qvalues
,
FDR
=
pFDR
,
percentile_positive
=
pp
,
sens
=
sens
)
)
df
[
"svalue"
]
=
df
.
sens
[
:
:
1
]
.
cummax
(
)
[
:
:
1
]
return
df
,
num_null
,
m


def to_one_dim_array(values, as_type=None):
    """ Converts list or flattens n-dim array to 1-dim array if possible """

    if isinstance(values, (list, tuple)):
        values = np.array(values, dtype=np.float32)
    elif isinstance(values, pd.Series):
        values = values.values
    values = values.flatten()
    assert values.ndim == 1, "values has wrong dimension"
    if as_type is not None:
        return values.astype(as_type)
    return values

def
to_one_dim_array
(
values
,
as_type
=
)
:
if
isinstance
(
values
,
(
list
,
tuple
)
)
:
values
=
np
.
array
(
values
,
dtype
=
np
.
float32
)
elif
isinstance
(
values
,
pd
.
Series
)
:
values
=
values
.
values
values
=
values
.
flatten
(
)
assert
values
.
ndim
==
1
,
"values has wrong dimension"
if
as_type
is
not
:
return
values
.
astype
(
as_type
)
return
values

Converts list or flattens n - dim array to 1 - dim array if possible


def lookup_values_from_error_table(scores, err_df):
    """ Find matching q-value for each score in 'scores' """
    ix = find_nearest_matches(np.float32(err_df.cutoff.values), np.float32(scores))
    return err_df.pvalue.iloc[ix].values, err_df.svalue.iloc[ix].values, err_df.pep.iloc[ix].values, err_df.qvalue.iloc[ix].values

def lookup_values_from_error_table ( scores , err_df ) : ix = find_nearest_matches ( np . float32 ( err_df . cutoff . values ) , np . float32 ( scores ) ) return err_df . pvalue . iloc [ ix ] . values , err_df . svalue . iloc [ ix ] . values , err_df . pep . iloc [ ix ] . values , err_df . qvalue . iloc [ ix ] . values

Find matching q - value for each score in scores


def posterior_chromatogram_hypotheses_fast(experiment, prior_chrom_null):
    """ Compute posterior probabilities for each chromatogram

    For each chromatogram (each group_id / peptide precursor), all hypothesis of all peaks
    being correct (and all others false) as well as the h0 (all peaks are
    false) are computed.

    The prior probability that the  are given in the function

    This assumes that the input data is sorted by tg_num_id

        Args:
            experiment(:class:`data_handling.Multipeptide`): the data of one experiment
            prior_chrom_null(float): the prior probability that any precursor
                is absent (all peaks are false)

        Returns:
            tuple(hypothesis, h0): two vectors that contain for each entry in
            the input dataframe the probabilities for the hypothesis that the
            peak is correct and the probability for the h0
    """

    tg_ids = experiment.df.tg_num_id.values
    pp_values = 1-experiment.df["pep"].values

    current_tg_id = tg_ids[0]
    scores = []
    final_result = []
    final_result_h0 = []
    for i in range(tg_ids.shape[0]):

        id_ = tg_ids[i]
        if id_ != current_tg_id:

            # Actual computation for a single transition group (chromatogram)
            prior_pg_true = (1.0 - prior_chrom_null) / len(scores)
            rr = single_chromatogram_hypothesis_fast(
                np.array(scores), prior_chrom_null, prior_pg_true)
            final_result.extend(rr[1:])
            final_result_h0.extend(rr[0] for i in range(len(scores)))

            # Reset for next cycle
            scores = []
            current_tg_id = id_

        scores.append(1.0 - pp_values[i])

    # Last cycle
    prior_pg_true = (1.0 - prior_chrom_null) / len(scores)
    rr = single_chromatogram_hypothesis_fast(np.array(scores), prior_chrom_null, prior_pg_true)
    final_result.extend(rr[1:])
    final_result_h0.extend([rr[0]] * len(scores))

    return final_result, final_result_h0


def final_err_table(df, num_cut_offs=51):
    """ Create artificial cutoff sample points from given range of cutoff
    values in df, number of sample points is 'num_cut_offs' """

    cutoffs = df.cutoff.values
    min_ = min(cutoffs)
    max_ = max(cutoffs)
    # extend max_ and min_ by 5 % of full range
    margin = (max_ - min_) * 0.05
    sampled_cutoffs = np.linspace(min_ - margin, max_ + margin, num_cut_offs, dtype=np.float32)

    # find best matching row index for each sampled cut off:
    ix = find_nearest_matches(np.float32(df.cutoff.values), sampled_cutoffs)

    # create sub dataframe:
    sampled_df = df.iloc[ix].copy()
    sampled_df.cutoff = sampled_cutoffs
    # remove 'old' index from input df:
    sampled_df.reset_index(inplace=True, drop=True)

    return sampled_df

Create artificial cutoff sample points from given range of cutoff values in df number of sample points is num_cut_offs


def summary_err_table(df, qvalues=[0, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5]):
    """ Summary error table for some typical q-values """

    qvalues = to_one_dim_array(qvalues)
    # find best matching fows in df for given qvalues:
    ix = find_nearest_matches(np.float32(df.qvalue.values), qvalues)
    # extract sub table
    df_sub = df.iloc[ix].copy()
    # remove duplicate hits, mark them with None / NAN:
    for i_sub, (i0, i1) in enumerate(zip(ix, ix[1:])):
        if i1 == i0:
            df_sub.iloc[i_sub + 1, :] = None
    # attach q values column
    df_sub.qvalue = qvalues
    # remove old index from original df:
    df_sub.reset_index(inplace=True, drop=True)
    return df_sub[['qvalue','pvalue','svalue','pep','fdr','fnr','fpr','tp','tn','fp','fn','cutoff']]

Summary error table for some typical q - values


def error_statistics(target_scores, decoy_scores, parametric, pfdr, pi0_lambda, pi0_method = "smoother", pi0_smooth_df = 3, pi0_smooth_log_pi0 = False, compute_lfdr = False, lfdr_trunc = True, lfdr_monotone = True, lfdr_transf = "probit", lfdr_adj = 1.5, lfdr_eps = np.power(10.0,-8)):
    """ Takes list of decoy and target scores and creates error statistics for target values """

    target_scores = to_one_dim_array(target_scores)
    target_scores = np.sort(target_scores[~np.isnan(target_scores)])

    decoy_scores = to_one_dim_array(decoy_scores)
    decoy_scores = np.sort(decoy_scores[~np.isnan(decoy_scores)])

    # compute p-values using decoy scores
    if parametric:
        # parametric
        target_pvalues = pnorm(target_scores, decoy_scores)
    else:
        # non-parametric
        target_pvalues = pemp(target_scores, decoy_scores)

    # estimate pi0
    pi0 = pi0est(target_pvalues, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0)

    # compute q-value
    target_qvalues = qvalue(target_pvalues, pi0['pi0'], pfdr)

    # compute other metrics
    metrics = stat_metrics(target_pvalues, pi0['pi0'], pfdr)

    # generate main statistics table
    error_stat = pd.DataFrame({'cutoff': target_scores, 'pvalue': target_pvalues, 'qvalue': target_qvalues, 'svalue': metrics['svalue'], 'tp': metrics['tp'], 'fp': metrics['fp'], 'tn': metrics['tn'], 'fn': metrics['fn'], 'fpr': metrics['fpr'], 'fdr': metrics['fdr'], 'fnr': metrics['fnr']})

    # compute lfdr / PEP
    if compute_lfdr:
        error_stat['pep'] = lfdr(target_pvalues, pi0['pi0'], lfdr_trunc, lfdr_monotone, lfdr_transf, lfdr_adj, lfdr_eps)

    return error_stat, pi0

def
error_statistics
(
target_scores
,
decoy_scores
,
parametric
,
pfdr
,
pi0_lambda
,
pi0_method
=
"smoother"
,
pi0_smooth_df
=
3
,
pi0_smooth_log_pi0
=
False
,
compute_lfdr
=
False
,
lfdr_trunc
=
True
,
lfdr_monotone
=
True
,
lfdr_transf
=
"probit"
,
lfdr_adj
=
1.5
,
lfdr_eps
=
np
.
power
(
10.0
,
8
)
)
:
target_scores
=
to_one_dim_array
(
target_scores
)
target_scores
=
np
.
sort
(
target_scores
[
~
np
.
isnan
(
target_scores
)
]
)
decoy_scores
=
to_one_dim_array
(
decoy_scores
)
decoy_scores
=
np
.
sort
(
decoy_scores
[
~
np
.
isnan
(
decoy_scores
)
]
)
# compute p-values using decoy scores
if
parametric
:
# parametric
target_pvalues
=
pnorm
(
target_scores
,
decoy_scores
)
else
:
# non-parametric
target_pvalues
=
pemp
(
target_scores
,
decoy_scores
)
# estimate pi0
pi0
=
pi0est
(
target_pvalues
,
pi0_lambda
,
pi0_method
,
pi0_smooth_df
,
pi0_smooth_log_pi0
)
# compute q-value
target_qvalues
=
qvalue
(
target_pvalues
,
pi0
[
'pi0'
]
,
pfdr
)
# compute other metrics
metrics
=
stat_metrics
(
target_pvalues
,
pi0
[
'pi0'
]
,
pfdr
)
# generate main statistics table
error_stat
=
pd
.
DataFrame
(
{
'cutoff'
:
target_scores
,
'pvalue'
:
target_pvalues
,
'qvalue'
:
target_qvalues
,
'svalue'
:
metrics
[
'svalue'
]
,
'tp'
:
metrics
[
'tp'
]
,
'fp'
:
metrics
[
'fp'
]
,
'tn'
:
metrics
[
'tn'
]
,
'fn'
:
metrics
[
'fn'
]
,
'fpr'
:
metrics
[
'fpr'
]
,
'fdr'
:
metrics
[
'fdr'
]
,
'fnr'
:
metrics
[
'fnr'
]
}
)
# compute lfdr / PEP
if
compute_lfdr
:
error_stat
[
'pep'
]
=
lfdr
(
target_pvalues
,
pi0
[
'pi0'
]
,
lfdr_trunc
,
lfdr_monotone
,
lfdr_transf
,
lfdr_adj
,
lfdr_eps
)
return
error_stat
,
pi0


def find_cutoff(tt_scores, td_scores, cutoff_fdr, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0):
    """ Finds cut off target score for specified false discovery rate fdr """

    error_stat, pi0 = error_statistics(tt_scores, td_scores, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, False)
    if not len(error_stat):
        raise click.ClickException("Too little data for calculating error statistcs.")
    i0 = (error_stat.qvalue - cutoff_fdr).abs().idxmin()
    cutoff = error_stat.iloc[i0]["cutoff"]
    return cutoff


def score(infile, outfile, classifier, xgb_autotune, apply_weights, xeval_fraction, xeval_num_iter, ss_initial_fdr, ss_iteration_fdr, ss_num_iter, ss_main_score, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, level, ipf_max_peakgroup_rank, ipf_max_peakgroup_pep, ipf_max_transition_isotope_overlap, ipf_min_transition_sn, tric_chromprob, threads, test):
    """
    Conduct semi-supervised learning and error-rate estimation for MS1, MS2 and transition-level data.
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    # Prepare XGBoost-specific parameters
    xgb_hyperparams = {'autotune': xgb_autotune, 'autotune_num_rounds': 10, 'num_boost_round': 100, 'early_stopping_rounds': 10, 'test_size': 0.33}

    xgb_params = {'eta': 0.3, 'gamma': 0, 'max_depth': 6, 'min_child_weight': 1, 'subsample': 1, 'colsample_bytree': 1, 'colsample_bylevel': 1, 'colsample_bynode': 1, 'lambda': 1, 'alpha': 0, 'scale_pos_weight': 1, 'silent': 1, 'objective': 'binary:logitraw', 'nthread': 1, 'eval_metric': 'auc'}

    xgb_params_space = {'eta': hp.uniform('eta', 0.0, 0.3), 'gamma': hp.uniform('gamma', 0.0, 0.5), 'max_depth': hp.quniform('max_depth', 2, 8, 1), 'min_child_weight': hp.quniform('min_child_weight', 1, 5, 1), 'subsample': 1, 'colsample_bytree': 1, 'colsample_bylevel': 1, 'colsample_bynode': 1, 'lambda': hp.uniform('lambda', 0.0, 1.0), 'alpha': hp.uniform('alpha', 0.0, 1.0), 'scale_pos_weight': 1.0, 'silent': 1, 'objective': 'binary:logitraw', 'nthread': 1, 'eval_metric': 'auc'}

    if not apply_weights:
        PyProphetLearner(infile, outfile, classifier, xgb_hyperparams, xgb_params, xgb_params_space, xeval_fraction, xeval_num_iter, ss_initial_fdr, ss_iteration_fdr, ss_num_iter, ss_main_score, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, level, ipf_max_peakgroup_rank, ipf_max_peakgroup_pep, ipf_max_transition_isotope_overlap, ipf_min_transition_sn, tric_chromprob, threads, test).run()
    else:
        PyProphetWeightApplier(infile, outfile, classifier, xgb_hyperparams, xgb_params, xgb_params_space, xeval_fraction, xeval_num_iter, ss_initial_fdr, ss_iteration_fdr, ss_num_iter, ss_main_score, group_id, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps, level, ipf_max_peakgroup_rank, ipf_max_peakgroup_pep, ipf_max_transition_isotope_overlap, ipf_min_transition_sn, tric_chromprob, threads, test, apply_weights).run()

def
score
(
infile
,
outfile
,
classifier
,
xgb_autotune
,
apply_weights
,
xeval_fraction
,
xeval_num_iter
,
ss_initial_fdr
,
ss_iteration_fdr
,
ss_num_iter
,
ss_main_score
,
group_id
,
parametric
,
pfdr
,
pi0_lambda
,
pi0_method
,
pi0_smooth_df
,
pi0_smooth_log_pi0
,
lfdr_truncate
,
lfdr_monotone
,
lfdr_transformation
,
lfdr_adj
,
lfdr_eps
,
level
,
ipf_max_peakgroup_rank
,
ipf_max_peakgroup_pep
,
ipf_max_transition_isotope_overlap
,
ipf_min_transition_sn
,
tric_chromprob
,
threads
,
test
)
:
if
outfile
is
:
outfile
=
infile
else
:
outfile
=
outfile
# Prepare XGBoost-specific parameters
xgb_hyperparams
=
{
'autotune'
:
xgb_autotune
,
'autotune_num_rounds'
:
10
,
'num_boost_round'
:
100
,
'early_stopping_rounds'
:
10
,
'test_size'
:
0.33
}
xgb_params
=
{
'eta'
:
0.3
,
'gamma'
:
0
,
'max_depth'
:
6
,
'min_child_weight'
:
1
,
'subsample'
:
1
,
'colsample_bytree'
:
1
,
'colsample_bylevel'
:
1
,
'colsample_bynode'
:
1
,
'lambda'
:
1
,
'alpha'
:
0
,
'scale_pos_weight'
:
1
,
'silent'
:
1
,
'objective'
:
'binary:logitraw'
,
'nthread'
:
1
,
'eval_metric'
:
'auc'
}
xgb_params_space
=
{
'eta'
:
hp
.
uniform
(
'eta'
,
0.0
,
0.3
)
,
'gamma'
:
hp
.
uniform
(
'gamma'
,
0.0
,
0.5
)
,
'max_depth'
:
hp
.
quniform
(
'max_depth'
,
2
,
8
,
1
)
,
'min_child_weight'
:
hp
.
quniform
(
'min_child_weight'
,
1
,
5
,
1
)
,
'subsample'
:
1
,
'colsample_bytree'
:
1
,
'colsample_bylevel'
:
1
,
'colsample_bynode'
:
1
,
'lambda'
:
hp
.
uniform
(
'lambda'
,
0.0
,
1.0
)
,
'alpha'
:
hp
.
uniform
(
'alpha'
,
0.0
,
1.0
)
,
'scale_pos_weight'
:
1.0
,
'silent'
:
1
,
'objective'
:
'binary:logitraw'
,
'nthread'
:
1
,
'eval_metric'
:
'auc'
}
if
not
apply_weights
:
PyProphetLearner
(
infile
,
outfile
,
classifier
,
xgb_hyperparams
,
xgb_params
,
xgb_params_space
,
xeval_fraction
,
xeval_num_iter
,
ss_initial_fdr
,
ss_iteration_fdr
,
ss_num_iter
,
ss_main_score
,
group_id
,
parametric
,
pfdr
,
pi0_lambda
,
pi0_method
,
pi0_smooth_df
,
pi0_smooth_log_pi0
,
lfdr_truncate
,
lfdr_monotone
,
lfdr_transformation
,
lfdr_adj
,
lfdr_eps
,
level
,
ipf_max_peakgroup_rank
,
ipf_max_peakgroup_pep
,
ipf_max_transition_isotope_overlap
,
ipf_min_transition_sn
,
tric_chromprob
,
threads
,
test
)
.
run
(
)
else
:
PyProphetWeightApplier
(
infile
,
outfile
,
classifier
,
xgb_hyperparams
,
xgb_params
,
xgb_params_space
,
xeval_fraction
,
xeval_num_iter
,
ss_initial_fdr
,
ss_iteration_fdr
,
ss_num_iter
,
ss_main_score
,
group_id
,
parametric
,
pfdr
,
pi0_lambda
,
pi0_method
,
pi0_smooth_df
,
pi0_smooth_log_pi0
,
lfdr_truncate
,
lfdr_monotone
,
lfdr_transformation
,
lfdr_adj
,
lfdr_eps
,
level
,
ipf_max_peakgroup_rank
,
ipf_max_peakgroup_pep
,
ipf_max_transition_isotope_overlap
,
ipf_min_transition_sn
,
tric_chromprob
,
threads
,
test
,
apply_weights
)
.
run
(
)

Conduct semi - supervised learning and error - rate estimation for MS1 MS2 and transition - level data .


def ipf(infile, outfile, ipf_ms1_scoring, ipf_ms2_scoring, ipf_h0, ipf_grouped_fdr, ipf_max_precursor_pep, ipf_max_peakgroup_pep, ipf_max_precursor_peakgroup_pep, ipf_max_transition_pep):
    """
    Infer peptidoforms after scoring of MS1, MS2 and transition-level data.
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    infer_peptidoforms(infile, outfile, ipf_ms1_scoring, ipf_ms2_scoring, ipf_h0, ipf_grouped_fdr, ipf_max_precursor_pep, ipf_max_peakgroup_pep, ipf_max_precursor_peakgroup_pep, ipf_max_transition_pep)

def ipf ( infile , outfile , ipf_ms1_scoring , ipf_ms2_scoring , ipf_h0 , ipf_grouped_fdr , ipf_max_precursor_pep , ipf_max_peakgroup_pep , ipf_max_precursor_peakgroup_pep , ipf_max_transition_pep ) : if outfile is None : outfile = infile else : outfile = outfile infer_peptidoforms ( infile , outfile , ipf_ms1_scoring , ipf_ms2_scoring , ipf_h0 , ipf_grouped_fdr , ipf_max_precursor_pep , ipf_max_peakgroup_pep , ipf_max_precursor_peakgroup_pep , ipf_max_transition_pep )

Infer peptidoforms after scoring of MS1 MS2 and transition - level data .


def peptide(infile, outfile, context, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps):
    """
    Infer peptides and conduct error-rate estimation in different contexts.
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    infer_peptides(infile, outfile, context, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps)

def peptide ( infile , outfile , context , parametric , pfdr , pi0_lambda , pi0_method , pi0_smooth_df , pi0_smooth_log_pi0 , lfdr_truncate , lfdr_monotone , lfdr_transformation , lfdr_adj , lfdr_eps ) : if outfile is None : outfile = infile else : outfile = outfile infer_peptides ( infile , outfile , context , parametric , pfdr , pi0_lambda , pi0_method , pi0_smooth_df , pi0_smooth_log_pi0 , lfdr_truncate , lfdr_monotone , lfdr_transformation , lfdr_adj , lfdr_eps )

Infer peptides and conduct error - rate estimation in different contexts .


def protein(infile, outfile, context, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps):
    """
    Infer proteins and conduct error-rate estimation in different contexts.
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    infer_proteins(infile, outfile, context, parametric, pfdr, pi0_lambda, pi0_method, pi0_smooth_df, pi0_smooth_log_pi0, lfdr_truncate, lfdr_monotone, lfdr_transformation, lfdr_adj, lfdr_eps)

def protein ( infile , outfile , context , parametric , pfdr , pi0_lambda , pi0_method , pi0_smooth_df , pi0_smooth_log_pi0 , lfdr_truncate , lfdr_monotone , lfdr_transformation , lfdr_adj , lfdr_eps ) : if outfile is None : outfile = infile else : outfile = outfile infer_proteins ( infile , outfile , context , parametric , pfdr , pi0_lambda , pi0_method , pi0_smooth_df , pi0_smooth_log_pi0 , lfdr_truncate , lfdr_monotone , lfdr_transformation , lfdr_adj , lfdr_eps )

Infer proteins and conduct error - rate estimation in different contexts .


def subsample(infile, outfile, subsample_ratio, test):
    """
    Subsample OpenSWATH file to minimum for integrated scoring
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    subsample_osw(infile, outfile, subsample_ratio, test)

def subsample ( infile , outfile , subsample_ratio , test ) : if outfile is None : outfile = infile else : outfile = outfile subsample_osw ( infile , outfile , subsample_ratio , test )


def reduce(infile, outfile):
    """
    Reduce scored PyProphet file to minimum for global scoring
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    reduce_osw(infile, outfile)

def reduce ( infile , outfile ) : if outfile is None : outfile = infile else : outfile = outfile reduce_osw ( infile , outfile )


def backpropagate(infile, outfile, apply_scores):
    """
    Backpropagate multi-run peptide and protein scores to single files
    """

    if outfile is None:
        outfile = infile
    else:
        outfile = outfile

    backpropagate_oswr(infile, outfile, apply_scores)

def backpropagate ( infile , outfile , apply_scores ) : if outfile is None : outfile = infile else : outfile = outfile backpropagate_oswr ( infile , outfile , apply_scores )

Backpropagate multi - run peptide and protein scores to single files


def filter(sqmassfiles, infile, max_precursor_pep, max_peakgroup_pep, max_transition_pep):
    """
    Filter sqMass files
    """

    filter_sqmass(sqmassfiles, infile, max_precursor_pep, max_peakgroup_pep, max_transition_pep)

def filter ( sqmassfiles , infile , max_precursor_pep , max_peakgroup_pep , max_transition_pep ) : filter_sqmass ( sqmassfiles , infile , max_precursor_pep , max_peakgroup_pep , max_transition_pep )


def get_group_by_id(self, group_id):
        """
        Returns a restclients.Group object for the group identified by the
        passed group ID.
        """
        self._valid_group_id(group_id)

        url = "{}/group/{}".format(self.API, group_id)

        data = self._get_resource(url)

        return self._group_from_json(data.get("data"))

def get_group_by_id ( self , group_id ) : self . _valid_group_id ( group_id ) url = "{}/group/{}" . format ( self . API , group_id ) data = self . _get_resource ( url ) return self . _group_from_json ( data . get ( "data" ) )

Returns a restclients . Group object for the group identified by the passed group ID .


def create_group(self, group):
        """
        Creates a group from the passed restclients.Group object.
        """
        self._valid_group_id(group.id)

        body = {"data": group.json_data()}
        url = "{}/group/{}".format(self.API, group.name)

        data = self._put_resource(url, headers={}, body=body)

        return self._group_from_json(data.get("data"))

def create_group ( self , group ) : self . _valid_group_id ( group . id ) body = { "data" : group . json_data ( ) } url = "{}/group/{}" . format ( self . API , group . name ) data = self . _put_resource ( url , headers = { } , body = body ) return self . _group_from_json ( data . get ( "data" ) )

Creates a group from the passed restclients . Group object .


def delete_group(self, group_id):
        """
        Deletes the group identified by the passed group ID.
        """
        self._valid_group_id(group_id)

        url = "{}/group/{}".format(self.API, group_id)

        self._delete_resource(url)

        return True

def delete_group ( self , group_id ) : self . _valid_group_id ( group_id ) url = "{}/group/{}" . format ( self . API , group_id ) self . _delete_resource ( url ) return True

Deletes the group identified by the passed group ID .


def get_members(self, group_id):
        """
        Returns a list of restclients.GroupMember objects for the group
        identified by the passed group ID.
        """
        self._valid_group_id(group_id)

        url = "{}/group/{}/member".format(self.API, group_id)

        data = self._get_resource(url)

        members = []
        for datum in data.get("data"):
            members.append(self._group_member_from_json(datum))
        return members

def get_members ( self , group_id ) : self . _valid_group_id ( group_id ) url = "{}/group/{}/member" . format ( self . API , group_id ) data = self . _get_resource ( url ) members = [ ] for datum in data . get ( "data" ) : members . append ( self . _group_member_from_json ( datum ) ) return members

Returns a list of restclients . GroupMember objects for the group identified by the passed group ID .


def update_members(self, group_id, members):
        """
        Updates the membership of the group represented by the passed group id.
        Returns a list of members not found.
        """
        self._valid_group_id(group_id)

        body = {"data": [m.json_data() for m in members]}
        headers = {"If-Match": "*"}
        url = "{}/group/{}/member".format(self.API, group_id)

        data = self._put_resource(url, headers, body)

        errors = data.get("errors", [])
        if len(errors):
            return errors[0].get("notFound", [])
        return []

def update_members ( self , group_id , members ) : self . _valid_group_id ( group_id ) body = { "data" : [ m . json_data ( ) for m in members ] } headers = { "If-Match" : "*" } url = "{}/group/{}/member" . format ( self . API , group_id ) data = self . _put_resource ( url , headers , body ) errors = data . get ( "errors" , [ ] ) if len ( errors ) : return errors [ 0 ] . get ( "notFound" , [ ] ) return [ ]

Updates the membership of the group represented by the passed group id . Returns a list of members not found .


def get_effective_member_count(self, group_id):
        """
        Returns a count of effective members for the group identified by the
        passed group ID.
        """
        self._valid_group_id(group_id)

        url = "{}/group/{}/effective_member?view=count".format(self.API,
                                                               group_id)

        data = self._get_resource(url)

        count = data.get("data").get("count")
        return int(count)

Returns a count of effective members for the group identified by the passed group ID .


def is_effective_member(self, group_id, netid):
        """
        Returns True if the netid is in the group, False otherwise.
        """
        self._valid_group_id(group_id)

        # GWS doesn't accept EPPNs on effective member checks, for UW users
        netid = re.sub('@washington.edu', '', netid)

        url = "{}/group/{}/effective_member/{}".format(self.API,
                                                       group_id,
                                                       netid)

        try:
            data = self._get_resource(url)
            return True  # 200
        except DataFailureException as ex:
            if ex.status == 404:
                return False
            else:
                raise

Returns True if the netid is in the group False otherwise .


def modify_conf():
    """
    pip install redbaron
    """
    import redbaron
    import ubelt as ub
    conf_path = 'docs/conf.py'

    source = ub.readfrom(conf_path)
    red = redbaron.RedBaron(source)

    # Insert custom extensions
    extra_extensions = [
        '"sphinxcontrib.napoleon"'
    ]

    ext_node = red.find('name', value='extensions').parent
    ext_node.value.value.extend(extra_extensions)

    # Overwrite theme to read-the-docs
    theme_node = red.find('name', value='html_theme').parent
    theme_node.value.value = '"sphinx_rtd_theme"'

    ub.writeto(conf_path, red.dumps())

def
modify_conf
(
)
:
import
redbaron
import
ubelt
as
ub
conf_path
=
'docs/conf.py'
source
=
ub
.
readfrom
(
conf_path
)
red
=
redbaron
.
RedBaron
(
source
)
# Insert custom extensions
extra_extensions
=
[
'"sphinxcontrib.napoleon"'
]
ext_node
=
red
.
find
(
'name'
,
value
=
'extensions'
)
.
parent
ext_node
.
value
.
value
.
extend
(
extra_extensions
)
# Overwrite theme to read-the-docs
theme_node
=
red
.
find
(
'name'
,
value
=
'html_theme'
)
.
parent
theme_node
.
value
.
value
=
'"sphinx_rtd_theme"'
ub
.
writeto
(
conf_path
,
red
.
dumps
(
)
)