def _find_sections(md_ast, sections, last, last_class, total_lines=None):
    """
    Walks through a CommonMark AST to find section headers that delineate
    content that should be updated by this script

    :param md_ast:
        The AST of the markdown document

    :param sections:
        A dict to store the start and end lines of a section. The key will be
        a two-element tuple of the section type ("class", "function",
        "method" or "attribute") and identifier. The values are a two-element
        tuple of the start and end line number in the markdown document of the
        section.

    :param last:
        A dict containing information about the last section header seen.
        Includes the keys "type_name", "identifier", "start_line".

    :param last_class:
        A unicode string of the name of the last class found - used when
        processing methods and attributes.

    :param total_lines:
        An integer of the total number of lines in the markdown document -
        used to work around a bug in the API of the Python port of CommonMark
    """

    def child_walker(node):
        for child, entering in node.walker():
            if child == node:
                continue
            yield child, entering

    for child, entering in child_walker(md_ast):
        if child.t == 'heading':
            start_line = child.sourcepos[0][0]

            if child.level == 2:
                if last:
                    sections[(last['type_name'], last['identifier'])] = (last['start_line'], start_line - 1)
                    last.clear()

            if child.level in set([3, 5]):
                heading_elements = []
                for heading_child, _ in child_walker(child):
                    heading_elements.append(heading_child)
                if len(heading_elements) != 2:
                    continue
                first = heading_elements[0]
                second = heading_elements[1]
                if first.t != 'code':
                    continue
                if second.t != 'text':
                    continue

                type_name = second.literal.strip()
                identifier = first.literal.strip().replace('()', '').lstrip('.')

                if last:
                    sections[(last['type_name'], last['identifier'])] = (last['start_line'], start_line - 1)
                    last.clear()

                if type_name == 'function':
                    if child.level != 3:
                        continue

                if type_name == 'class':
                    if child.level != 3:
                        continue
                    last_class.append(identifier)

                if type_name in set(['method', 'attribute']):
                    if child.level != 5:
                        continue
                    identifier = last_class[-1] + '.' + identifier

                last.update({
                    'type_name': type_name,
                    'identifier': identifier,
                    'start_line': start_line,
                })

        elif child.t == 'block_quote':
            find_sections(child, sections, last, last_class)

    if last:
        sections[(last['type_name'], last['identifier'])] = (last['start_line'], total_lines)

def _find_sections ( md_ast , sections , last , last_class , total_lines = None ) : def child_walker ( node ) : for child , entering in node . walker ( ) : if child == node : continue yield child , entering for child , entering in child_walker ( md_ast ) : if child . t == 'heading' : start_line = child . sourcepos [ 0 ] [ 0 ] if child . level == 2 : if last : sections [ ( last [ 'type_name' ] , last [ 'identifier' ] ) ] = ( last [ 'start_line' ] , start_line - 1 ) last . clear ( ) if child . level in set ( [ 3 , 5 ] ) : heading_elements = [ ] for heading_child , _ in child_walker ( child ) : heading_elements . append ( heading_child ) if len ( heading_elements ) != 2 : continue first = heading_elements [ 0 ] second = heading_elements [ 1 ] if first . t != 'code' : continue if second . t != 'text' : continue type_name = second . literal . strip ( ) identifier = first . literal . strip ( ) . replace ( '()' , '' ) . lstrip ( '.' ) if last : sections [ ( last [ 'type_name' ] , last [ 'identifier' ] ) ] = ( last [ 'start_line' ] , start_line - 1 ) last . clear ( ) if type_name == 'function' : if child . level != 3 : continue if type_name == 'class' : if child . level != 3 : continue last_class . append ( identifier ) if type_name in set ( [ 'method' , 'attribute' ] ) : if child . level != 5 : continue identifier = last_class [ - 1 ] + '.' + identifier last . update ( { 'type_name' : type_name , 'identifier' : identifier , 'start_line' : start_line , } ) elif child . t == 'block_quote' : find_sections ( child , sections , last , last_class ) if last : sections [ ( last [ 'type_name' ] , last [ 'identifier' ] ) ] = ( last [ 'start_line' ] , total_lines )


def walk_ast(node, code_lines, sections, md_chunks):
    """
    A callback used to walk the Python AST looking for classes, functions,
    methods and attributes. Generates chunks of markdown markup to replace
    the existing content.

    :param node:
        An _ast module node object

    :param code_lines:
        A list of unicode strings - the source lines of the Python file

    :param sections:
        A dict of markdown document sections that need to be updated. The key
        will be a two-element tuple of the section type ("class", "function",
        "method" or "attribute") and identifier. The values are a two-element
        tuple of the start and end line number in the markdown document of the
        section.

    :param md_chunks:
        A dict with keys from the sections param and the values being a unicode
        string containing a chunk of markdown markup.
    """

    if isinstance(node, _ast.FunctionDef):
        key = ('function', node.name)
        if key not in sections:
            return

        docstring = ast.get_docstring(node)
        def_lineno = node.lineno + len(node.decorator_list)

        definition, description_md = _get_func_info(docstring, def_lineno, code_lines, '> ')

        md_chunk = textwrap.dedent("""
            ### `%s()` function

            > ```python
            > %s
            > ```
            >
            %s
        """).strip() % (
            node.name,
            definition,
            description_md
        ) + "\n"

        md_chunks[key] = md_chunk.replace('>\n\n', '')

    elif isinstance(node, _ast.ClassDef):
        if ('class', node.name) not in sections:
            return

        for subnode in node.body:
            if isinstance(subnode, _ast.FunctionDef):
                node_id = node.name + '.' + subnode.name

                method_key = ('method', node_id)
                is_method = method_key in sections

                attribute_key = ('attribute', node_id)
                is_attribute = attribute_key in sections

                is_constructor = subnode.name == '__init__'

                if not is_constructor and not is_attribute and not is_method:
                    continue

                docstring = ast.get_docstring(subnode)
                def_lineno = subnode.lineno + len(subnode.decorator_list)

                if not docstring:
                    continue

                if is_method or is_constructor:
                    definition, description_md = _get_func_info(docstring, def_lineno, code_lines, '> > ')

                    if is_constructor:
                        key = ('class', node.name)

                        class_docstring = ast.get_docstring(node) or ''
                        class_description = textwrap.dedent(class_docstring).strip()
                        if class_description:
                            class_description_md = "> %s\n>" % (class_description.replace("\n", "\n> "))
                        else:
                            class_description_md = ''

                        md_chunk = textwrap.dedent("""
                            ### `%s()` class

                            %s
                            > ##### constructor
                            >
                            > > ```python
                            > > %s
                            > > ```
                            > >
                            %s
                        """).strip() % (
                            node.name,
                            class_description_md,
                            definition,
                            description_md
                        )

                        md_chunk = md_chunk.replace('\n\n\n', '\n\n')

                    else:
                        key = method_key

                        md_chunk = textwrap.dedent("""
                            >
                            > ##### `.%s()` method
                            >
                            > > ```python
                            > > %s
                            > > ```
                            > >
                            %s
                        """).strip() % (
                            subnode.name,
                            definition,
                            description_md
                        )

                    if md_chunk[-5:] == '\n> >\n':
                        md_chunk = md_chunk[0:-5]

                else:
                    key = attribute_key

                    description = textwrap.dedent(docstring).strip()
                    description_md = "> > %s" % (description.replace("\n", "\n> > "))

                    md_chunk = textwrap.dedent("""
                        >
                        > ##### `.%s` attribute
                        >
                        %s
                    """).strip() % (
                        subnode.name,
                        description_md
                    )

                md_chunks[key] = re.sub('[ \\t]+\n', '\n', md_chunk.rstrip())

    elif isinstance(node, _ast.If):
        for subast in node.body:
            walk_ast(subast, code_lines, sections, md_chunks)
        for subast in node.orelse:
            walk_ast(subast, code_lines, sections, md_chunks)

A callback used to walk the Python AST looking for classes functions methods and attributes . Generates chunks of markdown markup to replace the existing content .


def system_path():
    """
    Tries to find a CA certs bundle in common locations

    :raises:
        OSError - when no valid CA certs bundle was found on the filesystem

    :return:
        The full filesystem path to a CA certs bundle file
    """

    ca_path = None

    # Common CA cert paths
    paths = [
        '/usr/lib/ssl/certs/ca-certificates.crt',
        '/etc/ssl/certs/ca-certificates.crt',
        '/etc/ssl/certs/ca-bundle.crt',
        '/etc/pki/tls/certs/ca-bundle.crt',
        '/etc/ssl/ca-bundle.pem',
        '/usr/local/share/certs/ca-root-nss.crt',
        '/etc/ssl/cert.pem'
    ]

    # First try SSL_CERT_FILE
    if 'SSL_CERT_FILE' in os.environ:
        paths.insert(0, os.environ['SSL_CERT_FILE'])

    for path in paths:
        if os.path.exists(path) and os.path.getsize(path) > 0:
            ca_path = path
            break

    if not ca_path:
        raise OSError(pretty_message(
            '''
            Unable to find a CA certs bundle in common locations - try
            setting the SSL_CERT_FILE environmental variable
            '''
        ))

    return ca_path


def extract_from_system(cert_callback=None, callback_only_on_failure=False):
    """
    Extracts trusted CA certs from the system CA cert bundle

    :param cert_callback:
        A callback that is called once for each certificate in the trust store.
        It should accept two parameters: an asn1crypto.x509.Certificate object,
        and a reason. The reason will be None if the certificate is being
        exported, otherwise it will be a unicode string of the reason it won't.

    :param callback_only_on_failure:
        A boolean - if the callback should only be called when a certificate is
        not exported.

    :return:
        A list of 3-element tuples:
         - 0: a byte string of a DER-encoded certificate
         - 1: a set of unicode strings that are OIDs of purposes to trust the
              certificate for
         - 2: a set of unicode strings that are OIDs of purposes to reject the
              certificate for
    """

    all_purposes = '2.5.29.37.0'
    ca_path = system_path()

    output = []
    with open(ca_path, 'rb') as f:
        for armor_type, _, cert_bytes in unarmor(f.read(), multiple=True):
            # Without more info, a certificate is trusted for all purposes
            if armor_type == 'CERTIFICATE':
                if cert_callback:
                    cert_callback(Certificate.load(cert_bytes), None)
                output.append((cert_bytes, set(), set()))

            # The OpenSSL TRUSTED CERTIFICATE construct adds OIDs for trusted
            # and rejected purposes, so we extract that info.
            elif armor_type == 'TRUSTED CERTIFICATE':
                cert, aux = TrustedCertificate.load(cert_bytes)
                reject_all = False
                trust_oids = set()
                reject_oids = set()
                for purpose in aux['trust']:
                    if purpose.dotted == all_purposes:
                        trust_oids = set([purpose.dotted])
                        break
                    trust_oids.add(purpose.dotted)
                for purpose in aux['reject']:
                    if purpose.dotted == all_purposes:
                        reject_all = True
                        break
                    reject_oids.add(purpose.dotted)
                if reject_all:
                    if cert_callback:
                        cert_callback(cert, 'explicitly distrusted')
                    continue
                if cert_callback and not callback_only_on_failure:
                    cert_callback(cert, None)
                output.append((cert.dump(), trust_oids, reject_oids))

    return output


def _convert_filetime_to_timestamp(filetime):
    """
    Windows returns times as 64-bit unsigned longs that are the number
    of hundreds of nanoseconds since Jan 1 1601. This converts it to
    a datetime object.

    :param filetime:
        A FILETIME struct object

    :return:
        An integer unix timestamp
    """

    hundreds_nano_seconds = struct.unpack(
        b'>Q',
        struct.pack(
            b'>LL',
            filetime.dwHighDateTime,
            filetime.dwLowDateTime
        )
    )[0]
    seconds_since_1601 = hundreds_nano_seconds / 10000000
    return seconds_since_1601 - 11644473600

def _convert_filetime_to_timestamp ( filetime ) : hundreds_nano_seconds = struct . unpack ( b'>Q' , struct . pack ( b'>LL' , filetime . dwHighDateTime , filetime . dwLowDateTime ) ) [ 0 ] seconds_since_1601 = hundreds_nano_seconds / 10000000 return seconds_since_1601 - 11644473600

Windows returns times as 64 - bit unsigned longs that are the number of hundreds of nanoseconds since Jan 1 1601 . This converts it to a datetime object .


def extract_chain(server_handshake_bytes):
    """
    Extracts the X.509 certificates from the server handshake bytes for use
    when debugging

    :param server_handshake_bytes:
        A byte string of the handshake data received from the server

    :return:
        A list of asn1crypto.x509.Certificate objects
    """

    output = []

    chain_bytes = None

    for record_type, _, record_data in parse_tls_records(server_handshake_bytes):
        if record_type != b'\x16':
            continue
        for message_type, message_data in parse_handshake_messages(record_data):
            if message_type == b'\x0b':
                chain_bytes = message_data
                break
        if chain_bytes:
            break

    if chain_bytes:
        # The first 3 bytes are the cert chain length
        pointer = 3
        while pointer < len(chain_bytes):
            cert_length = int_from_bytes(chain_bytes[pointer:pointer + 3])
            cert_start = pointer + 3
            cert_end = cert_start + cert_length
            pointer = cert_end
            cert_bytes = chain_bytes[cert_start:cert_end]
            output.append(Certificate.load(cert_bytes))

    return output

Extracts the X . 509 certificates from the server handshake bytes for use when debugging


def detect_client_auth_request(server_handshake_bytes):
    """
    Determines if a CertificateRequest message is sent from the server asking
    the client for a certificate

    :param server_handshake_bytes:
        A byte string of the handshake data received from the server

    :return:
        A boolean - if a client certificate request was found
    """

    for record_type, _, record_data in parse_tls_records(server_handshake_bytes):
        if record_type != b'\x16':
            continue
        for message_type, message_data in parse_handshake_messages(record_data):
            if message_type == b'\x0d':
                return True
    return False


def get_dh_params_length(server_handshake_bytes):
    """
    Determines the length of the DH params from the ServerKeyExchange

    :param server_handshake_bytes:
        A byte string of the handshake data received from the server

    :return:
        None or an integer of the bit size of the DH parameters
    """

    output = None

    dh_params_bytes = None

    for record_type, _, record_data in parse_tls_records(server_handshake_bytes):
        if record_type != b'\x16':
            continue
        for message_type, message_data in parse_handshake_messages(record_data):
            if message_type == b'\x0c':
                dh_params_bytes = message_data
                break
        if dh_params_bytes:
            break

    if dh_params_bytes:
        output = int_from_bytes(dh_params_bytes[0:2]) * 8

    return output


def parse_alert(server_handshake_bytes):
    """
    Parses the handshake for protocol alerts

    :param server_handshake_bytes:
        A byte string of the handshake data received from the server

    :return:
        None or an 2-element tuple of integers:
         0: 1 (warning) or 2 (fatal)
         1: The alert description (see https://tools.ietf.org/html/rfc5246#section-7.2)
    """

    for record_type, _, record_data in parse_tls_records(server_handshake_bytes):
        if record_type != b'\x15':
            continue
        if len(record_data) != 2:
            return None
        return (int_from_bytes(record_data[0:1]), int_from_bytes(record_data[1:2]))
    return None


def parse_session_info(server_handshake_bytes, client_handshake_bytes):
    """
    Parse the TLS handshake from the client to the server to extract information
    including the cipher suite selected, if compression is enabled, the
    session id and if a new or reused session ticket exists.

    :param server_handshake_bytes:
        A byte string of the handshake data received from the server

    :param client_handshake_bytes:
        A byte string of the handshake data sent to the server

    :return:
        A dict with the following keys:
         - "protocol": unicode string
         - "cipher_suite": unicode string
         - "compression": boolean
         - "session_id": "new", "reused" or None
         - "session_ticket: "new", "reused" or None
    """

    protocol = None
    cipher_suite = None
    compression = False
    session_id = None
    session_ticket = None

    server_session_id = None
    client_session_id = None

    for record_type, _, record_data in parse_tls_records(server_handshake_bytes):
        if record_type != b'\x16':
            continue
        for message_type, message_data in parse_handshake_messages(record_data):
            # Ensure we are working with a ServerHello message
            if message_type != b'\x02':
                continue
            protocol = {
                b'\x03\x00': "SSLv3",
                b'\x03\x01': "TLSv1",
                b'\x03\x02': "TLSv1.1",
                b'\x03\x03': "TLSv1.2",
                b'\x03\x04': "TLSv1.3",
            }[message_data[0:2]]

            session_id_length = int_from_bytes(message_data[34:35])
            if session_id_length > 0:
                server_session_id = message_data[35:35 + session_id_length]

            cipher_suite_start = 35 + session_id_length
            cipher_suite_bytes = message_data[cipher_suite_start:cipher_suite_start + 2]
            cipher_suite = CIPHER_SUITE_MAP[cipher_suite_bytes]

            compression_start = cipher_suite_start + 2
            compression = message_data[compression_start:compression_start + 1] != b'\x00'

            extensions_length_start = compression_start + 1
            extensions_data = message_data[extensions_length_start:]
            for extension_type, extension_data in _parse_hello_extensions(extensions_data):
                if extension_type == 35:
                    session_ticket = "new"
                    break
            break

    for record_type, _, record_data in parse_tls_records(client_handshake_bytes):
        if record_type != b'\x16':
            continue
        for message_type, message_data in parse_handshake_messages(record_data):
            # Ensure we are working with a ClientHello message
            if message_type != b'\x01':
                continue

            session_id_length = int_from_bytes(message_data[34:35])
            if session_id_length > 0:
                client_session_id = message_data[35:35 + session_id_length]

            cipher_suite_start = 35 + session_id_length
            cipher_suite_length = int_from_bytes(message_data[cipher_suite_start:cipher_suite_start + 2])

            compression_start = cipher_suite_start + 2 + cipher_suite_length
            compression_length = int_from_bytes(message_data[compression_start:compression_start + 1])

            # On subsequent requests, the session ticket will only be seen
            # in the ClientHello message
            if server_session_id is None and session_ticket is None:
                extensions_length_start = compression_start + 1 + compression_length
                extensions_data = message_data[extensions_length_start:]
                for extension_type, extension_data in _parse_hello_extensions(extensions_data):
                    if extension_type == 35:
                        session_ticket = "reused"
                        break
            break

    if server_session_id is not None:
        if client_session_id is None:
            session_id = "new"
        else:
            if client_session_id != server_session_id:
                session_id = "new"
            else:
                session_id = "reused"

    return {
        "protocol": protocol,
        "cipher_suite": cipher_suite,
        "compression": compression,
        "session_id": session_id,
        "session_ticket": session_ticket,
    }

Parse the TLS handshake from the client to the server to extract information including the cipher suite selected if compression is enabled the session id and if a new or reused session ticket exists .


def parse_tls_records(data):
    """
    Creates a generator returning tuples of information about each record
    in a byte string of data from a TLS client or server. Stops as soon as it
    find a ChangeCipherSpec message since all data from then on is encrypted.

    :param data:
        A byte string of TLS records

    :return:
        A generator that yields 3-element tuples:
        [0] Byte string of record type
        [1] Byte string of protocol version
        [2] Byte string of record data
    """

    pointer = 0
    data_len = len(data)
    while pointer < data_len:
        # Don't try to parse any more once the ChangeCipherSpec is found
        if data[pointer:pointer + 1] == b'\x14':
            break
        length = int_from_bytes(data[pointer + 3:pointer + 5])
        yield (
            data[pointer:pointer + 1],
            data[pointer + 1:pointer + 3],
            data[pointer + 5:pointer + 5 + length]
        )
        pointer += 5 + length

Creates a generator returning tuples of information about each record in a byte string of data from a TLS client or server . Stops as soon as it find a ChangeCipherSpec message since all data from then on is encrypted .


def parse_handshake_messages(data):
    """
    Creates a generator returning tuples of information about each message in
    a byte string of data from a TLS handshake record

    :param data:
        A byte string of a TLS handshake record data

    :return:
        A generator that yields 2-element tuples:
        [0] Byte string of message type
        [1] Byte string of message data
    """

    pointer = 0
    data_len = len(data)
    while pointer < data_len:
        length = int_from_bytes(data[pointer + 1:pointer + 4])
        yield (
            data[pointer:pointer + 1],
            data[pointer + 4:pointer + 4 + length]
        )
        pointer += 4 + length

def parse_handshake_messages ( data ) : pointer = 0 data_len = len ( data ) while pointer < data_len : length = int_from_bytes ( data [ pointer + 1 : pointer + 4 ] ) yield ( data [ pointer : pointer + 1 ] , data [ pointer + 4 : pointer + 4 + length ] ) pointer += 4 + length


def _parse_hello_extensions(data):
    """
    Creates a generator returning tuples of information about each extension
    from a byte string of extension data contained in a ServerHello ores
    ClientHello message

    :param data:
        A byte string of a extension data from a TLS ServerHello or ClientHello
        message

    :return:
        A generator that yields 2-element tuples:
        [0] Byte string of extension type
        [1] Byte string of extension data
    """

    if data == b'':
        return

    extentions_length = int_from_bytes(data[0:2])
    extensions_start = 2
    extensions_end = 2 + extentions_length

    pointer = extensions_start
    while pointer < extensions_end:
        extension_type = int_from_bytes(data[pointer:pointer + 2])
        extension_length = int_from_bytes(data[pointer + 2:pointer + 4])
        yield (
            extension_type,
            data[pointer + 4:pointer + 4 + extension_length]
        )
        pointer += 4 + extension_length

def _parse_hello_extensions ( data ) : if data == b'' : return extentions_length = int_from_bytes ( data [ 0 : 2 ] ) extensions_start = 2 extensions_end = 2 + extentions_length pointer = extensions_start while pointer < extensions_end : extension_type = int_from_bytes ( data [ pointer : pointer + 2 ] ) extension_length = int_from_bytes ( data [ pointer + 2 : pointer + 4 ] ) yield ( extension_type , data [ pointer + 4 : pointer + 4 + extension_length ] ) pointer += 4 + extension_length


def raise_hostname(certificate, hostname):
    """
    Raises a TLSVerificationError due to a hostname mismatch

    :param certificate:
        An asn1crypto.x509.Certificate object

    :raises:
        TLSVerificationError
    """

    is_ip = re.match('^\\d+\\.\\d+\\.\\d+\\.\\d+$', hostname) or hostname.find(':') != -1
    if is_ip:
        hostname_type = 'IP address %s' % hostname
    else:
        hostname_type = 'domain name %s' % hostname
    message = 'Server certificate verification failed - %s does not match' % hostname_type
    valid_ips = ', '.join(certificate.valid_ips)
    valid_domains = ', '.join(certificate.valid_domains)
    if valid_domains:
        message += ' valid domains: %s' % valid_domains
    if valid_domains and valid_ips:
        message += ' or'
    if valid_ips:
        message += ' valid IP addresses: %s' % valid_ips
    raise TLSVerificationError(message, certificate)


def raise_expired_not_yet_valid(certificate):
    """
    Raises a TLSVerificationError due to certificate being expired, or not yet
    being valid

    :param certificate:
        An asn1crypto.x509.Certificate object

    :raises:
        TLSVerificationError
    """

    validity = certificate['tbs_certificate']['validity']
    not_after = validity['not_after'].native
    not_before = validity['not_before'].native

    now = datetime.now(timezone.utc)

    if not_before > now:
        formatted_before = not_before.strftime('%Y-%m-%d %H:%M:%SZ')
        message = 'Server certificate verification failed - certificate not valid until %s' % formatted_before
    elif not_after < now:
        formatted_after = not_after.strftime('%Y-%m-%d %H:%M:%SZ')
        message = 'Server certificate verification failed - certificate expired %s' % formatted_after

    raise TLSVerificationError(message, certificate)

Raises a TLSVerificationError due to certificate being expired or not yet being valid


def detect_other_protocol(server_handshake_bytes):
    """
    Looks at the server handshake bytes to try and detect a different protocol

    :param server_handshake_bytes:
        A byte string of the handshake data received from the server

    :return:
        None, or a unicode string of "ftp", "http", "imap", "pop3", "smtp"
    """

    if server_handshake_bytes[0:5] == b'HTTP/':
        return 'HTTP'

    if server_handshake_bytes[0:4] == b'220 ':
        if re.match(b'^[^\r\n]*ftp', server_handshake_bytes, re.I):
            return 'FTP'
        else:
            return 'SMTP'

    if server_handshake_bytes[0:4] == b'220-':
        return 'FTP'

    if server_handshake_bytes[0:4] == b'+OK ':
        return 'POP3'

    if server_handshake_bytes[0:4] == b'* OK' or server_handshake_bytes[0:9] == b'* PREAUTH':
        return 'IMAP'

    return None

def
detect_other_protocol
(
server_handshake_bytes
)
:
if
server_handshake_bytes
[
0
:
5
]
==
b'HTTP/'
:
return
'HTTP'
if
server_handshake_bytes
[
0
:
4
]
==
b'220 '
:
if
re
.
match
(
b'^[^\r\n]*ftp'
,
server_handshake_bytes
,
re
.
I
)
:
return
'FTP'
else
:
return
'SMTP'
if
server_handshake_bytes
[
0
:
4
]
==
b'220-'
:
return
'FTP'
if
server_handshake_bytes
[
0
:
4
]
==
b'+OK '
:
return
'POP3'
if
server_handshake_bytes
[
0
:
4
]
==
b'* OK'
or
server_handshake_bytes
[
0
:
9
]
==
b'* PREAUTH'
:
return
'IMAP'
return


def _try_decode(byte_string):
    """
    Tries decoding a byte string from the OS into a unicode string

    :param byte_string:
        A byte string

    :return:
        A unicode string
    """

    try:
        return str_cls(byte_string, _encoding)

    # If the "correct" encoding did not work, try some defaults, and then just
    # obliterate characters that we can't seen to decode properly
    except (UnicodeDecodeError):
        for encoding in _fallback_encodings:
            try:
                return str_cls(byte_string, encoding, errors='strict')
            except (UnicodeDecodeError):
                pass

    return str_cls(byte_string, errors='replace')


def _read_callback(connection_id, data_buffer, data_length_pointer):
    """
    Callback called by Secure Transport to actually read the socket

    :param connection_id:
        An integer identifing the connection

    :param data_buffer:
        A char pointer FFI type to write the data to

    :param data_length_pointer:
        A size_t pointer FFI type of the amount of data to read. Will be
        overwritten with the amount of data read on return.

    :return:
        An integer status code of the result - 0 for success
    """

    self = None
    try:
        self = _connection_refs.get(connection_id)
        if not self:
            socket = _socket_refs.get(connection_id)
        else:
            socket = self._socket

        if not self and not socket:
            return 0

        bytes_requested = deref(data_length_pointer)

        timeout = socket.gettimeout()
        error = None
        data = b''
        try:
            while len(data) < bytes_requested:
                # Python 2 on Travis CI seems to have issues with blocking on
                # recv() for longer than the socket timeout value, so we select
                if timeout is not None and timeout > 0.0:
                    read_ready, _, _ = select.select([socket], [], [], timeout)
                    if len(read_ready) == 0:
                        raise socket_.error(errno.EAGAIN, 'timed out')
                chunk = socket.recv(bytes_requested - len(data))
                data += chunk
                if chunk == b'':
                    if len(data) == 0:
                        if timeout is None:
                            return SecurityConst.errSSLClosedNoNotify
                        return SecurityConst.errSSLClosedAbort
                    break
        except (socket_.error) as e:
            error = e.errno

        if error is not None and error != errno.EAGAIN:
            if error == errno.ECONNRESET or error == errno.EPIPE:
                return SecurityConst.errSSLClosedNoNotify
            return SecurityConst.errSSLClosedAbort

        if self and not self._done_handshake:
            # SecureTransport doesn't bother to check if the TLS record header
            # is valid before asking to read more data, which can result in
            # connection hangs. Here we do basic checks to get around the issue.
            if len(data) >= 3 and len(self._server_hello) == 0:
                # Check to ensure it is an alert or handshake first
                valid_record_type = data[0:1] in set([b'\x15', b'\x16'])
                # Check if the protocol version is SSL 3.0 or TLS 1.0-1.3
                valid_protocol_version = data[1:3] in set([
                    b'\x03\x00',
                    b'\x03\x01',
                    b'\x03\x02',
                    b'\x03\x03',
                    b'\x03\x04'
                ])
                if not valid_record_type or not valid_protocol_version:
                    self._server_hello += data + _read_remaining(socket)
                    return SecurityConst.errSSLProtocol
            self._server_hello += data

        write_to_buffer(data_buffer, data)
        pointer_set(data_length_pointer, len(data))

        if len(data) != bytes_requested:
            return SecurityConst.errSSLWouldBlock

        return 0
    except (KeyboardInterrupt) as e:
        if self:
            self._exception = e
        return SecurityConst.errSSLClosedAbort


def _read_remaining(socket):
    """
    Reads everything available from the socket - used for debugging when there
    is a protocol error

    :param socket:
        The socket to read from

    :return:
        A byte string of the remaining data
    """

    output = b''
    old_timeout = socket.gettimeout()
    try:
        socket.settimeout(0.0)
        output += socket.recv(8192)
    except (socket_.error):
        pass
    finally:
        socket.settimeout(old_timeout)
    return output

def _read_remaining ( socket ) : output = b'' old_timeout = socket . gettimeout ( ) try : socket . settimeout ( 0.0 ) output += socket . recv ( 8192 ) except ( socket_ . error ) : pass finally : socket . settimeout ( old_timeout ) return output


def _write_callback(connection_id, data_buffer, data_length_pointer):
    """
    Callback called by Secure Transport to actually write to the socket

    :param connection_id:
        An integer identifing the connection

    :param data_buffer:
        A char pointer FFI type containing the data to write

    :param data_length_pointer:
        A size_t pointer FFI type of the amount of data to write. Will be
        overwritten with the amount of data actually written on return.

    :return:
        An integer status code of the result - 0 for success
    """

    try:
        self = _connection_refs.get(connection_id)
        if not self:
            socket = _socket_refs.get(connection_id)
        else:
            socket = self._socket

        if not self and not socket:
            return 0

        data_length = deref(data_length_pointer)
        data = bytes_from_buffer(data_buffer, data_length)

        if self and not self._done_handshake:
            self._client_hello += data

        error = None
        try:
            sent = socket.send(data)
        except (socket_.error) as e:
            error = e.errno

        if error is not None and error != errno.EAGAIN:
            if error == errno.ECONNRESET or error == errno.EPIPE:
                return SecurityConst.errSSLClosedNoNotify
            return SecurityConst.errSSLClosedAbort

        if sent != data_length:
            pointer_set(data_length_pointer, sent)
            return SecurityConst.errSSLWouldBlock

        return 0
    except (KeyboardInterrupt) as e:
        self._exception = e
        return SecurityConst.errSSLPeerUserCancelled

def _write_callback ( connection_id , data_buffer , data_length_pointer ) : try : self = _connection_refs . get ( connection_id ) if not self : socket = _socket_refs . get ( connection_id ) else : socket = self . _socket if not self and not socket : return 0 data_length = deref ( data_length_pointer ) data = bytes_from_buffer ( data_buffer , data_length ) if self and not self . _done_handshake : self . _client_hello += data error = None try : sent = socket . send ( data ) except ( socket_ . error ) as e : error = e . errno if error is not None and error != errno . EAGAIN : if error == errno . ECONNRESET or error == errno . EPIPE : return SecurityConst . errSSLClosedNoNotify return SecurityConst . errSSLClosedAbort if sent != data_length : pointer_set ( data_length_pointer , sent ) return SecurityConst . errSSLWouldBlock return 0 except ( KeyboardInterrupt ) as e : self . _exception = e return SecurityConst . errSSLPeerUserCancelled


def get_path(temp_dir=None, cache_length=24, cert_callback=None):
    """
    Get the filesystem path to a file that contains OpenSSL-compatible CA certs.

    On OS X and Windows, there are extracted from the system certificate store
    and cached in a file on the filesystem. This path should not be writable
    by other users, otherwise they could inject CA certs into the trust list.

    :param temp_dir:
        The temporary directory to cache the CA certs in on OS X and Windows.
        Needs to have secure permissions so other users can not modify the
        contents.

    :param cache_length:
        The number of hours to cache the CA certs on OS X and Windows

    :param cert_callback:
        A callback that is called once for each certificate in the trust store.
        It should accept two parameters: an asn1crypto.x509.Certificate object,
        and a reason. The reason will be None if the certificate is being
        exported, otherwise it will be a unicode string of the reason it won't.
        This is only called on Windows and OS X when passed to this function.

    :raises:
        oscrypto.errors.CACertsError - when an error occurs exporting/locating certs

    :return:
        The full filesystem path to a CA certs file
    """

    ca_path, temp = _ca_path(temp_dir)

    # Windows and OS X
    if temp and _cached_path_needs_update(ca_path, cache_length):
        empty_set = set()

        any_purpose = '2.5.29.37.0'
        apple_ssl = '1.2.840.113635.100.1.3'
        win_server_auth = '1.3.6.1.5.5.7.3.1'

        with path_lock:
            if _cached_path_needs_update(ca_path, cache_length):
                with open(ca_path, 'wb') as f:
                    for cert, trust_oids, reject_oids in extract_from_system(cert_callback, True):
                        if sys.platform == 'darwin':
                            if trust_oids != empty_set and any_purpose not in trust_oids \
                                    and apple_ssl not in trust_oids:
                                if cert_callback:
                                    cert_callback(Certificate.load(cert), 'implicitly distrusted for TLS')
                                continue
                            if reject_oids != empty_set and (apple_ssl in reject_oids
                                                             or any_purpose in reject_oids):
                                if cert_callback:
                                    cert_callback(Certificate.load(cert), 'explicitly distrusted for TLS')
                                continue
                        elif sys.platform == 'win32':
                            if trust_oids != empty_set and any_purpose not in trust_oids \
                                    and win_server_auth not in trust_oids:
                                if cert_callback:
                                    cert_callback(Certificate.load(cert), 'implicitly distrusted for TLS')
                                continue
                            if reject_oids != empty_set and (win_server_auth in reject_oids
                                                             or any_purpose in reject_oids):
                                if cert_callback:
                                    cert_callback(Certificate.load(cert), 'explicitly distrusted for TLS')
                                continue
                        if cert_callback:
                            cert_callback(Certificate.load(cert), None)
                        f.write(armor('CERTIFICATE', cert))

    if not ca_path:
        raise CACertsError('No CA certs found')

    return ca_path

def
get_path
(
temp_dir
=
,
cache_length
=
24
,
cert_callback
=
)
:
ca_path
,
temp
=
_ca_path
(
temp_dir
)
# Windows and OS X
if
temp
and
_cached_path_needs_update
(
ca_path
,
cache_length
)
:
empty_set
=
set
(
)
any_purpose
=
'2.5.29.37.0'
apple_ssl
=
'1.2.840.113635.100.1.3'
win_server_auth
=
'1.3.6.1.5.5.7.3.1'
with
path_lock
:
if
_cached_path_needs_update
(
ca_path
,
cache_length
)
:
with
open
(
ca_path
,
'wb'
)
as
f
:
for
cert
,
trust_oids
,
reject_oids
in
extract_from_system
(
cert_callback
,
True
)
:
if
sys
.
platform
==
'darwin'
:
if
trust_oids
!=
empty_set
and
any_purpose
not
in
trust_oids
and
apple_ssl
not
in
trust_oids
:
if
cert_callback
:
cert_callback
(
Certificate
.
load
(
cert
)
,
'implicitly distrusted for TLS'
)
continue
if
reject_oids
!=
empty_set
and
(
apple_ssl
in
reject_oids
or
any_purpose
in
reject_oids
)
:
if
cert_callback
:
cert_callback
(
Certificate
.
load
(
cert
)
,
'explicitly distrusted for TLS'
)
continue
elif
sys
.
platform
==
'win32'
:
if
trust_oids
!=
empty_set
and
any_purpose
not
in
trust_oids
and
win_server_auth
not
in
trust_oids
:
if
cert_callback
:
cert_callback
(
Certificate
.
load
(
cert
)
,
'implicitly distrusted for TLS'
)
continue
if
reject_oids
!=
empty_set
and
(
win_server_auth
in
reject_oids
or
any_purpose
in
reject_oids
)
:
if
cert_callback
:
cert_callback
(
Certificate
.
load
(
cert
)
,
'explicitly distrusted for TLS'
)
continue
if
cert_callback
:
cert_callback
(
Certificate
.
load
(
cert
)
,
)
f
.
write
(
armor
(
'CERTIFICATE'
,
cert
)
)
if
not
ca_path
:
raise
CACertsError
(
'No CA certs found'
)
return
ca_path

Get the filesystem path to a file that contains OpenSSL - compatible CA certs .


def _map_oids ( oids ) : new_oids = set ( ) for oid in oids : if oid in _oid_map : new_oids |= _oid_map [ oid ] return oids | new_oids

Takes a set of unicode string OIDs and converts vendor - specific OIDs into generics OIDs from RFCs .


def _cached_path_needs_update(ca_path, cache_length):
    """
    Checks to see if a cache file needs to be refreshed

    :param ca_path:
        A unicode string of the path to the cache file

    :param cache_length:
        An integer representing the number of hours the cache is valid for

    :return:
        A boolean - True if the cache needs to be updated, False if the file
        is up-to-date
    """

    exists = os.path.exists(ca_path)
    if not exists:
        return True

    stats = os.stat(ca_path)

    if stats.st_mtime < time.time() - cache_length * 60 * 60:
        return True

    if stats.st_size == 0:
        return True

    return False

def _cached_path_needs_update ( ca_path , cache_length ) : exists = os . path . exists ( ca_path ) if not exists : return True stats = os . stat ( ca_path ) if stats . st_mtime < time . time ( ) - cache_length * 60 * 60 : return True if stats . st_size == 0 : return True return False


def version(self):
        """ Create a new version under this service. """
        ver = Version()
        ver.conn = self.conn

        ver.attrs = {
            # Parent params
            'service_id': self.attrs['id'],
        }

        ver.save()

        return ver

def
version
(
self
)
:
ver
=
Version
(
)
ver
.
conn
=
self
.
conn
ver
.
attrs
=
{
# Parent params
'service_id'
:
self
.
attrs
[
'id'
]
,
}
ver
.
save
(
)
return
ver

Create a new version under this service .


def vcl(self, name, content):
        """ Create a new VCL under this version. """
        vcl = VCL()
        vcl.conn = self.conn

        vcl.attrs = {
            # Parent params
            'service_id': self.attrs['service_id'],
            'version': self.attrs['number'],

            # New instance params
            'name': name,
            'content': content,
        }

        vcl.save()

        return vcl

def
vcl
(
self
,
name
,
content
)
:
vcl
=
VCL
(
)
vcl
.
conn
=
self
.
conn
vcl
.
attrs
=
{
# Parent params
'service_id'
:
self
.
attrs
[
'service_id'
]
,
'version'
:
self
.
attrs
[
'number'
]
,
# New instance params
'name'
:
name
,
'content'
:
content
,
}
vcl
.
save
(
)
return
vcl


def to_dict(self):
        """
        Converts the column to a dictionary representation accepted
        by the Citrination server.

        :return: Dictionary with basic options, plus any column type specific
            options held under the "options" key
        :rtype: dict
        """
        return {
            "type": self.type,
            "name": self.name,
            "group_by_key": self.group_by_key,
            "role": self.role,
            "units": self.units,
            "options": self.build_options()
        }

def to_dict ( self ) : return { "type" : self . type , "name" : self . name , "group_by_key" : self . group_by_key , "role" : self . role , "units" : self . units , "options" : self . build_options ( ) }

Converts the column to a dictionary representation accepted by the Citrination server .


def add_descriptor(self, descriptor, role='ignore', group_by_key=False):
        """
        Add a descriptor column.

        :param descriptor: A Descriptor instance (e.g., RealDescriptor, InorganicDescriptor, etc.)
        :param role: Specify a role (input, output, latentVariable, or ignore)
        :param group_by_key: Whether or not to group by this key during cross validation
        """

        descriptor.validate()

        if descriptor.key in self.configuration["roles"]:
            raise ValueError("Cannot add a descriptor with the same name twice")

        self.configuration['descriptors'].append(descriptor.as_dict())
        self.configuration["roles"][descriptor.key] = role

        if group_by_key:
            self.configuration["group_by"].append(descriptor.key)


def _patch(self, route, data, headers=None, failure_message=None):
        """
        Execute a patch request and return the result
        """
        headers = self._get_headers(headers)
        response_lambda = (
            lambda: requests.patch(
                self._get_qualified_route(route), headers=headers, data=data, verify=False, proxies=self.proxies
            )
        )
        response = check_for_rate_limiting(response_lambda(), response_lambda)
        return self._handle_response(response, failure_message)

def _patch ( self , route , data , headers = None , failure_message = None ) : headers = self . _get_headers ( headers ) response_lambda = ( lambda : requests . patch ( self . _get_qualified_route ( route ) , headers = headers , data = data , verify = False , proxies = self . proxies ) ) response = check_for_rate_limiting ( response_lambda ( ) , response_lambda ) return self . _handle_response ( response , failure_message )


def _validate_search_query(self, returning_query):
        """
        Checks to see that the query will not exceed the max query depth

        :param returning_query: The PIF system or Dataset query to execute.
        :type returning_query: :class:`PifSystemReturningQuery` or :class: `DatasetReturningQuery`
        """

        start_index = returning_query.from_index or 0
        size = returning_query.size or 0

        if start_index < 0:
            raise CitrinationClientError(
                "start_index cannot be negative. Please enter a value greater than or equal to zero")
        if size < 0:
            raise CitrinationClientError("Size cannot be negative. Please enter a value greater than or equal to zero")
        if start_index + size > MAX_QUERY_DEPTH:
            raise CitrinationClientError(
                "Citrination does not support pagination past the {0}th result. Please reduce either the from_index and/or size such that their sum is below {0}".format(
                    MAX_QUERY_DEPTH))


def dataset_search(self, dataset_returning_query):
        """
        Run a dataset query against Citrination.

        :param dataset_returning_query: :class:`DatasetReturningQuery` to execute.
        :type dataset_returning_query: :class:`DatasetReturningQuery`
        :return: Dataset search result object with the results of the query.
        :rtype: :class:`DatasetSearchResult`
        """

        self._validate_search_query(dataset_returning_query)
        return self._execute_search_query(
            dataset_returning_query,
            DatasetSearchResult
        )

def dataset_search ( self , dataset_returning_query ) : self . _validate_search_query ( dataset_returning_query ) return self . _execute_search_query ( dataset_returning_query , DatasetSearchResult )

Run a dataset query against Citrination .


def pif_multi_search(self, multi_query):
        """
        Run each in a list of PIF queries against Citrination.

        :param multi_query: :class:`MultiQuery` object to execute.
        :return: :class:`PifMultiSearchResult` object with the results of the query.
        """
        failure_message = "Error while making PIF multi search request"
        response_dict = self._get_success_json(
            self._post(routes.pif_multi_search, data=json.dumps(multi_query, cls=QueryEncoder),
                       failure_message=failure_message))

        return PifMultiSearchResult(**keys_to_snake_case(response_dict['results']))

Run each in a list of PIF queries against Citrination .


def check_for_rate_limiting(response, response_lambda, timeout=1, attempts=0):
    """
    Takes an initial response, and a way to repeat the request that produced it and retries the request with an increasing sleep period between requests if rate limiting resposne codes are encountered.

    If more than 3 attempts are made, a RateLimitingException is raised

    :param response: A response from Citrination
    :type response: requests.Response
    :param response_lambda: a callable that runs the request that returned the
        response
    :type response_lambda: function
    :param timeout: the time to wait before retrying
    :type timeout: int
    :param attempts: the number of the retry being executed
    :type attempts: int
    """
    if attempts >= 3:
        raise RateLimitingException()
    if response.status_code == 429:
        sleep(timeout)
        new_timeout = timeout + 1
        new_attempts = attempts + 1
        return check_for_rate_limiting(response_lambda(timeout, attempts), response_lambda, timeout=new_timeout, attempts=new_attempts)
    return response

def check_for_rate_limiting ( response , response_lambda , timeout = 1 , attempts = 0 ) : if attempts >= 3 : raise RateLimitingException ( ) if response . status_code == 429 : sleep ( timeout ) new_timeout = timeout + 1 new_attempts = attempts + 1 return check_for_rate_limiting ( response_lambda ( timeout , attempts ) , response_lambda , timeout = new_timeout , attempts = new_attempts ) return response

Takes an initial response and a way to repeat the request that produced it and retries the request with an increasing sleep period between requests if rate limiting resposne codes are encountered .


def create(self, configuration, name, description):
        """
        Creates a data view from the search template and ml template given

        :param configuration: Information to construct the data view from (eg descriptors, datasets etc)
        :param name: Name of the data view
        :param description: Description for the data view
        :return: The data view id
        """

        data = {
            "configuration":
                configuration,
            "name":
                name,
            "description":
                description
        }

        failure_message = "Dataview creation failed"

        result = self._get_success_json(self._post_json(
            'v1/data_views', data, failure_message=failure_message))
        data_view_id = result['data']['id']

        return data_view_id

def
create
(
self
,
configuration
,
name
,
description
)
:
data
=
{
"configuration"
:
configuration
,
"name"
:
name
,
"description"
:
description
}
failure_message
=
"Dataview creation failed"
result
=
self
.
_get_success_json
(
self
.
_post_json
(
'v1/data_views'
,
data
,
failure_message
=
failure_message
)
)
data_view_id
=
result
[
'data'
]
[
'id'
]
return
data_view_id


def update(self, id, configuration, name, description):
        """
        Updates an existing data view from the search template and ml template given

        :param id: Identifier for the data view.  This returned from the create method.
        :param configuration: Information to construct the data view from (eg descriptors, datasets etc)
        :param name: Name of the data view
        :param description: Description for the data view
        """

        data = {
            "configuration":
                configuration,
            "name":
                name,
            "description":
                description
        }

        failure_message = "Dataview creation failed"

        self._patch_json(
            'v1/data_views/' + id, data, failure_message=failure_message)

def
update
(
self
,
id
,
configuration
,
name
,
description
)
:
data
=
{
"configuration"
:
configuration
,
"name"
:
name
,
"description"
:
description
}
failure_message
=
"Dataview creation failed"
self
.
_patch_json
(
'v1/data_views/'
+
id
,
data
,
failure_message
=
failure_message
)


def get(self, data_view_id):
        """
        Gets basic information about a view

        :param data_view_id: Identifier of the data view
        :return: Metadata about the view as JSON
        """

        failure_message = "Dataview get failed"
        return self._get_success_json(self._get(
            'v1/data_views/' + data_view_id, None, failure_message=failure_message))['data']['data_view']

def
get
(
self
,
data_view_id
)
:
failure_message
=
"Dataview get failed"
return
self
.
_get_success_json
(
self
.
_get
(
'v1/data_views/'
+
data_view_id
,
,
failure_message
=
failure_message
)
)
[
'data'
]
[
'data_view'
]


def create_ml_configuration_from_datasets(self, dataset_ids):
        """
        Creates an ml configuration from dataset_ids and extract_as_keys

        :param dataset_ids: Array of dataset identifiers to make search template from
        :return: An identifier used to request the status of the builder job (get_ml_configuration_status)
        """
        available_columns = self.search_template_client.get_available_columns(dataset_ids)

        # Create a search template from dataset ids
        search_template = self.search_template_client.create(dataset_ids, available_columns)
        return self.create_ml_configuration(search_template, available_columns, dataset_ids)


def create_ml_configuration(self, search_template, extract_as_keys, dataset_ids):
        """
        This method will spawn a server job to create a default ML configuration based on a search template and
        the extract as keys.
        This function will submit the request to build, and wait for the configuration to finish before returning.

        :param search_template: A search template defining the query (properties, datasets etc)
        :param extract_as_keys: Array of extract-as keys defining the descriptors
        :param dataset_ids: Array of dataset identifiers to make search template from
        :return: An identifier used to request the status of the builder job (get_ml_configuration_status)
        """
        data = {
            "search_template":
                search_template,
            "extract_as_keys":
                extract_as_keys
        }

        failure_message = "ML Configuration creation failed"
        config_job_id = self._get_success_json(self._post_json(
            'v1/descriptors/builders/simple/default/trigger', data, failure_message=failure_message))['data'][
            'result']['uid']

        while True:
            config_status = self.__get_ml_configuration_status(config_job_id)
            print('Configuration status: ', config_status)
            if config_status['status'] == 'Finished':
                ml_config = self.__convert_response_to_configuration(config_status['result'], dataset_ids)
                return ml_config
            time.sleep(5)

This method will spawn a server job to create a default ML configuration based on a search template and the extract as keys . This function will submit the request to build and wait for the configuration to finish before returning .


def __convert_response_to_configuration(self, result_blob, dataset_ids):
        """
        Utility function to turn the result object from the configuration builder endpoint into something that
        can be used directly as a configuration.

        :param result_blob: Nested dicts representing the possible descriptors
        :param dataset_ids: Array of dataset identifiers to make search template from
        :return: An object suitable to be used as a parameter to data view create
        """

        builder = DataViewBuilder()
        builder.dataset_ids(dataset_ids)
        for i, (k, v) in enumerate(result_blob['descriptors'].items()):
            try:
                descriptor = self.__snake_case(v[0])
                print(json.dumps(descriptor))
                descriptor['descriptor_key'] = k
                builder.add_raw_descriptor(descriptor)
            except IndexError:
                pass

        for i, (k, v) in enumerate(result_blob['types'].items()):
            builder.set_role(k, v.lower())

        return builder.build()

def __convert_response_to_configuration ( self , result_blob , dataset_ids ) : builder = DataViewBuilder ( ) builder . dataset_ids ( dataset_ids ) for i , ( k , v ) in enumerate ( result_blob [ 'descriptors' ] . items ( ) ) : try : descriptor = self . __snake_case ( v [ 0 ] ) print ( json . dumps ( descriptor ) ) descriptor [ 'descriptor_key' ] = k builder . add_raw_descriptor ( descriptor ) except IndexError : pass for i , ( k , v ) in enumerate ( result_blob [ 'types' ] . items ( ) ) : builder . set_role ( k , v . lower ( ) ) return builder . build ( )

Utility function to turn the result object from the configuration builder endpoint into something that can be used directly as a configuration .


def __get_ml_configuration_status(self, job_id):
        """
        After invoking the create_ml_configuration async method, you can use this method to
        check on the status of the builder job.

        :param job_id: The identifier returned from create_ml_configuration
        :return: Job status
        """

        failure_message = "Get status on ml configuration failed"
        response = self._get_success_json(self._get(
            'v1/descriptors/builders/simple/default/' + job_id + '/status', None, failure_message=failure_message))[
            'data']
        return response

After invoking the create_ml_configuration async method you can use this method to check on the status of the builder job .


def tsne(self, data_view_id):
        """
        Get the t-SNE projection, including responses and tags.

        :param data_view_id: The ID of the data view to retrieve TSNE from
        :type data_view_id: int
        :return: The TSNE analysis
        :rtype: :class:`Tsne`
        """
        analysis = self._data_analysis(data_view_id)
        projections = analysis['projections']
        tsne = Tsne()
        for k, v in projections.items():
            projection = Projection(
                xs=v['x'],
                ys=v['y'],
                responses=v['label'],
                tags=v['inputs'],
                uids=v['uid']
            )
            tsne.add_projection(k, projection)

        return tsne

def tsne ( self , data_view_id ) : analysis = self . _data_analysis ( data_view_id ) projections = analysis [ 'projections' ] tsne = Tsne ( ) for k , v in projections . items ( ) : projection = Projection ( xs = v [ 'x' ] , ys = v [ 'y' ] , responses = v [ 'label' ] , tags = v [ 'inputs' ] , uids = v [ 'uid' ] ) tsne . add_projection ( k , projection ) return tsne

Get the t - SNE projection including responses and tags .


def _data_analysis(self, data_view_id):
        """
        Data analysis endpoint.

        :param data_view_id: The model identifier (id number for data views)
        :type data_view_id: str
        :return: dictionary containing information about the data, e.g. dCorr and tsne
        """
        failure_message = "Error while retrieving data analysis for data view {}".format(data_view_id)
        return self._get_success_json(self._get(routes.data_analysis(data_view_id), failure_message=failure_message))


def submit_predict_request(self, data_view_id, candidates, prediction_source='scalar', use_prior=True):
        """
        Submits an async prediction request.

        :param data_view_id: The id returned from create
        :param candidates: Array of candidates
        :param prediction_source: 'scalar' or 'scalar_from_distribution'
        :param use_prior: True to use prior prediction, otherwise False
        :return: Predict request Id (used to check status)
        """

        data = {
            "prediction_source":
                prediction_source,
            "use_prior":
                use_prior,
            "candidates":
                candidates
        }

        failure_message = "Configuration creation failed"
        post_url = 'v1/data_views/' + str(data_view_id) + '/predict/submit'
        return self._get_success_json(
            self._post_json(post_url, data, failure_message=failure_message)
        )['data']['uid']

def
submit_predict_request
(
self
,
data_view_id
,
candidates
,
prediction_source
=
'scalar'
,
use_prior
=
True
)
:
data
=
{
"prediction_source"
:
prediction_source
,
"use_prior"
:
use_prior
,
"candidates"
:
candidates
}
failure_message
=
"Configuration creation failed"
post_url
=
'v1/data_views/'
+
str
(
data_view_id
)
+
'/predict/submit'
return
self
.
_get_success_json
(
self
.
_post_json
(
post_url
,
data
,
failure_message
=
failure_message
)
)
[
'data'
]
[
'uid'
]


def check_predict_status(self, view_id, predict_request_id):
        """
        Returns a string indicating the status of the prediction job

        :param view_id: The data view id returned from data view create
        :param predict_request_id: The id returned from predict
        :return: Status data, also includes results if state is finished
        """

        failure_message = "Get status on predict failed"

        bare_response = self._get_success_json(self._get(
            'v1/data_views/' + str(view_id) + '/predict/' + str(predict_request_id) + '/status',
            None, failure_message=failure_message))

        result = bare_response["data"]
        # result.update({"message": bare_response["message"]})

        return result


def submit_design_run(self, data_view_id, num_candidates, effort, target=None, constraints=[], sampler="Default"):
        """
        Submits a new experimental design run.

        :param data_view_id: The ID number of the data view to which the
            run belongs, as a string
        :type data_view_id: str
        :param num_candidates: The number of candidates to return
        :type num_candidates: int
        :param target: An :class:``Target`` instance representing
            the design run optimization target
        :type target: :class:``Target``
        :param constraints: An array of design constraints (instances of
            objects which extend :class:``BaseConstraint``)
        :type constraints: list of :class:``BaseConstraint``
        :param sampler: The name of the sampler to use during the design run:
            either "Default" or "This view"
        :type sampler: str
        :return: A :class:`DesignRun` instance containing the UID of the
            new run
        """
        if effort > 30:
            raise CitrinationClientError("Parameter effort must be less than 30 to trigger a design run")

        if target is not None:
            target = target.to_dict()

        constraint_dicts = [c.to_dict() for c in constraints]

        body = {
            "num_candidates": num_candidates,
            "target": target,
            "effort": effort,
            "constraints": constraint_dicts,
            "sampler": sampler
        }

        url = routes.submit_data_view_design(data_view_id)

        response = self._post_json(url, body).json()

        return DesignRun(response["data"]["design_run"]["uid"])


def get_design_run_status(self, data_view_id, run_uuid):
        """
        Retrieves the status of an in progress or completed design run

        :param data_view_id: The ID number of the data view to which the
            run belongs, as a string
        :type data_view_id: str
        :param run_uuid: The UUID of the design run to retrieve status for
        :type run_uuid: str
        :return: A :class:`ProcessStatus` object
        """

        url = routes.get_data_view_design_status(data_view_id, run_uuid)

        response = self._get(url).json()

        status = response["data"]

        return ProcessStatus(
            result=status.get("result"),
            progress=status.get("progress"),
            status=status.get("status"),
            messages=status.get("messages")
        )

def get_design_run_status ( self , data_view_id , run_uuid ) : url = routes . get_data_view_design_status ( data_view_id , run_uuid ) response = self . _get ( url ) . json ( ) status = response [ "data" ] return ProcessStatus ( result = status . get ( "result" ) , progress = status . get ( "progress" ) , status = status . get ( "status" ) , messages = status . get ( "messages" ) )


def get_design_run_results(self, data_view_id, run_uuid):
        """
        Retrieves the results of an existing designrun

        :param data_view_id: The ID number of the data view to which the
            run belongs, as a string
        :type data_view_id: str
        :param run_uuid: The UUID of the design run to retrieve results from
        :type run_uuid: str
        :return: A :class:`DesignResults` object
        """

        url = routes.get_data_view_design_results(data_view_id, run_uuid)

        response = self._get(url).json()

        result = response["data"]

        return DesignResults(
            best_materials=result.get("best_material_results"),
            next_experiments=result.get("next_experiment_results")
        )

def get_design_run_results ( self , data_view_id , run_uuid ) : url = routes . get_data_view_design_results ( data_view_id , run_uuid ) response = self . _get ( url ) . json ( ) result = response [ "data" ] return DesignResults ( best_materials = result . get ( "best_material_results" ) , next_experiments = result . get ( "next_experiment_results" ) )


def get_data_view(self, data_view_id):
        """
        Retrieves a summary of information for a given data view
            - view id
            - name
            - description
            - columns

        :param data_view_id: The ID number of the data view to which the
            run belongs, as a string
        :type data_view_id: str
        """

        url = routes.get_data_view(data_view_id)

        response = self._get(url).json()

        result = response["data"]["data_view"]

        datasets_list = []
        for dataset in result["datasets"]:
            datasets_list.append(Dataset(
                name=dataset["name"],
                id=dataset["id"],
                description=dataset["description"]
            ))

        columns_list = []
        for column in result["columns"]:
            columns_list.append(ColumnFactory.from_dict(column))

        return DataView(
            view_id=data_view_id,
            name=result["name"],
            description=result["description"],
            datasets=datasets_list,
            columns=columns_list,
        )

def get_data_view ( self , data_view_id ) : url = routes . get_data_view ( data_view_id ) response = self . _get ( url ) . json ( ) result = response [ "data" ] [ "data_view" ] datasets_list = [ ] for dataset in result [ "datasets" ] : datasets_list . append ( Dataset ( name = dataset [ "name" ] , id = dataset [ "id" ] , description = dataset [ "description" ] ) ) columns_list = [ ] for column in result [ "columns" ] : columns_list . append ( ColumnFactory . from_dict ( column ) ) return DataView ( view_id = data_view_id , name = result [ "name" ] , description = result [ "description" ] , datasets = datasets_list , columns = columns_list , )


def kill_design_run(self, data_view_id, run_uuid):
        """
        Kills an in progress experimental design run

        :param data_view_id: The ID number of the data view to which the
            run belongs, as a string
        :type data_view_id: str
        :param run_uuid: The UUID of the design run to kill
        :type run_uuid: str
        :return: The UUID of the design run
        """

        url = routes.kill_data_view_design_run(data_view_id, run_uuid)

        response = self._delete(url).json()
        return response["data"]["uid"]

def kill_design_run ( self , data_view_id , run_uuid ) : url = routes . kill_data_view_design_run ( data_view_id , run_uuid ) response = self . _delete ( url ) . json ( ) return response [ "data" ] [ "uid" ]


def load_file_as_yaml(path):
    """
    Given a filepath, loads the file as a dictionary from YAML

    :param path: The path to a YAML file
    """
    with open(path, "r") as f:
      raw_yaml = f.read()
      parsed_dict = yaml.load(raw_yaml)
    return parsed_dict

def load_file_as_yaml ( path ) : with open ( path , "r" ) as f : raw_yaml = f . read ( ) parsed_dict = yaml . load ( raw_yaml ) return parsed_dict

Given a filepath loads the file as a dictionary from YAML


def get_credentials_from_file(filepath):
    """
    Extracts credentials from the yaml formatted credential filepath
    passed in. Uses the default profile if the CITRINATION_PROFILE env var
    is not set, otherwise looks for a profile with that name in the credentials file.

    :param filepath: The path of the credentials file
    """
    try:
        creds = load_file_as_yaml(filepath)
    except Exception:
        creds = {}

    profile_name = os.environ.get(citr_env_vars.CITRINATION_PROFILE)
    if profile_name is None or len(profile_name) == 0:
        profile_name = DEFAULT_CITRINATION_PROFILE
    api_key = None
    site = None
    try:
        profile = creds[profile_name]
        api_key = profile[CREDENTIALS_API_KEY_KEY]
        site = profile[CREDENTIALS_SITE_KEY]
    except KeyError:
        pass

    return (api_key, site)

def get_credentials_from_file ( filepath ) : try : creds = load_file_as_yaml ( filepath ) except Exception : creds = { } profile_name = os . environ . get ( citr_env_vars . CITRINATION_PROFILE ) if profile_name is None or len ( profile_name ) == 0 : profile_name = DEFAULT_CITRINATION_PROFILE api_key = None site = None try : profile = creds [ profile_name ] api_key = profile [ CREDENTIALS_API_KEY_KEY ] site = profile [ CREDENTIALS_SITE_KEY ] except KeyError : pass return ( api_key , site )

Extracts credentials from the yaml formatted credential filepath passed in . Uses the default profile if the CITRINATION_PROFILE env var is not set otherwise looks for a profile with that name in the credentials file .


def get_preferred_credentials(api_key, site, cred_file=DEFAULT_CITRINATION_CREDENTIALS_FILE):
    """
    Given an API key, a site url and a credentials file path, runs through a prioritized list of credential sources to find credentials.

    Specifically, this method ranks credential priority as follows:
        1. Those passed in as the first two parameters to this method
        2. Those found in the environment as variables
        3. Those found in the credentials file at the profile specified
           by the profile environment variable
        4. Those found in the default stanza in the credentials file

    :param api_key: A Citrination API Key or None
    :param site: A Citrination site URL or None
    :param cred_file: The path to a credentials file
    """
    profile_api_key, profile_site = get_credentials_from_file(cred_file)
    if api_key is None:
        api_key =  os.environ.get(citr_env_vars.CITRINATION_API_KEY)
    if api_key is None or len(api_key) == 0:
        api_key = profile_api_key

    if site is None:
        site = os.environ.get(citr_env_vars.CITRINATION_SITE)
    if site is None or len(site) == 0:
        site = profile_site
    if site is None:
        site = "https://citrination.com"

    return api_key, site

def get_preferred_credentials ( api_key , site , cred_file = DEFAULT_CITRINATION_CREDENTIALS_FILE ) : profile_api_key , profile_site = get_credentials_from_file ( cred_file ) if api_key is None : api_key = os . environ . get ( citr_env_vars . CITRINATION_API_KEY ) if api_key is None or len ( api_key ) == 0 : api_key = profile_api_key if site is None : site = os . environ . get ( citr_env_vars . CITRINATION_SITE ) if site is None or len ( site ) == 0 : site = profile_site if site is None : site = "https://citrination.com" return api_key , site

Given an API key a site url and a credentials file path runs through a prioritized list of credential sources to find credentials .


def list_files(self, dataset_id, glob=".", is_dir=False):
        """
        List matched filenames in a dataset on Citrination.

        :param dataset_id: The ID of the dataset to search for files.
        :type dataset_id: int
        :param glob: A pattern which will be matched against files in the dataset.
        :type glob: str
        :param is_dir: A boolean indicating whether or not the pattern should match against the beginning of paths in the dataset.
        :type is_dir: bool
        :return: A list of filepaths in the dataset matching the provided glob.
        :rtype: list of strings
        """
        data = {
            "list": {
                "glob": glob,
                "isDir": is_dir
            }
        }
        return self._get_success_json(self._post_json(routes.list_files(dataset_id), data, failure_message="Failed to list files for dataset {}".format(dataset_id)))['files']

def
list_files
(
self
,
dataset_id
,
glob
=
"."
,
is_dir
=
False
)
:
data
=
{
"list"
:
{
"glob"
:
glob
,
"isDir"
:
is_dir
}
}
return
self
.
_get_success_json
(
self
.
_post_json
(
routes
.
list_files
(
dataset_id
)
,
data
,
failure_message
=
"Failed to list files for dataset {}"
.
format
(
dataset_id
)
)
)
[
'files'
]

List matched filenames in a dataset on Citrination .


def matched_file_count(self, dataset_id, glob=".", is_dir=False):
        """
        Returns the number of files matching a pattern in a dataset.

        :param dataset_id: The ID of the dataset to search for files.
        :type dataset_id: int
        :param glob: A pattern which will be matched against files in the dataset.
        :type glob: str
        :param is_dir: A boolean indicating whether or not the pattern should match against the beginning of paths in the dataset.
        :type is_dir: bool
        :return: The number of matching files
        :rtype: int
        """
        list_result = self.list_files(dataset_id, glob, is_dir)
        return len(list_result)

def matched_file_count ( self , dataset_id , glob = "." , is_dir = False ) : list_result = self . list_files ( dataset_id , glob , is_dir ) return len ( list_result )

Returns the number of files matching a pattern in a dataset .


def get_dataset_files(self, dataset_id, glob=".", is_dir=False, version_number=None):
        """
        Retrieves URLs for the files matched by a glob or a path to a directory
        in a given dataset.

        :param dataset_id: The id of the dataset to retrieve files from
        :type dataset_id: int
        :param glob: A regex used to select one or more files in the dataset
        :type glob: str
        :param is_dir: Whether or not the supplied pattern should be treated as a directory to search in
        :type is_dir: bool
        :param version_number: The version number of the dataset to retrieve files from
        :type version_number: int
        :return: A list of dataset files whose paths match the provided pattern.
        :rtype: list of :class:`DatasetFile`
        """
        if version_number is None:
            latest = True
        else:
            latest = False

        data = {
            "download_request": {
                "glob": glob,
                "isDir": is_dir,
                "latest": latest
            }
        }

        failure_message = "Failed to get matched files in dataset {}".format(dataset_id)

        versions = self._get_success_json(self._post_json(routes.matched_files(dataset_id), data, failure_message=failure_message))['versions']

        # if you don't provide a version number, only the latest
        # will be included in the response body
        if version_number is None:
            version = versions[0]
        else:
            try:
                version = list(filter(lambda v: v['number'] == version_number, versions))[0]
            except IndexError:
                raise ResourceNotFoundException()

        return list(
            map(
                lambda f: DatasetFile(path=f['filename'], url=f['url']), version['files']
                )
            )

Retrieves URLs for the files matched by a glob or a path to a directory in a given dataset .


def get_dataset_file(self, dataset_id, file_path, version = None):
        """
        Retrieves a dataset file matching a provided file path

        :param dataset_id: The id of the dataset to retrieve file from
        :type dataset_id: int
        :param file_path: The file path within the dataset
        :type file_path: str
        :param version: The dataset version to look for the file in. If nothing is supplied, the latest dataset version will be searched
        :type version: int
        :return: A dataset file matching the filepath provided
        :rtype: :class:`DatasetFile`
        """
        return self.get_dataset_files(dataset_id, "^{}$".format(file_path), version_number=version)[0]

def get_dataset_file ( self , dataset_id , file_path , version = None ) : return self . get_dataset_files ( dataset_id , "^{}$" . format ( file_path ) , version_number = version ) [ 0 ]


def get_pif(self, dataset_id, uid, dataset_version = None):
        """
        Retrieves a PIF from a given dataset.

        :param dataset_id: The id of the dataset to retrieve PIF from
        :type dataset_id: int
        :param uid: The uid of the PIF to retrieve
        :type uid: str
        :param dataset_version: The dataset version to look for the PIF in. If nothing is supplied, the latest dataset version will be searched
        :type dataset_version: int
        :return: A :class:`Pif` object
        :rtype: :class:`Pif`
        """
        failure_message = "An error occurred retrieving PIF {}".format(uid)
        if dataset_version == None:
            response = self._get(routes.pif_dataset_uid(dataset_id, uid), failure_message=failure_message)
        else:
            response = self._get(routes.pif_dataset_version_uid(dataset_id, uid, dataset_version), failure_message=failure_message)

        return pif.loads(response.content.decode("utf-8"))

def
get_pif
(
self
,
dataset_id
,
uid
,
dataset_version
=
)
:
failure_message
=
"An error occurred retrieving PIF {}"
.
format
(
uid
)
if
dataset_version
==
:
response
=
self
.
_get
(
routes
.
pif_dataset_uid
(
dataset_id
,
uid
)
,
failure_message
=
failure_message
)
else
:
response
=
self
.
_get
(
routes
.
pif_dataset_version_uid
(
dataset_id
,
uid
,
dataset_version
)
,
failure_message
=
failure_message
)
return
pif
.
loads
(
response
.
content
.
decode
(
"utf-8"
)
)


def create_dataset(self, name=None, description=None, public=False):
        """
        Create a new data set.

        :param name: name of the dataset
        :type name: str
        :param description: description for the dataset
        :type description: str
        :param public: A boolean indicating whether or not the dataset should be public.
        :type public: bool
        :return: The newly created dataset.
        :rtype: :class:`Dataset`
        """
        data = {
            "public": _convert_bool_to_public_value(public)
        }
        if name:
            data["name"] = name
        if description:
            data["description"] = description
        dataset = {"dataset": data}
        failure_message = "Unable to create dataset"
        result = self._get_success_json(self._post_json(routes.create_dataset(), dataset, failure_message=failure_message))

        return _dataset_from_response_dict(result)

def
create_dataset
(
self
,
name
=
,
description
=
,
public
=
False
)
:
data
=
{
"public"
:
_convert_bool_to_public_value
(
public
)
}
if
name
:
data
[
"name"
]
=
name
if
description
:
data
[
"description"
]
=
description
dataset
=
{
"dataset"
:
data
}
failure_message
=
"Unable to create dataset"
result
=
self
.
_get_success_json
(
self
.
_post_json
(
routes
.
create_dataset
(
)
,
dataset
,
failure_message
=
failure_message
)
)
return
_dataset_from_response_dict
(
result
)


def update_dataset(self, dataset_id, name=None, description=None, public=None):
        """
        Update a data set.

        :param dataset_id: The ID of the dataset to update
        :type dataset_id: int
        :param name: name of the dataset
        :type name: str
        :param description: description for the dataset
        :type description: str
        :param public: A boolean indicating whether or not the dataset should
            be public.
        :type public: bool
        :return: The updated dataset.
        :rtype: :class:`Dataset`
        """
        data = {
            "public": _convert_bool_to_public_value(public)
        }

        if name:
            data["name"] = name
        if description:
            data["description"] = description

        dataset = {"dataset": data}
        failure_message = "Failed to update dataset {}".format(dataset_id)
        response = self._get_success_json(self._post_json(routes.update_dataset(dataset_id), data=dataset, failure_message=failure_message))

        return _dataset_from_response_dict(response)

def
update_dataset
(
self
,
dataset_id
,
name
=
,
description
=
,
public
=
)
:
data
=
{
"public"
:
_convert_bool_to_public_value
(
public
)
}
if
name
:
data
[
"name"
]
=
name
if
description
:
data
[
"description"
]
=
description
dataset
=
{
"dataset"
:
data
}
failure_message
=
"Failed to update dataset {}"
.
format
(
dataset_id
)
response
=
self
.
_get_success_json
(
self
.
_post_json
(
routes
.
update_dataset
(
dataset_id
)
,
data
=
dataset
,
failure_message
=
failure_message
)
)
return
_dataset_from_response_dict
(
response
)


def create_dataset_version(self, dataset_id):
        """
        Create a new data set version.

        :param dataset_id: The ID of the dataset for which the version must be bumped.
        :type dataset_id: int
        :return: The new dataset version.
        :rtype: :class:`DatasetVersion`
        """
        failure_message = "Failed to create dataset version for dataset {}".format(dataset_id)
        number = self._get_success_json(self._post_json(routes.create_dataset_version(dataset_id), data={}, failure_message=failure_message))['dataset_scoped_id']

        return DatasetVersion(number=number)


def get_available_columns(self, dataset_ids):
        """
        Retrieves the set of columns from the combination of dataset ids given

        :param dataset_ids: The id of the dataset to retrieve columns from
        :type dataset_ids: list of int
        :return: A list of column names from the dataset ids given.
        :rtype: list of str
        """
        if not isinstance(dataset_ids, list):
            dataset_ids = [dataset_ids]

        data = {
            "dataset_ids":
                dataset_ids
        }

        failure_message = "Failed to get available columns in dataset(s) {}".format(dataset_ids)

        return self._get_success_json(self._post_json(
            'v1/datasets/get-available-columns', data, failure_message=failure_message))['data']


def __generate_search_template(self, dataset_ids):
        """
        Generates a default search templates from the available columns in the dataset ids given.

        :param dataset_ids: The id of the dataset to retrieve files from
        :type dataset_ids: list of int
        :return: A search template based on the columns in the datasets given
        """

        data = {
            "dataset_ids":
                dataset_ids
        }

        failure_message = "Failed to generate a search template from columns in dataset(s) {}".format(dataset_ids)

        return self._get_success_json(self._post_json(
            'v1/search_templates/builders/from-dataset-ids', data, failure_message=failure_message))['data']

Generates a default search templates from the available columns in the dataset ids given .


def __prune_search_template(self, extract_as_keys, search_template):
        """
        Returns a new search template, but the new template has only the extract_as_keys given.

        :param extract_as_keys: List of extract as keys to keep
        :param search_template: The search template to prune
        :return: New search template with pruned columns
        """

        data = {
            "extract_as_keys":
                extract_as_keys,
            "search_template":
                search_template
        }

        failure_message = "Failed to prune a search template"

        return self._get_success_json(self._post_json(
            'v1/search_templates/prune-to-extract-as', data, failure_message=failure_message))['data']

Returns a new search template but the new template has only the extract_as_keys given .


def default(self, obj):
        """
        Convert an object to a form ready to dump to json.

        :param obj: Object being serialized. The type of this object must be one of the following: None; a single object derived from the Pio class; or a list of objects, each derived from the Pio class.
        :return: List of dictionaries, each representing a physical information object, ready to be serialized.
        """
        if obj is None:
            return []
        elif isinstance(obj, list):
            return [i.as_dictionary() for i in obj]
        elif isinstance(obj, dict):
            return self._keys_to_camel_case(obj)
        else:
            return obj.as_dictionary()

def default ( self , obj ) : if obj is None : return [ ] elif isinstance ( obj , list ) : return [ i . as_dictionary ( ) for i in obj ] elif isinstance ( obj , dict ) : return self . _keys_to_camel_case ( obj ) else : return obj . as_dictionary ( )

Convert an object to a form ready to dump to json .


def _keys_to_camel_case(self, obj):
        """
        Make a copy of a dictionary with all keys converted to camel case. This is just calls to_camel_case on each of the keys in the dictionary and returns a new dictionary.

        :param obj: Dictionary to convert keys to camel case.
        :return: Dictionary with the input values and all keys in camel case
        """
        return dict((to_camel_case(key), value) for (key, value) in obj.items())

def _keys_to_camel_case ( self , obj ) : return dict ( ( to_camel_case ( key ) , value ) for ( key , value ) in obj . items ( ) )

Make a copy of a dictionary with all keys converted to camel case . This is just calls to_camel_case on each of the keys in the dictionary and returns a new dictionary .


def validate(self, ml_template):
        """
        Runs the template against the validation endpoint, returns a message indicating status of the templte

        :param ml_template: Template to validate
        :return: OK or error message if validation failed
        """

        data = {
            "ml_template":
                ml_template
        }

        failure_message = "ML template validation invoke failed"

        res = self._get_success_json(self._post_json(
            'ml_templates/validate', data, failure_message=failure_message))['data']
        if res['valid']:
            return 'OK'
        return res['reason']

def
validate
(
self
,
ml_template
)
:
data
=
{
"ml_template"
:
ml_template
}
failure_message
=
"ML template validation invoke failed"
res
=
self
.
_get_success_json
(
self
.
_post_json
(
'ml_templates/validate'
,
data
,
failure_message
=
failure_message
)
)
[
'data'
]
if
res
[
'valid'
]
:
return
'OK'
return
res
[
'reason'
]

Runs the template against the validation endpoint returns a message indicating status of the templte


def add_organization_course(organization_data, course_key):
    """
    Adds a organization-course link to the system
    """
    _validate_course_key(course_key)
    _validate_organization_data(organization_data)
    data.create_organization_course(
        organization=organization_data,
        course_key=course_key
    )

Adds a organization - course link to the system


def remove_organization_course(organization, course_key):
    """
    Removes the specfied course from the specified organization
    """
    _validate_organization_data(organization)
    _validate_course_key(course_key)
    return data.delete_organization_course(course_key=course_key, organization=organization)

def remove_organization_course ( organization , course_key ) : _validate_organization_data ( organization ) _validate_course_key ( course_key ) return data . delete_organization_course ( course_key = course_key , organization = organization )


def course_key_is_valid(course_key):
    """
    Course key object validation
    """
    if course_key is None:
        return False
    try:
        CourseKey.from_string(text_type(course_key))
    except (InvalidKeyError, UnicodeDecodeError):
        return False
    return True

def course_key_is_valid ( course_key ) : if course_key is None : return False try : CourseKey . from_string ( text_type ( course_key ) ) except ( InvalidKeyError , UnicodeDecodeError ) : return False return True


def organization_data_is_valid(organization_data):
    """
    Organization data validation
    """
    if organization_data is None:
        return False
    if 'id' in organization_data and not organization_data.get('id'):
        return False
    if 'name' in organization_data and not organization_data.get('name'):
        return False
    return True

def organization_data_is_valid ( organization_data ) : if organization_data is None : return False if 'id' in organization_data and not organization_data . get ( 'id' ) : return False if 'name' in organization_data and not organization_data . get ( 'name' ) : return False return True


def _inactivate_organization(organization):
    """
    Inactivates an activated organization as well as any active relationships
    """
    [_inactivate_organization_course_relationship(record) for record
     in internal.OrganizationCourse.objects.filter(organization_id=organization.id, active=True)]

    [_inactivate_record(record) for record
     in internal.Organization.objects.filter(id=organization.id, active=True)]


def _activate_organization_course_relationship(relationship):  # pylint: disable=invalid-name
    """
    Activates an inactive organization-course relationship
    """
    # If the relationship doesn't exist or the organization isn't active we'll want to raise an error
    relationship = internal.OrganizationCourse.objects.get(
        id=relationship.id,
        active=False,
        organization__active=True
    )
    _activate_record(relationship)

Activates an inactive organization - course relationship


def _inactivate_organization_course_relationship(relationship):  # pylint: disable=invalid-name
    """
    Inactivates an active organization-course relationship
    """
    relationship = internal.OrganizationCourse.objects.get(
        id=relationship.id,
        active=True
    )
    _inactivate_record(relationship)

Inactivates an active organization - course relationship


def fetch_organization_courses(organization):
    """
    Retrieves the set of courses currently linked to the specified organization
    """
    organization_obj = serializers.deserialize_organization(organization)
    queryset = internal.OrganizationCourse.objects.filter(
        organization=organization_obj,
        active=True
    ).select_related('organization')
    return [serializers.serialize_organization_with_course(organization) for organization in queryset]

def fetch_organization_courses ( organization ) : organization_obj = serializers . deserialize_organization ( organization ) queryset = internal . OrganizationCourse . objects . filter ( organization = organization_obj , active = True ) . select_related ( 'organization' ) return [ serializers . serialize_organization_with_course ( organization ) for organization in queryset ]


def fetch_course_organizations(course_key):
    """
    Retrieves the organizations linked to the specified course
    """
    queryset = internal.OrganizationCourse.objects.filter(
        course_id=text_type(course_key),
        active=True
    ).select_related('organization')
    return [serializers.serialize_organization_with_course(organization) for organization in queryset]

def fetch_course_organizations ( course_key ) : queryset = internal . OrganizationCourse . objects . filter ( course_id = text_type ( course_key ) , active = True ) . select_related ( 'organization' ) return [ serializers . serialize_organization_with_course ( organization ) for organization in queryset ]


def serialize_organization(organization):
    """
    Organization object-to-dict serialization
    """
    return {
        'id': organization.id,
        'name': organization.name,
        'short_name': organization.short_name,
        'description': organization.description,
        'logo': organization.logo
    }

def serialize_organization ( organization ) : return { 'id' : organization . id , 'name' : organization . name , 'short_name' : organization . short_name , 'description' : organization . description , 'logo' : organization . logo }

Organization object - to - dict serialization


def deserialize_organization(organization_dict):
    """
    Organization dict-to-object serialization
    """
    return models.Organization(
        id=organization_dict.get('id'),
        name=organization_dict.get('name', ''),
        short_name=organization_dict.get('short_name', ''),
        description=organization_dict.get('description', ''),
        logo=organization_dict.get('logo', '')
    )

def deserialize_organization ( organization_dict ) : return models . Organization ( id = organization_dict . get ( 'id' ) , name = organization_dict . get ( 'name' , '' ) , short_name = organization_dict . get ( 'short_name' , '' ) , description = organization_dict . get ( 'description' , '' ) , logo = organization_dict . get ( 'logo' , '' ) )

Organization dict - to - object serialization


def is_valid_filename(self, image_node):
        """\
        will check the image src against a list
        of bad image files we know of like buttons, etc...
        """
        src = self.parser.getAttribute(image_node, attr='src')

        if not src:
            return False

        if self.badimages_names_re.search(src):
            return False

        return True

def is_valid_filename ( self , image_node ) : src = self . parser . getAttribute ( image_node , attr = 'src' ) if not src : return False if self . badimages_names_re . search ( src ) : return False return True

\ will check the image src against a list of bad image files we know of like buttons etc ...


def get_images_bytesize_match(self, images):
        """\
        loop through all the images and find the ones
        that have the best bytez to even make them a candidate
        """
        cnt = 0
        max_bytes_size = 15728640
        good_images = []
        for image in images:
            if cnt > 30:
                return good_images
            src = self.parser.getAttribute(image, attr='src')
            src = self.build_image_path(src)
            src = self.add_schema_if_none(src)
            local_image = self.get_local_image(src)
            if local_image:
                filesize = local_image.bytes
                if (filesize == 0 or filesize > self.images_min_bytes) and filesize < max_bytes_size:
                    good_images.append(image)
                else:
                    images.remove(image)
            cnt += 1
        return good_images if len(good_images) > 0 else None

def get_images_bytesize_match ( self , images ) : cnt = 0 max_bytes_size = 15728640 good_images = [ ] for image in images : if cnt > 30 : return good_images src = self . parser . getAttribute ( image , attr = 'src' ) src = self . build_image_path ( src ) src = self . add_schema_if_none ( src ) local_image = self . get_local_image ( src ) if local_image : filesize = local_image . bytes if ( filesize == 0 or filesize > self . images_min_bytes ) and filesize < max_bytes_size : good_images . append ( image ) else : images . remove ( image ) cnt += 1 return good_images if len ( good_images ) > 0 else None


def check_link_tag(self):
        """\
        checks to see if we were able to
        find open link_src on this page
        """
        node = self.article.raw_doc
        meta = self.parser.getElementsByTag(node, tag='link', attr='rel', value='image_src')
        for item in meta:
            src = self.parser.getAttribute(item, attr='href')
            if src:
                return self.get_image(src, extraction_type='linktag')
        return None

def check_link_tag ( self ) : node = self . article . raw_doc meta = self . parser . getElementsByTag ( node , tag = 'link' , attr = 'rel' , value = 'image_src' ) for item in meta : src = self . parser . getAttribute ( item , attr = 'href' ) if src : return self . get_image ( src , extraction_type = 'linktag' ) return None


def get_local_image(self, src):
        """\
        returns the bytes of the image file on disk
        """
        return ImageUtils.store_image(self.fetcher, self.article.link_hash, src, self.config)

def get_local_image ( self , src ) : return ImageUtils . store_image ( self . fetcher , self . article . link_hash , src , self . config )