async def _chunked_upload(self, media, media_size,
                              path=None,
                              media_type=None,
                              media_category=None,
                              chunk_size=2**20,
                              **params):
        """
            upload media in chunks

        Parameters
        ----------
        media : file object
            a file object of the media
        media_size : int
            size of the media
        path : str, optional
            filename of the media
        media_type : str, optional
            mime type of the media
        media_category : str, optional
            twitter media category, must be used with ``media_type``
        chunk_size : int, optional
            size of a chunk in bytes
        params : dict, optional
            additional parameters of the request

        Returns
        -------
        .data_processing.PeonyResponse
            Response of the request
        """
        if isinstance(media, bytes):
            media = io.BytesIO(media)

        chunk = media.read(chunk_size)
        is_coro = asyncio.iscoroutine(chunk)

        if is_coro:
            chunk = await chunk

        if media_type is None:
            media_metadata = await utils.get_media_metadata(chunk, path)
            media_type, media_category = media_metadata
        elif media_category is None:
            media_category = utils.get_category(media_type)

        response = await self.upload.media.upload.post(
            command="INIT",
            total_bytes=media_size,
            media_type=media_type,
            media_category=media_category,
            **params
        )

        media_id = response['media_id']
        i = 0

        while chunk:
            if is_coro:
                req = self.upload.media.upload.post(command="APPEND",
                                                    media_id=media_id,
                                                    media=chunk,
                                                    segment_index=i)
                chunk, _ = await asyncio.gather(media.read(chunk_size), req)
            else:
                await self.upload.media.upload.post(command="APPEND",
                                                    media_id=media_id,
                                                    media=chunk,
                                                    segment_index=i)

                chunk = media.read(chunk_size)

            i += 1

        status = await self.upload.media.upload.post(command="FINALIZE",
                                                     media_id=media_id)

        if 'processing_info' in status:
            while status['processing_info'].get('state') != "succeeded":
                processing_info = status['processing_info']
                if processing_info.get('state') == "failed":
                    error = processing_info.get('error', {})

                    message = error.get('message', str(status))

                    raise exceptions.MediaProcessingError(data=status,
                                                          message=message,
                                                          **params)

                delay = processing_info['check_after_secs']
                await asyncio.sleep(delay)

                status = await self.upload.media.upload.get(
                    command="STATUS",
                    media_id=media_id,
                    **params
                )

        return response

async def _chunked_upload ( self , media , media_size , path = None , media_type = None , media_category = None , chunk_size = 2 ** 20 , * * params ) : if isinstance ( media , bytes ) : media = io . BytesIO ( media ) chunk = media . read ( chunk_size ) is_coro = asyncio . iscoroutine ( chunk ) if is_coro : chunk = await chunk if media_type is None : media_metadata = await utils . get_media_metadata ( chunk , path ) media_type , media_category = media_metadata elif media_category is None : media_category = utils . get_category ( media_type ) response = await self . upload . media . upload . post ( command = "INIT" , total_bytes = media_size , media_type = media_type , media_category = media_category , * * params ) media_id = response [ 'media_id' ] i = 0 while chunk : if is_coro : req = self . upload . media . upload . post ( command = "APPEND" , media_id = media_id , media = chunk , segment_index = i ) chunk , _ = await asyncio . gather ( media . read ( chunk_size ) , req ) else : await self . upload . media . upload . post ( command = "APPEND" , media_id = media_id , media = chunk , segment_index = i ) chunk = media . read ( chunk_size ) i += 1 status = await self . upload . media . upload . post ( command = "FINALIZE" , media_id = media_id ) if 'processing_info' in status : while status [ 'processing_info' ] . get ( 'state' ) != "succeeded" : processing_info = status [ 'processing_info' ] if processing_info . get ( 'state' ) == "failed" : error = processing_info . get ( 'error' , { } ) message = error . get ( 'message' , str ( status ) ) raise exceptions . MediaProcessingError ( data = status , message = message , * * params ) delay = processing_info [ 'check_after_secs' ] await asyncio . sleep ( delay ) status = await self . upload . media . upload . get ( command = "STATUS" , media_id = media_id , * * params ) return response


async def upload_media(self, file_,
                           media_type=None,
                           media_category=None,
                           chunked=None,
                           size_limit=None,
                           **params):
        """
            upload a media on twitter

        Parameters
        ----------
        file_ : str or pathlib.Path or file
            Path to the file or file object
        media_type : str, optional
            mime type of the media
        media_category : str, optional
            Twitter's media category of the media, must be used with
            ``media_type``
        chunked : bool, optional
            If True, force the use of the chunked upload for the media
        size_limit : int, optional
            If set, the media will be sent using a multipart upload if
            its size is over ``size_limit`` bytes
        params : dict
            parameters used when making the request

        Returns
        -------
        .data_processing.PeonyResponse
            Response of the request
        """
        if isinstance(file_, str):
            url = urlparse(file_)
            if url.scheme.startswith('http'):
                media = await self._session.get(file_)
            else:
                path = urlparse(file_).path.strip(" \"'")
                media = await utils.execute(open(path, 'rb'))
        elif hasattr(file_, 'read') or isinstance(file_, bytes):
            media = file_
        else:
            raise TypeError("upload_media input must be a file object or a "
                            "filename or binary data or an aiohttp request")

        media_size = await utils.get_size(media)
        if chunked is not None:
            size_test = False
        else:
            size_test = await self._size_test(media_size, size_limit)

        if isinstance(media, aiohttp.ClientResponse):
            # send the content of the response
            media = media.content

        if chunked or (size_test and chunked is None):
            args = media, media_size, file_, media_type, media_category
            response = await self._chunked_upload(*args, **params)
        else:
            response = await self.upload.media.upload.post(media=media,
                                                           **params)

        if not hasattr(file_, 'read') and not getattr(media, 'closed', True):
            media.close()

        return response


def _parse_iedb_response(response):
    """Take the binding predictions returned by IEDB's web API
    and parse them into a DataFrame

    Expect response to look like:
    allele  seq_num start   end length  peptide ic50   percentile_rank
    HLA-A*01:01 1   2   10  9   LYNTVATLY   2145.70 3.7
    HLA-A*01:01 1   5   13  9   TVATLYCVH   2216.49 3.9
    HLA-A*01:01 1   7   15  9   ATLYCVHQR   2635.42 5.1
    HLA-A*01:01 1   4   12  9   NTVATLYCV   6829.04 20
    HLA-A*01:01 1   1   9   9   SLYNTVATL   8032.38 24
    HLA-A*01:01 1   8   16  9   TLYCVHQRI   8853.90 26
    HLA-A*01:01 1   3   11  9   YNTVATLYC   9865.62 29
    HLA-A*01:01 1   6   14  9   VATLYCVHQ   27575.71    58
    HLA-A*01:01 1   10  18  9   YCVHQRIDV   48929.64    74
    HLA-A*01:01 1   9   17  9   LYCVHQRID   50000.00    75
    """
    if len(response) == 0:
        raise ValueError("Empty response from IEDB!")
    df = pd.read_csv(io.BytesIO(response), delim_whitespace=True, header=0)

    # pylint doesn't realize that df is a DataFrame, so tell is
    assert type(df) == pd.DataFrame
    df = pd.DataFrame(df)

    if len(df) == 0:
        raise ValueError(
            "No binding predictions in response from IEDB: %s" % (response,))
    required_columns = [
        "allele",
        "peptide",
        "ic50",
        "start",
        "end",
    ]
    for column in required_columns:
        if column not in df.columns:
            raise ValueError(
                "Response from IEDB is missing '%s' column: %s. Full "
                "response:\n%s" % (
                    column,
                    df.ix[0],
                    response))
    # since IEDB has allowed multiple column names for percentile rank,
    # we're defensively normalizing all of them to just 'rank'
    df = df.rename(columns={
        "percentile_rank": "rank",
        "percentile rank": "rank"})
    return df

Take the binding predictions returned by IEDB s web API and parse them into a DataFrame


def predict_subsequences(self, sequence_dict, peptide_lengths=None):
        """Given a dictionary mapping unique keys to amino acid sequences,
        run MHC binding predictions on all candidate epitopes extracted from
        sequences and return a EpitopeCollection.

        Parameters
        ----------
        fasta_dictionary : dict or string
            Mapping of protein identifiers to protein amino acid sequences.
            If string then converted to dictionary.
        """
        sequence_dict = check_sequence_dictionary(sequence_dict)
        peptide_lengths = self._check_peptide_lengths(peptide_lengths)

        # take each mutated sequence in the dataframe
        # and general MHC binding scores for all k-mer substrings
        binding_predictions = []
        expected_peptides = set([])

        normalized_alleles = []
        for key, amino_acid_sequence in sequence_dict.items():
            for l in peptide_lengths:
                for i in range(len(amino_acid_sequence) - l + 1):
                    expected_peptides.add(amino_acid_sequence[i:i + l])
            self._check_peptide_inputs(expected_peptides)
            for allele in self.alleles:
                # IEDB MHCII predictor expects DRA1 to be omitted.
                allele = normalize_allele_name(allele, omit_dra1=True)
                normalized_alleles.append(allele)
                request = self._get_iedb_request_params(
                    amino_acid_sequence, allele)
                logger.info(
                    "Calling IEDB (%s) with request %s",
                    self.url,
                    request)
                response_df = _query_iedb(request, self.url)
                for _, row in response_df.iterrows():
                    binding_predictions.append(
                        BindingPrediction(
                            source_sequence_name=key,
                            offset=row['start'] - 1,
                            allele=row['allele'],
                            peptide=row['peptide'],
                            affinity=row['ic50'],
                            percentile_rank=row['rank'],
                            prediction_method_name="iedb-" + self.prediction_method))
        self._check_results(
            binding_predictions,
            alleles=normalized_alleles,
            peptides=expected_peptides)
        return BindingPredictionCollection(binding_predictions)

Given a dictionary mapping unique keys to amino acid sequences run MHC binding predictions on all candidate epitopes extracted from sequences and return a EpitopeCollection .


def get_args(func, skip=0):
    """
        Hackish way to get the arguments of a function

    Parameters
    ----------
    func : callable
        Function to get the arguments from
    skip : int, optional
        Arguments to skip, defaults to 0 set it to 1 to skip the
        ``self`` argument of a method.

    Returns
    -------
    tuple
        Function's arguments
    """

    code = getattr(func, '__code__', None)
    if code is None:
        code = func.__call__.__code__

    return code.co_varnames[skip:code.co_argcount]

def get_args ( func , skip = 0 ) : code = getattr ( func , '__code__' , None ) if code is None : code = func . __call__ . __code__ return code . co_varnames [ skip : code . co_argcount ]


def log_error(msg=None, exc_info=None, logger=None, **kwargs):
    """
        log an exception and its traceback on the logger defined

    Parameters
    ----------
    msg : str, optional
        A message to add to the error
    exc_info : tuple
        Information about the current exception
    logger : logging.Logger
        logger to use
    """
    if logger is None:
        logger = _logger

    if not exc_info:
        exc_info = sys.exc_info()

    if msg is None:
        msg = ""

    exc_class, exc_msg, _ = exc_info

    if all(info is not None for info in exc_info):
        logger.error(msg, exc_info=exc_info)

def log_error ( msg = None , exc_info = None , logger = None , * * kwargs ) : if logger is None : logger = _logger if not exc_info : exc_info = sys . exc_info ( ) if msg is None : msg = "" exc_class , exc_msg , _ = exc_info if all ( info is not None for info in exc_info ) : logger . error ( msg , exc_info = exc_info )


async def get_media_metadata(data, path=None):
    """
        Get all the file's metadata and read any kind of file object

    Parameters
    ----------
    data : bytes
        first bytes of the file (the mimetype shoudl be guessed from the
        file headers
    path : str, optional
        path to the file

    Returns
    -------
    str
        The mimetype of the media
    str
        The category of the media on Twitter
    """
    if isinstance(data, bytes):
        media_type = await get_type(data, path)

    else:
        raise TypeError("get_metadata input must be a bytes")

    media_category = get_category(media_type)

    _logger.info("media_type: %s, media_category: %s" % (media_type,
                                                         media_category))

    return media_type, media_category

async
def
get_media_metadata
(
data
,
path
=
)
:
if
isinstance
(
data
,
bytes
)
:
media_type
=
await
get_type
(
data
,
path
)
else
:
raise
TypeError
(
"get_metadata input must be a bytes"
)
media_category
=
get_category
(
media_type
)
_logger
.
info
(
"media_type: %s, media_category: %s"
%
(
media_type
,
media_category
)
)
return
media_type
,
media_category

Get all the file s metadata and read any kind of file object


async def get_size(media):
    """
        Get the size of a file

    Parameters
    ----------
    media : file object
        The file object of the media

    Returns
    -------
    int
        The size of the file
    """
    if hasattr(media, 'seek'):
        await execute(media.seek(0, os.SEEK_END))
        size = await execute(media.tell())
        await execute(media.seek(0))
    elif hasattr(media, 'headers'):
        size = int(media.headers['Content-Length'])
    elif isinstance(media, bytes):
        size = len(media)
    else:
        raise TypeError("Can't get size of media of type:",
                        type(media).__name__)

    _logger.info("media size: %dB" % size)
    return size

async
def
get_size
(
media
)
:
if
hasattr
(
media
,
'seek'
)
:
await
execute
(
media
.
seek
(
0
,
os
.
SEEK_END
)
)
size
=
await
execute
(
media
.
tell
(
)
)
await
execute
(
media
.
seek
(
0
)
)
elif
hasattr
(
media
,
'headers'
)
:
size
=
int
(
media
.
headers
[
'Content-Length'
]
)
elif
isinstance
(
media
,
bytes
)
:
size
=
len
(
media
)
else
:
raise
TypeError
(
"Can't get size of media of type:"
,
type
(
media
)
.
__name__
)
_logger
.
info
(
"media size: %dB"
%
size
)
return
size


def set_debug():
    """ activates error messages, useful during development """
    logging.basicConfig(level=logging.WARNING)
    peony.logger.setLevel(logging.DEBUG)

def set_debug ( ) : logging . basicConfig ( level = logging . WARNING ) peony . logger . setLevel ( logging . DEBUG )

activates error messages useful during development


def clone_with_updates(self, **kwargs):
        """Returns new BindingPrediction with updated fields"""
        fields_dict = self.to_dict()
        fields_dict.update(kwargs)
        return BindingPrediction(**fields_dict)

def clone_with_updates ( self , * * kwargs ) : fields_dict = self . to_dict ( ) fields_dict . update ( kwargs ) return BindingPrediction ( * * fields_dict )


def get_data(self, response):
        """ Get the data from the response """
        if self._response_list:
            return response
        elif self._response_key is None:
            if hasattr(response, "items"):
                for key, data in response.items():
                    if (hasattr(data, "__getitem__")
                            and not hasattr(data, "items")
                            and len(data) > 0
                            and 'id' in data[0]):
                        self._response_key = key
                        return data
            else:
                self._response_list = True
                return response
        else:
            return response[self._response_key]

        raise NoDataFound(response=response, url=self.request.get_url())

def get_data ( self , response ) : if self . _response_list : return response elif self . _response_key is None : if hasattr ( response , "items" ) : for key , data in response . items ( ) : if ( hasattr ( data , "__getitem__" ) and not hasattr ( data , "items" ) and len ( data ) > 0 and 'id' in data [ 0 ] ) : self . _response_key = key return data else : self . _response_list = True return response else : return response [ self . _response_key ] raise NoDataFound ( response = response , url = self . request . get_url ( ) )


async def call_on_response(self, data):
        """
        Try to fill the gaps and strip last tweet from the response
        if its id is that of the first tweet of the last response

        Parameters
        ----------
        data : list
            The response data
        """
        since_id = self.kwargs.get(self.param, 0) + 1

        if self.fill_gaps:
            if data[-1]['id'] != since_id:
                max_id = data[-1]['id'] - 1
                responses = with_max_id(self.request(**self.kwargs,
                                                     max_id=max_id))

                async for tweets in responses:
                    data.extend(tweets)

            if data[-1]['id'] == self.last_id:
                data = data[:-1]

        if not data and not self.force:
            raise StopAsyncIteration

        await self.set_param(data)

async def call_on_response ( self , data ) : since_id = self . kwargs . get ( self . param , 0 ) + 1 if self . fill_gaps : if data [ - 1 ] [ 'id' ] != since_id : max_id = data [ - 1 ] [ 'id' ] - 1 responses = with_max_id ( self . request ( * * self . kwargs , max_id = max_id ) ) async for tweets in responses : data . extend ( tweets ) if data [ - 1 ] [ 'id' ] == self . last_id : data = data [ : - 1 ] if not data and not self . force : raise StopAsyncIteration await self . set_param ( data )


async def get_oauth_token(consumer_key, consumer_secret, callback_uri="oob"):
    """
    Get a temporary oauth token

    Parameters
    ----------
    consumer_key : str
        Your consumer key
    consumer_secret : str
        Your consumer secret
    callback_uri : str, optional
        Callback uri, defaults to 'oob'

    Returns
    -------
    dict
        Temporary tokens
    """

    client = BasePeonyClient(consumer_key=consumer_key,
                             consumer_secret=consumer_secret,
                             api_version="",
                             suffix="")

    response = await client.api.oauth.request_token.post(
        _suffix="",
        oauth_callback=callback_uri
    )

    return parse_token(response)

async def get_oauth_token ( consumer_key , consumer_secret , callback_uri = "oob" ) : client = BasePeonyClient ( consumer_key = consumer_key , consumer_secret = consumer_secret , api_version = "" , suffix = "" ) response = await client . api . oauth . request_token . post ( _suffix = "" , oauth_callback = callback_uri ) return parse_token ( response )


async def get_oauth_verifier(oauth_token):
    """
    Open authorize page in a browser,
    print the url if it didn't work

    Arguments
    ---------
    oauth_token : str
        The oauth token received in :func:`get_oauth_token`

    Returns
    -------
    str
        The PIN entered by the user
    """
    url = "https://api.twitter.com/oauth/authorize?oauth_token=" + oauth_token

    try:
        browser = webbrowser.open(url)
        await asyncio.sleep(2)

        if not browser:
            raise RuntimeError
    except RuntimeError:
        print("could not open a browser\ngo here to enter your PIN: " + url)

    verifier = input("\nEnter your PIN: ")
    return verifier

Open authorize page in a browser print the url if it didn t work


async def get_access_token(consumer_key, consumer_secret,
                           oauth_token, oauth_token_secret,
                           oauth_verifier, **kwargs):
    """
        get the access token of the user

    Parameters
    ----------
    consumer_key : str
        Your consumer key
    consumer_secret : str
        Your consumer secret
    oauth_token : str
        OAuth token from :func:`get_oauth_token`
    oauth_token_secret : str
        OAuth token secret from :func:`get_oauth_token`
    oauth_verifier : str
        OAuth verifier from :func:`get_oauth_verifier`

    Returns
    -------
    dict
        Access tokens
    """

    client = BasePeonyClient(consumer_key=consumer_key,
                             consumer_secret=consumer_secret,
                             access_token=oauth_token,
                             access_token_secret=oauth_token_secret,
                             api_version="",
                             suffix="")

    response = await client.api.oauth.access_token.get(
        _suffix="",
        oauth_verifier=oauth_verifier
    )

    return parse_token(response)

async def get_access_token ( consumer_key , consumer_secret , oauth_token , oauth_token_secret , oauth_verifier , * * kwargs ) : client = BasePeonyClient ( consumer_key = consumer_key , consumer_secret = consumer_secret , access_token = oauth_token , access_token_secret = oauth_token_secret , api_version = "" , suffix = "" ) response = await client . api . oauth . access_token . get ( _suffix = "" , oauth_verifier = oauth_verifier ) return parse_token ( response )


def parse_token(response):
    """
    parse the responses containing the tokens

    Parameters
    ----------
    response : str
        The response containing the tokens

    Returns
    -------
    dict
        The parsed tokens
    """
    items = response.split("&")
    items = [item.split("=") for item in items]

    return {key: value for key, value in items}

def parse_token ( response ) : items = response . split ( "&" ) items = [ item . split ( "=" ) for item in items ] return { key : value for key , value in items }


def predict(self, sequences):
        """
        Return netChop predictions for each position in each sequence.

        Parameters
        -----------
        sequences : list of string
            Amino acid sequences to predict cleavage for

        Returns
        -----------
        list of list of float

        The i'th list corresponds to the i'th sequence. Each list gives
        the cleavage probability for each position in the sequence.
        """
        with tempfile.NamedTemporaryFile(suffix=".fsa", mode="w") as input_fd:
            for (i, sequence) in enumerate(sequences):
                input_fd.write("> %d\n" % i)
                input_fd.write(sequence)
                input_fd.write("\n")
            input_fd.flush()
            try:
                output = subprocess.check_output(["netChop", input_fd.name])
            except subprocess.CalledProcessError as e:
                logging.error("Error calling netChop: %s:\n%s" % (e, e.output))
                raise

        parsed = self.parse_netchop(output)
        assert len(parsed) == len(sequences), \
            "Expected %d results but got %d" % (
                len(sequences), len(parsed))
        assert [len(x) for x in parsed] == [len(x) for x in sequences]
        return parsed

def
predict
(
self
,
sequences
)
:
with
tempfile
.
NamedTemporaryFile
(
suffix
=
".fsa"
,
mode
=
"w"
)
as
input_fd
:
for
(
i
,
sequence
)
in
enumerate
(
sequences
)
:
input_fd
.
write
(
"> %d\n"
%
i
)
input_fd
.
write
(
sequence
)
input_fd
.
write
(
"\n"
)
input_fd
.
flush
(
)
try
:
output
=
subprocess
.
check_output
(
[
"netChop"
,
input_fd
.
name
]
)
except
subprocess
.
CalledProcessError
as
e
:
logging
.
error
(
"Error calling netChop: %s:\n%s"
%
(
e
,
e
.
output
)
)
raise
parsed
=
self
.
parse_netchop
(
output
)
assert
len
(
parsed
)
==
len
(
sequences
)
,
"Expected %d results but got %d"
%
(
len
(
sequences
)
,
len
(
parsed
)
)
assert
[
len
(
x
)
for
x
in
parsed
]
==
[
len
(
x
)
for
x
in
sequences
]
return
parsed

Return netChop predictions for each position in each sequence .


def parse_netchop(netchop_output):
        """
        Parse netChop stdout.
        """
        line_iterator = iter(netchop_output.decode().split("\n"))
        scores = []
        for line in line_iterator:
            if "pos" in line and 'AA' in line and 'score' in line:
                scores.append([])
                if "----" not in next(line_iterator):
                    raise ValueError("Dashes expected")
                line = next(line_iterator)
                while '-------' not in line:
                    score = float(line.split()[3])
                    scores[-1].append(score)
                    line = next(line_iterator)
        return scores

def
parse_netchop
(
netchop_output
)
:
line_iterator
=
iter
(
netchop_output
.
decode
(
)
.
split
(
"\n"
)
)
scores
=
[
]
for
line
in
line_iterator
:
if
"pos"
in
line
and
'AA'
in
line
and
'score'
in
line
:
scores
.
append
(
[
]
)
if
"----"
not
in
next
(
line_iterator
)
:
raise
ValueError
(
"Dashes expected"
)
line
=
next
(
line_iterator
)
while
'-------'
not
in
line
:
score
=
float
(
line
.
split
(
)
[
3
]
)
scores
[
1
]
.
append
(
score
)
line
=
next
(
line_iterator
)
return
scores


def to_dataframe(
            self,
            columns=BindingPrediction.fields + ("length",)):
        """
        Converts collection of BindingPrediction objects to DataFrame
        """
        return pd.DataFrame.from_records(
            [tuple([getattr(x, name) for name in columns]) for x in self],
            columns=columns)

def to_dataframe ( self , columns = BindingPrediction . fields + ( "length" , ) ) : return pd . DataFrame . from_records ( [ tuple ( [ getattr ( x , name ) for name in columns ] ) for x in self ] , columns = columns )


def NetMHC(alleles,
           default_peptide_lengths=[9],
           program_name="netMHC"):
    """
    This function wraps NetMHC3 and NetMHC4 to automatically detect which class
    to use. Currently based on running the '-h' command and looking for
    discriminating substrings between the versions.
    """
    # run NetMHC's help command and parse discriminating substrings out of
    # the resulting str output
    with open(os.devnull, 'w') as devnull:
        help_output = check_output([program_name, "-h"], stderr=devnull)
    help_output_str = help_output.decode("ascii", "ignore")

    substring_to_netmhc_class = {
        "-listMHC": NetMHC4,
        "--Alleles": NetMHC3,
    }

    successes = []

    for substring, netmhc_class in substring_to_netmhc_class.items():
        if substring in help_output_str:
            successes.append(netmhc_class)

    if len(successes) > 1:
        raise SystemError("Command %s is valid for multiple NetMHC versions. "
                          "This is likely an mhctools bug." % program_name)
    if len(successes) == 0:
        raise SystemError("Command %s is not a valid way of calling any NetMHC software."
                          % program_name)

    netmhc_class = successes[0]
    return netmhc_class(
        alleles=alleles,
        default_peptide_lengths=default_peptide_lengths,
        program_name=program_name)

This function wraps NetMHC3 and NetMHC4 to automatically detect which class to use . Currently based on running the - h command and looking for discriminating substrings between the versions .


def predict_peptides(self, peptides):
        """
        Predict MHC affinity for peptides.
        """

        # importing locally to avoid slowing down CLI applications which
        # don't use MHCflurry
        from mhcflurry.encodable_sequences import EncodableSequences

        binding_predictions = []
        encodable_sequences = EncodableSequences.create(peptides)
        for allele in self.alleles:
            predictions_df = self.predictor.predict_to_dataframe(
                encodable_sequences, allele=allele)
            for (_, row) in predictions_df.iterrows():
                binding_prediction = BindingPrediction(
                    allele=allele,
                    peptide=row.peptide,
                    affinity=row.prediction,
                    percentile_rank=(
                        row.prediction_percentile
                        if 'prediction_percentile' in row else nan),
                    prediction_method_name="mhcflurry"
                )
                binding_predictions.append(binding_prediction)
        return BindingPredictionCollection(binding_predictions)


def seq_to_str(obj, sep=","):
    """
    Given a sequence convert it to a comma separated string.
    If, however, the argument is a single object, return its string
    representation.
    """
    if isinstance(obj, string_classes):
        return obj
    elif isinstance(obj, (list, tuple)):
        return sep.join([str(x) for x in obj])
    else:
        return str(obj)

def seq_to_str ( obj , sep = "," ) : if isinstance ( obj , string_classes ) : return obj elif isinstance ( obj , ( list , tuple ) ) : return sep . join ( [ str ( x ) for x in obj ] ) else : return str ( obj )

Given a sequence convert it to a comma separated string . If however the argument is a single object return its string representation .


def create_input_peptides_files(
        peptides,
        max_peptides_per_file=None,
        group_by_length=False):
    """
    Creates one or more files containing one peptide per line,
    returns names of files.
    """
    if group_by_length:
        peptide_lengths = {len(p) for p in peptides}
        peptide_groups = {l: [] for l in peptide_lengths}
        for p in peptides:
            peptide_groups[len(p)].append(p)
    else:
        peptide_groups = {"": peptides}

    file_names = []
    for key, group in peptide_groups.items():
        n_peptides = len(group)
        if not max_peptides_per_file:
            max_peptides_per_file = n_peptides
        input_file = None
        for i, p in enumerate(group):
            if i % max_peptides_per_file == 0:
                if input_file is not None:
                    file_names.append(input_file.name)
                    input_file.close()
                input_file = make_writable_tempfile(
                    prefix_number=i // max_peptides_per_file,
                    prefix_name=key,
                    suffix=".txt")
            input_file.write("%s\n" % p)
        if input_file is not None:
            file_names.append(input_file.name)
            input_file.close()
    return file_names

def create_input_peptides_files ( peptides , max_peptides_per_file = None , group_by_length = False ) : if group_by_length : peptide_lengths = { len ( p ) for p in peptides } peptide_groups = { l : [ ] for l in peptide_lengths } for p in peptides : peptide_groups [ len ( p ) ] . append ( p ) else : peptide_groups = { "" : peptides } file_names = [ ] for key , group in peptide_groups . items ( ) : n_peptides = len ( group ) if not max_peptides_per_file : max_peptides_per_file = n_peptides input_file = None for i , p in enumerate ( group ) : if i % max_peptides_per_file == 0 : if input_file is not None : file_names . append ( input_file . name ) input_file . close ( ) input_file = make_writable_tempfile ( prefix_number = i // max_peptides_per_file , prefix_name = key , suffix = ".txt" ) input_file . write ( "%s\n" % p ) if input_file is not None : file_names . append ( input_file . name ) input_file . close ( ) return file_names

Creates one or more files containing one peptide per line returns names of files .


def _check_peptide_lengths(self, peptide_lengths=None):
        """
        If peptide lengths not specified, then try using the default
        lengths associated with this predictor object. If those aren't
        a valid non-empty sequence of integers, then raise an exception.
        Otherwise return the peptide lengths.
        """
        if not peptide_lengths:
            peptide_lengths = self.default_peptide_lengths

        if not peptide_lengths:
            raise ValueError(
                ("Must either provide 'peptide_lengths' argument "
                "or set 'default_peptide_lengths"))
        if isinstance(peptide_lengths, int):
            peptide_lengths = [peptide_lengths]
        require_iterable_of(peptide_lengths, int)
        for peptide_length in peptide_lengths:
            if (self.min_peptide_length is not None and
                    peptide_length < self.min_peptide_length):
                raise ValueError(
                    "Invalid peptide length %d, shorter than min %d" % (
                        peptide_length,
                        self.min_peptide_length))
            elif (self.max_peptide_length is not None and
                    peptide_length > self.max_peptide_length):
                raise ValueError(
                    "Invalid peptide length %d, longer than max %d" % (
                        peptide_length,
                        self.max_peptide_length))
        return peptide_lengths

If peptide lengths not specified then try using the default lengths associated with this predictor object . If those aren t a valid non - empty sequence of integers then raise an exception . Otherwise return the peptide lengths .


def _check_peptide_inputs(self, peptides):
        """
        Check peptide sequences to make sure they are valid for this predictor.
        """
        require_iterable_of(peptides, string_types)
        check_X = not self.allow_X_in_peptides
        check_lower = not self.allow_lowercase_in_peptides
        check_min_length = self.min_peptide_length is not None
        min_length = self.min_peptide_length
        check_max_length = self.max_peptide_length is not None
        max_length = self.max_peptide_length
        for p in peptides:
            if not p.isalpha():
                raise ValueError("Invalid characters in peptide '%s'" % p)
            elif check_X and "X" in p:
                raise ValueError("Invalid character 'X' in peptide '%s'" % p)
            elif check_lower and not p.isupper():
                raise ValueError("Invalid lowercase letters in peptide '%s'" % p)
            elif check_min_length and len(p) < min_length:
                raise ValueError(
                    "Peptide '%s' too short (%d chars), must be at least %d" % (
                        p, len(p), min_length))
            elif check_max_length and len(p) > max_length:
                raise ValueError(
                    "Peptide '%s' too long (%d chars), must be at least %d" % (
                        p, len(p), max_length))

Check peptide sequences to make sure they are valid for this predictor .


def predict_subsequences(
            self,
            sequence_dict,
            peptide_lengths=None):
        """
        Given a dictionary mapping sequence names to amino acid strings,
        and an optional list of peptide lengths, returns a
        BindingPredictionCollection.
        """
        if isinstance(sequence_dict, string_types):
            sequence_dict = {"seq": sequence_dict}
        elif isinstance(sequence_dict, (list, tuple)):
            sequence_dict = {seq: seq for seq in sequence_dict}

        peptide_lengths = self._check_peptide_lengths(peptide_lengths)

        # convert long protein sequences to set of peptides and
        # associated sequence name / offsets that each peptide may have come
        # from
        peptide_set = set([])
        peptide_to_name_offset_pairs = defaultdict(list)

        for name, sequence in sequence_dict.items():
            for peptide_length in peptide_lengths:
                for i in range(len(sequence) - peptide_length + 1):
                    peptide = sequence[i:i + peptide_length]
                    peptide_set.add(peptide)
                    peptide_to_name_offset_pairs[peptide].append((name, i))
        peptide_list = sorted(peptide_set)

        binding_predictions = self.predict_peptides(peptide_list)

        # create BindingPrediction objects with sequence name and offset
        results = []
        for binding_prediction in binding_predictions:
            for name, offset in peptide_to_name_offset_pairs[
                    binding_prediction.peptide]:
                results.append(binding_prediction.clone_with_updates(
                    source_sequence_name=name,
                    offset=offset))
        self._check_results(
            results,
            peptides=peptide_set,
            alleles=self.alleles)
        return BindingPredictionCollection(results)

Given a dictionary mapping sequence names to amino acid strings and an optional list of peptide lengths returns a BindingPredictionCollection .


def _check_hla_alleles(
            alleles,
            valid_alleles=None):
        """
        Given a list of HLA alleles and an optional list of valid
        HLA alleles, return a set of alleles that we will pass into
        the MHC binding predictor.
        """
        require_iterable_of(alleles, string_types, "HLA alleles")

        # Don't run the MHC predictor twice for homozygous alleles,
        # only run it for unique alleles
        alleles = {
            normalize_allele_name(allele.strip().upper())
            for allele in alleles
        }
        if valid_alleles:
            # For some reason netMHCpan drops the '*' in names, so
            # 'HLA-A*03:01' becomes 'HLA-A03:01'
            missing_alleles = [
                allele
                for allele in alleles
                if allele not in valid_alleles
            ]
            if len(missing_alleles) > 0:
                raise UnsupportedAllele(
                    "Unsupported HLA alleles: %s" % missing_alleles)

        return list(alleles)

Given a list of HLA alleles and an optional list of valid HLA alleles return a set of alleles that we will pass into the MHC binding predictor .


async def _connect(self):
        """
            Connect to the stream

        Returns
        -------
        asyncio.coroutine
            The streaming response
        """
        logger.debug("connecting to the stream")
        await self.client.setup
        if self.session is None:
            self.session = self.client._session
        kwargs = await self.client.headers.prepare_request(**self.kwargs)
        request = self.client.error_handler(self.session.request)

        return await request(timeout=0, **kwargs)

async
def
_connect
(
self
)
:
logger
.
debug
(
"connecting to the stream"
)
await
self
.
client
.
setup
if
self
.
session
is
:
self
.
session
=
self
.
client
.
_session
kwargs
=
await
self
.
client
.
headers
.
prepare_request
(
*
*
self
.
kwargs
)
request
=
self
.
client
.
error_handler
(
self
.
session
.
request
)
return
await
request
(
timeout
=
0
,
*
*
kwargs
)


async def connect(self):
        """
            Create the connection

        Returns
        -------
        self

        Raises
        ------
        exception.PeonyException
            On a response status in 4xx that are not status 420 or 429
            Also on statuses in 1xx or 3xx since this should not be the status
            received here
        """
        with async_timeout.timeout(self.timeout):
            self.response = await self._connect()

        if self.response.status in range(200, 300):
            self._error_timeout = 0
            self.state = NORMAL
        elif self.response.status == 500:
            self.state = DISCONNECTION
        elif self.response.status in range(501, 600):
            self.state = RECONNECTION
        elif self.response.status in (420, 429):
            self.state = ENHANCE_YOUR_CALM
        else:
            logger.debug("raising error during stream connection")
            raise await exceptions.throw(self.response,
                                         loads=self.client._loads,
                                         url=self.kwargs['url'])

        logger.debug("stream state: %d" % self.state)


def with_prefix(self, prefix, strict=False):
        """
            decorator to handle commands with prefixes

        Parameters
        ----------
        prefix : str
            the prefix of the command
        strict : bool, optional
            If set to True the command must be at the beginning
            of the message. Defaults to False.

        Returns
        -------
        function
            a decorator that returns an :class:`EventHandler` instance
        """

        def decorated(func):
            return EventHandler(func=func, event=self.event,
                                prefix=prefix, strict=strict)

        return decorated

def with_prefix ( self , prefix , strict = False ) : def decorated ( func ) : return EventHandler ( func = func , event = self . event , prefix = prefix , strict = strict ) return decorated


async def set_tz(self):
        """
            set the environment timezone to the timezone
            set in your twitter settings
        """
        settings = await self.api.account.settings.get()

        tz = settings.time_zone.tzinfo_name

        os.environ['TZ'] = tz
        time.tzset()

async def set_tz ( self ) : settings = await self . api . account . settings . get ( ) tz = settings . time_zone . tzinfo_name os . environ [ 'TZ' ] = tz time . tzset ( )


def run_command(args, **kwargs):
    """
    Given a list whose first element is a command name, followed by arguments,
    execute it and show timing info.
    """
    assert len(args) > 0
    start_time = time.time()
    process = AsyncProcess(args, **kwargs)
    process.wait()
    elapsed_time = time.time() - start_time
    logger.info("%s took %0.4f seconds", args[0], elapsed_time)

def
run_command
(
args
,
*
*
kwargs
)
:
assert
len
(
args
)
>
0
start_time
=
time
.
time
(
)
process
=
AsyncProcess
(
args
,
*
*
kwargs
)
process
.
wait
(
)
elapsed_time
=
time
.
time
(
)
start_time
logger
.
info
(
"%s took %0.4f seconds"
,
args
[
0
]
,
elapsed_time
)

Given a list whose first element is a command name followed by arguments execute it and show timing info .


def run_multiple_commands_redirect_stdout(
        multiple_args_dict,
        print_commands=True,
        process_limit=-1,
        polling_freq=0.5,
        **kwargs):
    """
    Run multiple shell commands in parallel, write each of their
    stdout output to files associated with each command.

    Parameters
    ----------
    multiple_args_dict : dict
        A dictionary whose keys are files and values are args list.
        Run each args list as a subprocess and write stdout to the
        corresponding file.

    print_commands : bool
        Print shell commands before running them.

    process_limit : int
        Limit the number of concurrent processes to this number. 0
        if there is no limit, -1 to use max number of processors

    polling_freq : int
        Number of seconds between checking for done processes, if
        we have a process limit
    """
    assert len(multiple_args_dict) > 0
    assert all(len(args) > 0 for args in multiple_args_dict.values())
    assert all(hasattr(f, 'name') for f in multiple_args_dict.keys())
    if process_limit < 0:
        logger.debug("Using %d processes" % cpu_count())
        process_limit = cpu_count()

    start_time = time.time()
    processes = Queue(maxsize=process_limit)

    def add_to_queue(process):
        process.start()
        if print_commands:
            handler = logging.FileHandler(process.redirect_stdout_file.name)
            handler.setLevel(logging.DEBUG)
            logger.addHandler(handler)
            logger.debug(" ".join(process.args))
            logger.removeHandler(handler)
        processes.put(process)

    for f, args in multiple_args_dict.items():
        p = AsyncProcess(
                args,
                redirect_stdout_file=f,
                **kwargs)
        if not processes.full():
            add_to_queue(p)
        else:
            while processes.full():
                # Are there any done processes?
                to_remove = []
                for possibly_done in processes.queue:
                    if possibly_done.poll() is not None:
                        possibly_done.wait()
                        to_remove.append(possibly_done)
                # Remove them from the queue and stop checking
                if to_remove:
                    for process_to_remove in to_remove:
                        processes.queue.remove(process_to_remove)
                    break
                # Check again in a second if there weren't
                time.sleep(polling_freq)
            add_to_queue(p)

    # Wait for all the rest of the processes
    while not processes.empty():
        processes.get().wait()

    elapsed_time = time.time() - start_time
    logger.info(
        "Ran %d commands in %0.4f seconds",
        len(multiple_args_dict),
        elapsed_time)

Run multiple shell commands in parallel write each of their stdout output to files associated with each command .


def loads(json_data, encoding="utf-8", **kwargs):
    """
        Custom loads function with an object_hook and automatic decoding

    Parameters
    ----------
    json_data : str
        The JSON data to decode
    *args
        Positional arguments, passed to :func:`json.loads`
    encoding : :obj:`str`, optional
        The encoding of the bytestring
    **kwargs
        Keyword arguments passed to :func:`json.loads`

    Returns
    -------
    :obj:`dict` or :obj:`list`
        Decoded json data
    """
    if isinstance(json_data, bytes):
        json_data = json_data.decode(encoding)

    return json.loads(json_data, object_hook=JSONData, **kwargs)

def loads ( json_data , encoding = "utf-8" , * * kwargs ) : if isinstance ( json_data , bytes ) : json_data = json_data . decode ( encoding ) return json . loads ( json_data , object_hook = JSONData , * * kwargs )


async def read(response, loads=loads, encoding=None):
    """
        read the data of the response

    Parameters
    ----------
    response : aiohttp.ClientResponse
        response
    loads : callable
        json loads function
    encoding : :obj:`str`, optional
        character encoding of the response, if set to None
        aiohttp should guess the right encoding

    Returns
    -------
    :obj:`bytes`, :obj:`str`, :obj:`dict` or :obj:`list`
        the data returned depends on the response
    """
    ctype = response.headers.get('Content-Type', "").lower()

    try:
        if "application/json" in ctype:
            logger.info("decoding data as json")
            return await response.json(encoding=encoding, loads=loads)

        if "text" in ctype:
            logger.info("decoding data as text")
            return await response.text(encoding=encoding)

    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
        data = await response.read()
        raise exceptions.PeonyDecodeError(response=response,
                                          data=data,
                                          exception=exc)

    return await response.read()

async
def
read
(
response
,
loads
=
loads
,
encoding
=
)
:
ctype
=
response
.
headers
.
get
(
'Content-Type'
,
""
)
.
lower
(
)
try
:
if
"application/json"
in
ctype
:
logger
.
info
(
"decoding data as json"
)
return
await
response
.
json
(
encoding
=
encoding
,
loads
=
loads
)
if
"text"
in
ctype
:
logger
.
info
(
"decoding data as text"
)
return
await
response
.
text
(
encoding
=
encoding
)
except
(
UnicodeDecodeError
,
json
.
JSONDecodeError
)
as
exc
:
data
=
await
response
.
read
(
)
raise
exceptions
.
PeonyDecodeError
(
response
=
response
,
data
=
data
,
exception
=
exc
)
return
await
response
.
read
(
)


def doc(func):
    """
        Find the message shown when someone calls the help command

    Parameters
    ----------
    func : function
        the function

    Returns
    -------
    str
        The help message for this command
    """
    stripped_chars = " \t"

    if hasattr(func, '__doc__'):
        docstring = func.__doc__.lstrip(" \n\t")
        if "\n" in docstring:
            i = docstring.index("\n")
            return docstring[:i].rstrip(stripped_chars)
        elif docstring:
            return docstring.rstrip(stripped_chars)

    return ""

def
doc
(
func
)
:
stripped_chars
=
" \t"
if
hasattr
(
func
,
'__doc__'
)
:
docstring
=
func
.
__doc__
.
lstrip
(
" \n\t"
)
if
"\n"
in
docstring
:
i
=
docstring
.
index
(
"\n"
)
return
docstring
[
:
i
]
.
rstrip
(
stripped_chars
)
elif
docstring
:
return
docstring
.
rstrip
(
stripped_chars
)
return
""


def permission_check(data, command_permissions,
                     command=None, permissions=None):
    """
        Check the permissions of the user requesting a command

    Parameters
    ----------
    data : dict
        message data
    command_permissions : dict
        permissions of the command, contains all the roles as key and users
        with these permissions as values
    command : function
        the command that is run
    permissions : tuple or list
        a list of permissions for the command

    Returns
    -------
    bool
        True if the user has the right permissions, False otherwise
    """
    if permissions:
        pass
    elif command:
        if hasattr(command, 'permissions'):
            permissions = command.permissions
        else:
            return True  # true if no permission is required
    else:
        msg = "{name} must be called with command or permissions argument"
        raise RuntimeError(msg.format(name="_permission_check"))

    return any(data['sender']['id'] in command_permissions[permission]
               for permission in permissions
               if permission in command_permissions)


def main(args_list=None):
    """
    Script to make pMHC binding predictions from amino acid sequences.

    Usage example:
        mhctools
            --sequence SFFPIQQQQQAAALLLI \
            --sequence SILQQQAQAQQAQAASSSC \
            --extract-subsequences \
            --mhc-predictor netmhc \
            --mhc-alleles HLA-A0201 H2-Db \
            --mhc-predictor netmhc \
            --output-csv epitope.csv
    """
    args = parse_args(args_list)
    binding_predictions = run_predictor(args)
    df = binding_predictions.to_dataframe()
    logger.info('\n%s', df)
    if args.output_csv:
        df.to_csv(args.output_csv, index=False)
        print("Wrote: %s" % args.output_csv)

def
main
(
args_list
=
)
:
args
=
parse_args
(
args_list
)
binding_predictions
=
run_predictor
(
args
)
df
=
binding_predictions
.
to_dataframe
(
)
logger
.
info
(
'\n%s'
,
df
)
if
args
.
output_csv
:
df
.
to_csv
(
args
.
output_csv
,
index
=
False
)
print
(
"Wrote: %s"
%
args
.
output_csv
)

Script to make pMHC binding predictions from amino acid sequences .


def _prepare_drb_allele_name(self, parsed_beta_allele):
        """
        Assume that we're dealing with a human DRB allele
        which NetMHCIIpan treats differently because there is
        little population diversity in the DR-alpha gene
        """
        if "DRB" not in parsed_beta_allele.gene:
            raise ValueError("Unexpected allele %s" % parsed_beta_allele)
        return "%s_%s%s" % (
            parsed_beta_allele.gene,
            parsed_beta_allele.allele_family,
            parsed_beta_allele.allele_code)

def
_prepare_drb_allele_name
(
self
,
parsed_beta_allele
)
:
if
"DRB"
not
in
parsed_beta_allele
.
gene
:
raise
ValueError
(
"Unexpected allele %s"
%
parsed_beta_allele
)
return
"%s_%s%s"
%
(
parsed_beta_allele
.
gene
,
parsed_beta_allele
.
allele_family
,
parsed_beta_allele
.
allele_code
)

Assume that we re dealing with a human DRB allele which NetMHCIIpan treats differently because there is little population diversity in the DR - alpha gene


def get_error(data):
    """ return the error if there is a corresponding exception """
    if isinstance(data, dict):
        if 'errors' in data:
            error = data['errors'][0]
        else:
            error = data.get('error', None)

        if isinstance(error, dict):
            if error.get('code') in errors:
                return error

def get_error ( data ) : if isinstance ( data , dict ) : if 'errors' in data : error = data [ 'errors' ] [ 0 ] else : error = data . get ( 'error' , None ) if isinstance ( error , dict ) : if error . get ( 'code' ) in errors : return error


async def throw(response, loads=None, encoding=None, **kwargs):
    """ Get the response data if possible and raise an exception """
    if loads is None:
        loads = data_processing.loads

    data = await data_processing.read(response, loads=loads,
                                      encoding=encoding)

    error = get_error(data)
    if error is not None:
        exception = errors[error['code']]
        raise exception(response=response, error=error, data=data, **kwargs)

    if response.status in statuses:
        exception = statuses[response.status]
        raise exception(response=response, data=data, **kwargs)

    # raise PeonyException if no specific exception was found
    raise PeonyException(response=response, data=data, **kwargs)


def code(self, code):
        """ Decorator to associate a code to an exception """
        def decorator(exception):
            self[code] = exception
            return exception

        return decorator

def code ( self , code ) : def decorator ( exception ) : self [ code ] = exception return exception return decorator


async def prepare_request(self, method, url,
                              headers=None,
                              skip_params=False,
                              proxy=None,
                              **kwargs):
        """
        prepare all the arguments for the request

        Parameters
        ----------
        method : str
            HTTP method used by the request
        url : str
            The url to request
        headers : dict, optional
            Additionnal headers
        proxy : str
            proxy of the request
        skip_params : bool
            Don't use the parameters to sign the request

        Returns
        -------
        dict
            Parameters of the request correctly formatted
        """

        if method.lower() == "post":
            key = 'data'
        else:
            key = 'params'

        if key in kwargs and not skip_params:
            request_params = {key: kwargs.pop(key)}
        else:
            request_params = {}

        request_params.update(dict(method=method.upper(), url=url))

        coro = self.sign(**request_params, skip_params=skip_params,
                         headers=headers)
        request_params['headers'] = await utils.execute(coro)
        request_params['proxy'] = proxy

        kwargs.update(request_params)

        return kwargs

async def prepare_request ( self , method , url , headers = None , skip_params = False , proxy = None , * * kwargs ) : if method . lower ( ) == "post" : key = 'data' else : key = 'params' if key in kwargs and not skip_params : request_params = { key : kwargs . pop ( key ) } else : request_params = { } request_params . update ( dict ( method = method . upper ( ) , url = url ) ) coro = self . sign ( * * request_params , skip_params = skip_params , headers = headers ) request_params [ 'headers' ] = await utils . execute ( coro ) request_params [ 'proxy' ] = proxy kwargs . update ( request_params ) return kwargs


def _user_headers(self, headers=None):
        """ Make sure the user doesn't override the Authorization header """
        h = self.copy()

        if headers is not None:
            keys = set(headers.keys())
            if h.get('Authorization', False):
                keys -= {'Authorization'}

            for key in keys:
                h[key] = headers[key]

        return h

def _user_headers ( self , headers = None ) : h = self . copy ( ) if headers is not None : keys = set ( headers . keys ( ) ) if h . get ( 'Authorization' , False ) : keys -= { 'Authorization' } for key in keys : h [ key ] = headers [ key ] return h

Make sure the user doesn t override the Authorization header


def process_keys(func):
    """
    Raise error for keys that are not strings
    and add the prefix if it is missing
    """

    @wraps(func)
    def decorated(self, k, *args):
        if not isinstance(k, str):
            msg = "%s: key must be a string" % self.__class__.__name__
            raise ValueError(msg)

        if not k.startswith(self.prefix):
            k = self.prefix + k

        return func(self, k, *args)

    return decorated

def
process_keys
(
func
)
:
@
wraps
(
func
)
def
decorated
(
self
,
k
,
*
args
)
:
if
not
isinstance
(
k
,
str
)
:
msg
=
"%s: key must be a string"
%
self
.
__class__
.
__name__
raise
ValueError
(
msg
)
if
not
k
.
startswith
(
self
.
prefix
)
:
k
=
self
.
prefix
+
k
return
func
(
self
,
k
,
*
args
)
return
decorated


def _get(self, text):
        """
            Analyze the text to get the right function

        Parameters
        ----------
        text : str
            The text that could call a function
        """
        if self.strict:
            match = self.prog.match(text)
            if match:
                cmd = match.group()
                if cmd in self:
                    return cmd
        else:
            words = self.prog.findall(text)
            for word in words:
                if word in self:
                    return word

def _get ( self , text ) : if self . strict : match = self . prog . match ( text ) if match : cmd = match . group ( ) if cmd in self : return cmd else : words = self . prog . findall ( text ) for word in words : if word in self : return word


async def run(self, *args, data):
        """ run the function you want """
        cmd = self._get(data.text)

        try:
            if cmd is not None:
                command = self[cmd](*args, data=data)
                return await peony.utils.execute(command)

        except:
            fmt = "Error occurred while running function {cmd}:"
            peony.utils.log_error(fmt.format(cmd=cmd))


def simplified_edges(self):
        """
        A generator for getting all of the edges without consuming extra
        memory.
        """
        for group, edgelist in self.edges.items():
            for u, v, d in edgelist:
                yield (u, v)

def simplified_edges ( self ) : for group , edgelist in self . edges . items ( ) : for u , v , d in edgelist : yield ( u , v )

A generator for getting all of the edges without consuming extra memory .


def has_edge_within_group(self, group):
        """
        Checks whether there are within-group edges or not.
        """
        assert group in self.nodes.keys(),\
            "{0} not one of the group of nodes".format(group)
        nodelist = self.nodes[group]
        for n1, n2 in self.simplified_edges():
            if n1 in nodelist and n2 in nodelist:
                return True

def
has_edge_within_group
(
self
,
group
)
:
assert
group
in
self
.
nodes
.
keys
(
)
,
"{0} not one of the group of nodes"
.
format
(
group
)
nodelist
=
self
.
nodes
[
group
]
for
n1
,
n2
in
self
.
simplified_edges
(
)
:
if
n1
in
nodelist
and
n2
in
nodelist
:
return
True

Checks whether there are within - group edges or not .


def plot_axis(self, rs, theta):
        """
        Renders the axis.
        """
        xs, ys = get_cartesian(rs, theta)
        self.ax.plot(xs, ys, 'black', alpha=0.3)

def plot_axis ( self , rs , theta ) : xs , ys = get_cartesian ( rs , theta ) self . ax . plot ( xs , ys , 'black' , alpha = 0.3 )


def plot_nodes(self, nodelist, theta, group):
        """
        Plots nodes to screen.
        """
        for i, node in enumerate(nodelist):
            r = self.internal_radius + i * self.scale
            x, y = get_cartesian(r, theta)
            circle = plt.Circle(xy=(x, y), radius=self.dot_radius,
                                color=self.node_colormap[group], linewidth=0)
            self.ax.add_patch(circle)

def plot_nodes ( self , nodelist , theta , group ) : for i , node in enumerate ( nodelist ) : r = self . internal_radius + i * self . scale x , y = get_cartesian ( r , theta ) circle = plt . Circle ( xy = ( x , y ) , radius = self . dot_radius , color = self . node_colormap [ group ] , linewidth = 0 ) self . ax . add_patch ( circle )


def group_theta(self, group):
        """
        Computes the theta along which a group's nodes are aligned.
        """
        for i, g in enumerate(self.nodes.keys()):
            if g == group:
                break

        return i * self.major_angle

def group_theta ( self , group ) : for i , g in enumerate ( self . nodes . keys ( ) ) : if g == group : break return i * self . major_angle

Computes the theta along which a group s nodes are aligned .


def find_node_group_membership(self, node):
        """
        Identifies the group for which a node belongs to.
        """
        for group, nodelist in self.nodes.items():
            if node in nodelist:
                return group

def find_node_group_membership ( self , node ) : for group , nodelist in self . nodes . items ( ) : if node in nodelist : return group

Identifies the group for which a node belongs to .


def get_idx(self, node):
        """
        Finds the index of the node in the sorted list.
        """
        group = self.find_node_group_membership(node)
        return self.nodes[group].index(node)

def get_idx ( self , node ) : group = self . find_node_group_membership ( node ) return self . nodes [ group ] . index ( node )

Finds the index of the node in the sorted list .


def node_radius(self, node):
        """
        Computes the radial position of the node.
        """
        return self.get_idx(node) * self.scale + self.internal_radius

def node_radius ( self , node ) : return self . get_idx ( node ) * self . scale + self . internal_radius

Computes the radial position of the node .


def node_theta(self, node):
        """
        Convenience function to find the node's theta angle.
        """
        group = self.find_node_group_membership(node)
        return self.group_theta(group)

def node_theta ( self , node ) : group = self . find_node_group_membership ( node ) return self . group_theta ( group )

Convenience function to find the node s theta angle .


def add_edges(self):
        """
        Draws all of the edges in the graph.
        """
        for group, edgelist in self.edges.items():
            for (u, v, d) in edgelist:
                self.draw_edge(u, v, d, group)

def add_edges ( self ) : for group , edgelist in self . edges . items ( ) : for ( u , v , d ) in edgelist : self . draw_edge ( u , v , d , group )


def draw(self):
        """
        The master function that is called that draws everything.
        """
        self.ax.set_xlim(-self.plot_radius(), self.plot_radius())
        self.ax.set_ylim(-self.plot_radius(), self.plot_radius())

        self.add_axes_and_nodes()
        self.add_edges()

        self.ax.axis('off')

def draw ( self ) : self . ax . set_xlim ( - self . plot_radius ( ) , self . plot_radius ( ) ) self . ax . set_ylim ( - self . plot_radius ( ) , self . plot_radius ( ) ) self . add_axes_and_nodes ( ) self . add_edges ( ) self . ax . axis ( 'off' )

The master function that is called that draws everything .


def adjust_angles(self, start_node, start_angle, end_node, end_angle):
        """
        This function adjusts the start and end angles to correct for
        duplicated axes.
        """
        start_group = self.find_node_group_membership(start_node)
        end_group = self.find_node_group_membership(end_node)

        if start_group == 0 and end_group == len(self.nodes.keys())-1:
            if self.has_edge_within_group(start_group):
                start_angle = correct_negative_angle(start_angle -
                                                     self.minor_angle)
            if self.has_edge_within_group(end_group):
                end_angle = correct_negative_angle(end_angle +
                                                   self.minor_angle)

        elif start_group == len(self.nodes.keys())-1 and end_group == 0:
            if self.has_edge_within_group(start_group):
                start_angle = correct_negative_angle(start_angle +
                                                     self.minor_angle)
            if self.has_edge_within_group(end_group):
                end_angle = correct_negative_angle(end_angle -
                                                   self.minor_angle)

        elif start_group < end_group:
            if self.has_edge_within_group(end_group):
                end_angle = correct_negative_angle(end_angle -
                                                   self.minor_angle)
            if self.has_edge_within_group(start_group):
                start_angle = correct_negative_angle(start_angle +
                                                     self.minor_angle)

        elif end_group < start_group:
            if self.has_edge_within_group(start_group):
                start_angle = correct_negative_angle(start_angle -
                                                     self.minor_angle)
            if self.has_edge_within_group(end_group):
                end_angle = correct_negative_angle(end_angle +
                                                   self.minor_angle)

        return start_angle, end_angle

def adjust_angles ( self , start_node , start_angle , end_node , end_angle ) : start_group = self . find_node_group_membership ( start_node ) end_group = self . find_node_group_membership ( end_node ) if start_group == 0 and end_group == len ( self . nodes . keys ( ) ) - 1 : if self . has_edge_within_group ( start_group ) : start_angle = correct_negative_angle ( start_angle - self . minor_angle ) if self . has_edge_within_group ( end_group ) : end_angle = correct_negative_angle ( end_angle + self . minor_angle ) elif start_group == len ( self . nodes . keys ( ) ) - 1 and end_group == 0 : if self . has_edge_within_group ( start_group ) : start_angle = correct_negative_angle ( start_angle + self . minor_angle ) if self . has_edge_within_group ( end_group ) : end_angle = correct_negative_angle ( end_angle - self . minor_angle ) elif start_group < end_group : if self . has_edge_within_group ( end_group ) : end_angle = correct_negative_angle ( end_angle - self . minor_angle ) if self . has_edge_within_group ( start_group ) : start_angle = correct_negative_angle ( start_angle + self . minor_angle ) elif end_group < start_group : if self . has_edge_within_group ( start_group ) : start_angle = correct_negative_angle ( start_angle - self . minor_angle ) if self . has_edge_within_group ( end_group ) : end_angle = correct_negative_angle ( end_angle + self . minor_angle ) return start_angle , end_angle

This function adjusts the start and end angles to correct for duplicated axes .


def mods_genre(self):
		"""
		Guesses an appropriate MODS XML genre type.
		"""

		type2genre = {
				'conference': 'conference publication',
				'book chapter': 'bibliography',
				'unpublished': 'article'
			}
		tp = str(self.type).lower()
		return type2genre.get(tp, tp)

def
mods_genre
(
self
)
:
type2genre
=
{
'conference'
:
'conference publication'
,
'book chapter'
:
'bibliography'
,
'unpublished'
:
'article'
}
tp
=
str
(
self
.
type
)
.
lower
(
)
return
type2genre
.
get
(
tp
,
tp
)

Guesses an appropriate MODS XML genre type .


def get_publications(context, template='publications/publications.html'):
	"""
	Get all publications.
	"""

	types = Type.objects.filter(hidden=False)
	publications = Publication.objects.select_related()
	publications = publications.filter(external=False, type__in=types)
	publications = publications.order_by('-year', '-month', '-id')

	if not publications:
		return ''

	# load custom links and files
	populate(publications)

	return render_template(template, context['request'], {'publications': publications})

def
get_publications
(
context
,
template
=
'publications/publications.html'
)
:
types
=
Type
.
objects
.
filter
(
hidden
=
False
)
publications
=
Publication
.
objects
.
select_related
(
)
publications
=
publications
.
filter
(
external
=
False
,
type__in
=
types
)
publications
=
publications
.
order_by
(
'-year'
,
'-month'
,
'-id'
)
if
not
publications
:
return
''
# load custom links and files
populate
(
publications
)
return
render_template
(
template
,
context
[
'request'
]
,
{
'publications'
:
publications
}
)


def get_publication(context, id):
	"""
	Get a single publication.
	"""

	pbl = Publication.objects.filter(pk=int(id))

	if len(pbl) < 1:
		return ''

	pbl[0].links = pbl[0].customlink_set.all()
	pbl[0].files = pbl[0].customfile_set.all()

	return render_template(
		'publications/publication.html', context['request'], {'publication': pbl[0]})

def get_publication ( context , id ) : pbl = Publication . objects . filter ( pk = int ( id ) ) if len ( pbl ) < 1 : return '' pbl [ 0 ] . links = pbl [ 0 ] . customlink_set . all ( ) pbl [ 0 ] . files = pbl [ 0 ] . customfile_set . all ( ) return render_template ( 'publications/publication.html' , context [ 'request' ] , { 'publication' : pbl [ 0 ] } )


def get_publication_list(context, list, template='publications/publications.html'):
	"""
	Get a publication list.
	"""

	list = List.objects.filter(list__iexact=list)

	if not list:
		return ''

	list = list[0]
	publications = list.publication_set.all()
	publications = publications.order_by('-year', '-month', '-id')

	if not publications:
		return ''

	# load custom links and files
	populate(publications)

	return render_template(
		template, context['request'], {'list': list, 'publications': publications})

def
get_publication_list
(
context
,
list
,
template
=
'publications/publications.html'
)
:
list
=
List
.
objects
.
filter
(
list__iexact
=
list
)
if
not
list
:
return
''
list
=
list
[
0
]
publications
=
list
.
publication_set
.
all
(
)
publications
=
publications
.
order_by
(
'-year'
,
'-month'
,
'-id'
)
if
not
publications
:
return
''
# load custom links and files
populate
(
publications
)
return
render_template
(
template
,
context
[
'request'
]
,
{
'list'
:
list
,
'publications'
:
publications
}
)


def tex_parse(string):
	"""
	Renders some basic TeX math to HTML.
	"""

	string = string.replace('{', '').replace('}', '')
	def tex_replace(match):
		return \
			sub(r'\^(\w)', r'\1',
			sub(r'\^\{(.*?)\}', r'\1',
			sub(r'\_(\w)', r'\1',
			sub(r'\_\{(.*?)\}', r'\1',
			sub(r'\\(' + GREEK_LETTERS + ')', r'&\1;', match.group(1))))))
	return mark_safe(sub(r'\$([^\$]*)\$', tex_replace, escape(string)))

def tex_parse ( string ) : string = string . replace ( '{' , '' ) . replace ( '}' , '' ) def tex_replace ( match ) : return sub ( r'\^(\w)' , r'\1' , sub ( r'\^\{(.*?)\}' , r'\1' , sub ( r'\_(\w)' , r'\1' , sub ( r'\_\{(.*?)\}' , r'\1' , sub ( r'\\(' + GREEK_LETTERS + ')' , r'&\1;' , match . group ( 1 ) ) ) ) ) ) return mark_safe ( sub ( r'\$([^\$]*)\$' , tex_replace , escape ( string ) ) )


def parse(string):
	"""
	Takes a string in BibTex format and returns a list of BibTex entries, where
	each entry is a dictionary containing the entries' key-value pairs.

	@type  string: string
	@param string: bibliography in BibTex format

	@rtype: list
	@return: a list of dictionaries representing a bibliography
	"""

	# bibliography
	bib = []

	# make sure we are dealing with unicode strings
	if not isinstance(string, six.text_type):
		string = string.decode('utf-8')

	# replace special characters
	for key, value in special_chars:
		string = string.replace(key, value)
	string = re.sub(r'\\[cuHvs]{?([a-zA-Z])}?', r'\1', string)

	# split into BibTex entries
	entries = re.findall(
		r'(?u)@(\w+)[ \t]?{[ \t]*([^,\s]*)[ \t]*,?\s*((?:[^=,\s]+\s*\=\s*(?:"[^"]*"|{(?:[^{}]*|{[^{}]*})*}|[^,}]*),?\s*?)+)\s*}',
		string)

	for entry in entries:
		# parse entry
		pairs = re.findall(r'(?u)([^=,\s]+)\s*\=\s*("[^"]*"|{(?:[^{}]*|{[^{}]*})*}|[^,]*)', entry[2])

		# add to bibliography
		bib.append({'type': entry[0].lower(), 'key': entry[1]})

		for key, value in pairs:
			# post-process key and value
			key = key.lower()
			if value and value[0] == '"' and value[-1] == '"':
				value = value[1:-1]
			if value and value[0] == '{' and value[-1] == '}':
				value = value[1:-1]
			if key not in ['booktitle', 'title']:
				value = value.replace('}', '').replace('{', '')
			else:
				if value.startswith('{') and value.endswith('}'):
					value = value[1:]
					value = value[:-1]
			value = value.strip()
			value = re.sub(r'\s+', ' ', value)

			# store pair in bibliography
			bib[-1][key] = value

	return bib

Takes a string in BibTex format and returns a list of BibTex entries where each entry is a dictionary containing the entries key - value pairs .


def swap(self, qs):
        """
        Swap the positions of this object with a reference object.
        """
        try:
            replacement = qs[0]
        except IndexError:
            # already first/last
            return
        if not self._valid_ordering_reference(replacement):
            raise ValueError(
                "%r can only be swapped with instances of %r which %s equals %r." % (
                    self, self.__class__, self.order_with_respect_to,
                    self._get_order_with_respect_to()
                )
            )
        self.order, replacement.order = replacement.order, self.order
        self.save()
        replacement.save()

Swap the positions of this object with a reference object .


def up(self):
        """
        Move this object up one position.
        """
        self.swap(self.get_ordering_queryset().filter(order__lt=self.order).order_by('-order'))

def up ( self ) : self . swap ( self . get_ordering_queryset ( ) . filter ( order__lt = self . order ) . order_by ( '-order' ) )


def down(self):
        """
        Move this object down one position.
        """
        self.swap(self.get_ordering_queryset().filter(order__gt=self.order))

def down ( self ) : self . swap ( self . get_ordering_queryset ( ) . filter ( order__gt = self . order ) )


def to(self, order):
        """
        Move object to a certain position, updating all affected objects to move accordingly up or down.
        """
        if order is None or self.order == order:
            # object is already at desired position
            return
        qs = self.get_ordering_queryset()
        if self.order > order:
            qs.filter(order__lt=self.order, order__gte=order).update(order=F('order') + 1)
        else:
            qs.filter(order__gt=self.order, order__lte=order).update(order=F('order') - 1)
        self.order = order
        self.save()

def
to
(
self
,
order
)
:
if
order
is
or
self
.
order
==
order
:
# object is already at desired position
return
qs
=
self
.
get_ordering_queryset
(
)
if
self
.
order
>
order
:
qs
.
filter
(
order__lt
=
self
.
order
,
order__gte
=
order
)
.
update
(
order
=
F
(
'order'
)
+
1
)
else
:
qs
.
filter
(
order__gt
=
self
.
order
,
order__lte
=
order
)
.
update
(
order
=
F
(
'order'
)
1
)
self
.
order
=
order
self
.
save
(
)

Move object to a certain position updating all affected objects to move accordingly up or down .


def above(self, ref):
        """
        Move this object above the referenced object.
        """
        if not self._valid_ordering_reference(ref):
            raise ValueError(
                "%r can only be moved above instances of %r which %s equals %r." % (
                    self, self.__class__, self.order_with_respect_to,
                    self._get_order_with_respect_to()
                )
            )
        if self.order == ref.order:
            return
        if self.order > ref.order:
            o = ref.order
        else:
            o = self.get_ordering_queryset().filter(order__lt=ref.order).aggregate(Max('order')).get('order__max') or 0
        self.to(o)

Move this object above the referenced object .


def below(self, ref):
        """
        Move this object below the referenced object.
        """
        if not self._valid_ordering_reference(ref):
            raise ValueError(
                "%r can only be moved below instances of %r which %s equals %r." % (
                    self, self.__class__, self.order_with_respect_to,
                    self._get_order_with_respect_to()
                )
            )
        if self.order == ref.order:
            return
        if self.order > ref.order:
            o = self.get_ordering_queryset().filter(order__gt=ref.order).aggregate(Min('order')).get('order__min') or 0
        else:
            o = ref.order
        self.to(o)

Move this object below the referenced object .


def top(self):
        """
        Move this object to the top of the ordered stack.
        """
        o = self.get_ordering_queryset().aggregate(Min('order')).get('order__min')
        self.to(o)

def top ( self ) : o = self . get_ordering_queryset ( ) . aggregate ( Min ( 'order' ) ) . get ( 'order__min' ) self . to ( o )

Move this object to the top of the ordered stack .


def bottom(self):
        """
        Move this object to the bottom of the ordered stack.
        """
        o = self.get_ordering_queryset().aggregate(Max('order')).get('order__max')
        self.to(o)

def bottom ( self ) : o = self . get_ordering_queryset ( ) . aggregate ( Max ( 'order' ) ) . get ( 'order__max' ) self . to ( o )

Move this object to the bottom of the ordered stack .


def populate(publications):
	"""
	Load custom links and files from database and attach to publications.
	"""

	customlinks = CustomLink.objects.filter(publication__in=publications)
	customfiles = CustomFile.objects.filter(publication__in=publications)

	publications_ = {}
	for publication in publications:
		publication.links = []
		publication.files = []
		publications_[publication.id] = publication

	for link in customlinks:
		publications_[link.publication_id].links.append(link)
	for file in customfiles:
		publications_[file.publication_id].files.append(file)

def populate ( publications ) : customlinks = CustomLink . objects . filter ( publication__in = publications ) customfiles = CustomFile . objects . filter ( publication__in = publications ) publications_ = { } for publication in publications : publication . links = [ ] publication . files = [ ] publications_ [ publication . id ] = publication for link in customlinks : publications_ [ link . publication_id ] . links . append ( link ) for file in customfiles : publications_ [ file . publication_id ] . files . append ( file )

Load custom links and files from database and attach to publications .


def worker(self):
    """
    Calculates the quartet weights for the test at a random
    subsampled chunk of loci.
    """

    ## subsample loci
    fullseqs = self.sample_loci()

    ## find all iterations of samples for this quartet
    liters = itertools.product(*self.imap.values())

    ## run tree inference for each iteration of sampledict
    hashval = uuid.uuid4().hex
    weights = []
    for ridx, lidx in enumerate(liters):

        ## get subalignment for this iteration and make to nex
        a,b,c,d = lidx
        sub = {}
        for i in lidx:
            if self.rmap[i] == "p1":
                sub["A"] = fullseqs[i]
            elif self.rmap[i] == "p2":
                sub["B"] = fullseqs[i]
            elif self.rmap[i] == "p3":
                sub["C"] = fullseqs[i]
            else:
                sub["D"] = fullseqs[i]

        ## write as nexus file
        nex = []
        for tax in list("ABCD"):
            nex.append(">{}         {}".format(tax, sub[tax]))

        ## check for too much missing or lack of variants
        nsites, nvar = count_var(nex)

        ## only run test if there's variation present
        if nvar > self.minsnps:

            ## format as nexus file
            nexus = "{} {}\n".format(4, len(fullseqs[a])) + "\n".join(nex)

            ## infer ML tree
            treeorder = self.run_tree_inference(nexus, "{}.{}".format(hashval, ridx))

            ## add to list
            weights.append(treeorder)

    ## cleanup - remove all files with the hash val
    rfiles = glob.glob(os.path.join(tempfile.tempdir, "*{}*".format(hashval)))
    for rfile in rfiles:
        if os.path.exists(rfile):
            os.remove(rfile)

    ## return result as weights for the set topologies.
    trees = ["ABCD", "ACBD", "ADBC"]
    wdict = {i:float(weights.count(i))/len(weights) for i in trees}
    return wdict

Calculates the quartet weights for the test at a random subsampled chunk of loci .


def get_order(tre):
    """
    return tree order
    """
    anode = tre.tree&">A"
    sister = anode.get_sisters()[0]
    sisters = (anode.name[1:], sister.name[1:])
    others = [i for i in list("ABCD") if i not in sisters]
    return sorted(sisters) + sorted(others)

def get_order ( tre ) : anode = tre . tree & ">A" sister = anode . get_sisters ( ) [ 0 ] sisters = ( anode . name [ 1 : ] , sister . name [ 1 : ] ) others = [ i for i in list ( "ABCD" ) if i not in sisters ] return sorted ( sisters ) + sorted ( others )


def count_var(nex):
    """
    count number of sites with cov=4, and number of variable sites.
    """
    arr = np.array([list(i.split()[-1]) for i in nex])
    miss = np.any(arr=="N", axis=0)
    nomiss = arr[:, ~miss]
    nsnps = np.invert(np.all(nomiss==nomiss[0, :], axis=0)).sum()
    return nomiss.shape[1], nsnps

def count_var ( nex ) : arr = np . array ( [ list ( i . split ( ) [ - 1 ] ) for i in nex ] ) miss = np . any ( arr == "N" , axis = 0 ) nomiss = arr [ : , ~ miss ] nsnps = np . invert ( np . all ( nomiss == nomiss [ 0 , : ] , axis = 0 ) ) . sum ( ) return nomiss . shape [ 1 ] , nsnps

count number of sites with cov = 4 and number of variable sites .


def sample_loci(self):
        """ finds loci with sufficient sampling for this test"""

        ## store idx of passing loci
        idxs = np.random.choice(self.idxs, self.ntests)

        ## open handle, make a proper generator to reduce mem
        with open(self.data) as indata:
            liter = (indata.read().strip().split("|\n"))

        ## store data as dict
        seqdata = {i:"" for i in self.samples}

        ## put chunks into a list
        for idx, loc in enumerate(liter):
            if idx in idxs:
                ## parse chunk
                lines = loc.split("\n")[:-1]
                names = [i.split()[0] for i in lines]
                seqs = [i.split()[1] for i in lines]
                dd = {i:j for i,j in zip(names, seqs)}

                ## add data to concatenated seqdict
                for name in seqdata:
                    if name in names:
                        seqdata[name] += dd[name]
                    else:
                        seqdata[name] += "N"*len(seqs[0])

        ## concatenate into a phylip file
        return seqdata


def run_tree_inference(self, nexus, idx):
        """
        Write nexus to tmpfile, runs phyml tree inference, and parses
        and returns the resulting tree.
        """
        ## create a tmpdir for this test
        tmpdir = tempfile.tempdir
        tmpfile = os.path.join(tempfile.NamedTemporaryFile(
            delete=False,
            prefix=str(idx),
            dir=tmpdir,
        ))

        ## write nexus to tmpfile
        tmpfile.write(nexus)
        tmpfile.flush()

        ## infer the tree
        rax = raxml(name=str(idx), data=tmpfile.name, workdir=tmpdir, N=1, T=2)
        rax.run(force=True, block=True, quiet=True)

        ## clean up
        tmpfile.close()

        ## return tree order
        order = get_order(toytree.tree(rax.trees.bestTree))
        return "".join(order)

def
run_tree_inference
(
self
,
nexus
,
idx
)
:
## create a tmpdir for this test
tmpdir
=
tempfile
.
tempdir
tmpfile
=
os
.
path
.
join
(
tempfile
.
NamedTemporaryFile
(
delete
=
False
,
prefix
=
str
(
idx
)
,
dir
=
tmpdir
,
)
)
## write nexus to tmpfile
tmpfile
.
write
(
nexus
)
tmpfile
.
flush
(
)
## infer the tree
rax
=
raxml
(
name
=
str
(
idx
)
,
data
=
tmpfile
.
name
,
workdir
=
tmpdir
,
N
=
1
,
T
=
2
)
rax
.
run
(
force
=
True
,
block
=
True
,
quiet
=
True
)
## clean up
tmpfile
.
close
(
)
## return tree order
order
=
get_order
(
toytree
.
tree
(
rax
.
trees
.
bestTree
)
)
return
""
.
join
(
order
)

Write nexus to tmpfile runs phyml tree inference and parses and returns the resulting tree .


def plot(self):
        """
        return a toyplot barplot of the results table.
        """
        if self.results_table == None:
            return "no results found"
        else:
            bb = self.results_table.sort_values(
                by=["ABCD", "ACBD"],
                ascending=[False, True],
                )

            ## make a barplot
            import toyplot
            c = toyplot.Canvas(width=600, height=200)
            a = c.cartesian()
            m = a.bars(bb)
            return c, a, m

def
plot
(
self
)
:
if
self
.
results_table
==
:
return
"no results found"
else
:
bb
=
self
.
results_table
.
sort_values
(
by
=
[
"ABCD"
,
"ACBD"
]
,
ascending
=
[
False
,
True
]
,
)
## make a barplot
import
toyplot
c
=
toyplot
.
Canvas
(
width
=
600
,
height
=
200
)
a
=
c
.
cartesian
(
)
m
=
a
.
bars
(
bb
)
return
c
,
a
,
m

return a toyplot barplot of the results table .


def plot_pairwise_dist(self, labels=None, ax=None, cmap=None, cdict=None, metric="euclidean"):
        """
        Plot pairwise distances between all samples

        labels: bool or list
                by default labels aren't included. If labels == True, then labels are read in
                from the vcf file. Alternatively, labels can be passed in as a list, should
                be same length as the number of samples.
        """
        allele_counts = self.genotypes.to_n_alt()
        dist = allel.pairwise_distance(allele_counts, metric=metric)
        if not ax:
            fig = plt.figure(figsize=(5, 5))
            ax = fig.add_subplot(1, 1, 1)

        if isinstance(labels, bool):
            if labels:
                labels = list(self.samples_vcforder)
        elif isinstance(labels, type(None)):
            pass
        else:
            ## If not bool or None (default), then check to make sure the list passed in
            ## is the right length
            if not len(labels) == len(self.samples_vcforder):
                raise IPyradError(LABELS_LENGTH_ERROR.format(len(labels), len(self.samples_vcforder)))

        allel.plot.pairwise_distance(dist, labels=labels, ax=ax, colorbar=False)


def copy(self):
        """ returns a copy of the pca analysis object """
        cp = copy.deepcopy(self)
        cp.genotypes = allel.GenotypeArray(self.genotypes, copy=True)
        return cp

def copy ( self ) : cp = copy . deepcopy ( self ) cp . genotypes = allel . GenotypeArray ( self . genotypes , copy = True ) return cp


def loci2migrate(name, locifile, popdict, mindict=1):
    """
    A function to build an input file for the program migrate from an ipyrad
    .loci file, and a dictionary grouping Samples into populations.

    Parameters:
    -----------
    name: (str)
       The name prefix for the migrate formatted output file.
    locifile: (str)
       The path to the .loci file produced by ipyrad.
    popdict: (dict)
       A Python dictionary grouping Samples into Populations.

    Examples:
    ---------
    You can create the population dictionary by hand, and pass in the path
    to your .loci file as a string.
       >> popdict = {'A': ['a', 'b', 'c'], 'B': ['d', 'e', 'f']}
       >> loci2migrate("outfile.migrate", "./mydata.loci", popdict)

    Or, if you load your ipyrad.Assembly object from it's JSON file, you can
    access the loci file path and population information from there directly.
       >> data = ip.load_json("mydata.json")
       >> loci2migrate("outfile.migrate", data.outfiles.loci, data.populations)

    """

    ## I/O
    outfile = open(name+".migrate", 'w')
    infile = open(locifile, 'r')

    ## minhits dictionary can be an int (all same) or a dictionary (set each)
    if isinstance(mindict, int):
        mindict = {pop: mindict for pop in popdict}
    else:
        mindict = mindict

    ## filter data to only the loci that have data for mindict setting
    keep = []
    MINS = zip(taxa.keys(), minhits)

    ## read in data to sample names
    loci  = infile.read().strip().split("|")[:-1]
    for loc in loci:
        samps = [i.split()[0].replace(">","") for i in loc.split("\n") if ">" in i]
        ## filter for coverage
        GG = []
        for group,mins in MINS:
            GG.append( sum([i in samps for i in taxa[group]]) >= int(mins) )
        if all(GG):
            keep.append(loc)

    ## print data to file
    print >>outfile, len(taxa), len(keep), "( npops nloci for data set", data.name+".loci",")"

    ## print all data for each population at a time
    done = 0
    for group in taxa:
        ## print a list of lengths of each locus
        if not done:
            loclens = [len(loc.split("\n")[1].split()[-1].replace("x","n").replace("n","")) for loc in keep]
            print >>outfile, " ".join(map(str,loclens))
            done += 1

        ## print a list of number of individuals in each locus
        indslist = []
        for loc in keep:
            samps = [i.split()[0].replace(">","") for i in loc.split("\n") if ">" in i]
            inds = sum([i in samps for i in taxa[group]])
            indslist.append(inds)
        print >>outfile, " ".join(map(str,indslist)), group

        ## print sample id, spaces, and sequence data
        #for loc in range(len(keep)):
        for loc in range(len(keep)):
            seqs = [i.split()[-1] for i in keep[loc].split("\n") if \
                    i.split()[0].replace(">","") in taxa[group]]
            for i in range(len(seqs)):
                print >>outfile, group[0:8]+"_"+str(i)+\
                      (" "*(10-len(group[0:8]+"_"+str(i))))+seqs[i].replace("x","n").replace("n","")

    outfile.close()

A function to build an input file for the program migrate from an ipyrad . loci file and a dictionary grouping Samples into populations .


def update(assembly, idict, count):
    """ updates dictionary with the next .5M reads from the super long string
    phylip file. Makes for faster reading. """

    data = iter(open(os.path.join(assembly.dirs.outfiles,
                     assembly.name+".phy"), 'r'))

    ntax, nchar = data.next().strip().split()

    ## read in max N bp at a time
    for line in data:
        tax, seq = line.strip().split()
        idict[tax] = idict[tax][100000:]
        idict[tax] += seq[count:count+100000]
    del line

    return idict

updates dictionary with the next . 5M reads from the super long string phylip file . Makes for faster reading .


def make(assembly, samples):
    """ Make phylip and nexus formats. This is hackish since I'm recycling the
    code whole-hog from pyrad V3. Probably could be good to go back through
    and clean up the conversion code some time.
    """

    ## get the longest name
    longname = max([len(i) for i in assembly.samples.keys()])
    names = [i.name for i in samples]

    partitions = makephy(assembly, samples, longname)
    makenex(assembly, names, longname, partitions)

def
make
(
assembly
,
samples
)
:
## get the longest name
longname
=
max
(
[
len
(
i
)
for
i
in
assembly
.
samples
.
keys
(
)
]
)
names
=
[
i
.
name
for
i
in
samples
]
partitions
=
makephy
(
assembly
,
samples
,
longname
)
makenex
(
assembly
,
names
,
longname
,
partitions
)

Make phylip and nexus formats . This is hackish since I m recycling the code whole - hog from pyrad V3 . Probably could be good to go back through and clean up the conversion code some time .


def sample_cleanup(data, sample):
    """
    Clean up a bunch of loose files.
    """
    umap1file = os.path.join(data.dirs.edits, sample.name+"-tmp-umap1.fastq")
    umap2file = os.path.join(data.dirs.edits, sample.name+"-tmp-umap2.fastq")
    unmapped = os.path.join(data.dirs.refmapping, sample.name+"-unmapped.bam")
    samplesam = os.path.join(data.dirs.refmapping, sample.name+".sam")
    split1 = os.path.join(data.dirs.edits, sample.name+"-split1.fastq")
    split2 = os.path.join(data.dirs.edits, sample.name+"-split2.fastq")
    refmap_derep = os.path.join(data.dirs.edits, sample.name+"-refmap_derep.fastq")
    for f in [umap1file, umap2file, unmapped, samplesam, split1, split2, refmap_derep]:
        try:
            os.remove(f)
        except:
            pass

def sample_cleanup ( data , sample ) : umap1file = os . path . join ( data . dirs . edits , sample . name + "-tmp-umap1.fastq" ) umap2file = os . path . join ( data . dirs . edits , sample . name + "-tmp-umap2.fastq" ) unmapped = os . path . join ( data . dirs . refmapping , sample . name + "-unmapped.bam" ) samplesam = os . path . join ( data . dirs . refmapping , sample . name + ".sam" ) split1 = os . path . join ( data . dirs . edits , sample . name + "-split1.fastq" ) split2 = os . path . join ( data . dirs . edits , sample . name + "-split2.fastq" ) refmap_derep = os . path . join ( data . dirs . edits , sample . name + "-refmap_derep.fastq" ) for f in [ umap1file , umap2file , unmapped , samplesam , split1 , split2 , refmap_derep ] : try : os . remove ( f ) except : pass


def index_reference_sequence(data, force=False):
    """
    Index the reference sequence, unless it already exists. Also make a mapping
    of scaffolds to index numbers for later user in steps 5-6.
    """

    ## get ref file from params
    refseq_file = data.paramsdict['reference_sequence']
    index_files = []

    ## Check for existence of index files. Default to bwa unless you specify smalt
    if "smalt" in data._hackersonly["aligner"]:
        # These are smalt index files. Only referenced here to ensure they exist
        index_files.extend([".sma", ".smi"])
    else:
        index_files.extend([".amb", ".ann", ".bwt", ".pac", ".sa"])

    ## samtools specific index
    index_files.extend([".fai"])

    ## If reference sequence already exists then bail out of this func
    if not force:
        if all([os.path.isfile(refseq_file+i) for i in index_files]):
            return
    #if data._headers:
    #    print(INDEX_MSG.format(data._hackersonly["aligner"]))

    if "smalt" in data._hackersonly["aligner"]:
        ## Create smalt index for mapping
        ## smalt index [-k <wordlen>] [-s <stepsiz>]  <index_name> <reference_file>
        cmd1 = [ipyrad.bins.smalt, "index",
                "-k", str(data._hackersonly["smalt_index_wordlen"]),
                refseq_file,
                refseq_file]
    else:
        ## bwa index <reference_file>
        cmd1 = [ipyrad.bins.bwa, "index", refseq_file]

    ## call the command
    LOGGER.info(" ".join(cmd1))
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    error1 = proc1.communicate()[0]

    ## simple samtools index for grabbing ref seqs
    cmd2 = [ipyrad.bins.samtools, "faidx", refseq_file]
    LOGGER.info(" ".join(cmd2))
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)
    error2 = proc2.communicate()[0]

    ## error handling
    if proc1.returncode:
        raise IPyradWarningExit(error1)
    if error2:
        if "please use bgzip" in error2:
            raise IPyradWarningExit(NO_ZIP_BINS.format(refseq_file))
        else:
            raise IPyradWarningExit(error2)

Index the reference sequence unless it already exists . Also make a mapping of scaffolds to index numbers for later user in steps 5 - 6 .