def is_dot(ip):
    """Return true if the IP address is in dotted decimal notation."""
    octets = str(ip).split('.')
    if len(octets) != 4:
        return False
    for i in octets:
        try:
            val = int(i)
        except ValueError:
            return False
        if val > 255 or val < 0:
            return False
    return True

def is_dot ( ip ) : octets = str ( ip ) . split ( '.' ) if len ( octets ) != 4 : return False for i in octets : try : val = int ( i ) except ValueError : return False if val > 255 or val < 0 : return False return True

Return true if the IP address is in dotted decimal notation .


def is_bin(ip):
    """Return true if the IP address is in binary notation."""
    try:
        ip = str(ip)
        if len(ip) != 32:
            return False
        dec = int(ip, 2)
    except (TypeError, ValueError):
        return False
    if dec > 4294967295 or dec < 0:
        return False
    return True

def is_bin ( ip ) : try : ip = str ( ip ) if len ( ip ) != 32 : return False dec = int ( ip , 2 ) except ( TypeError , ValueError ) : return False if dec > 4294967295 or dec < 0 : return False return True

Return true if the IP address is in binary notation .


def is_oct(ip):
    """Return true if the IP address is in octal notation."""
    try:
        dec = int(str(ip), 8)
    except (TypeError, ValueError):
        return False
    if dec > 0o37777777777 or dec < 0:
        return False
    return True

def is_oct ( ip ) : try : dec = int ( str ( ip ) , 8 ) except ( TypeError , ValueError ) : return False if dec > 0o37777777777 or dec < 0 : return False return True

Return true if the IP address is in octal notation .


def is_dec(ip):
    """Return true if the IP address is in decimal notation."""
    try:
        dec = int(str(ip))
    except ValueError:
        return False
    if dec > 4294967295 or dec < 0:
        return False
    return True

def is_dec ( ip ) : try : dec = int ( str ( ip ) ) except ValueError : return False if dec > 4294967295 or dec < 0 : return False return True

Return true if the IP address is in decimal notation .


def _check_nm(nm, notation):
    """Function internally used to check if the given netmask
    is of the specified notation."""
    # Convert to decimal, and check if it's in the list of valid netmasks.
    _NM_CHECK_FUNCT = {
        NM_DOT: _dot_to_dec,
        NM_HEX: _hex_to_dec,
        NM_BIN: _bin_to_dec,
        NM_OCT: _oct_to_dec,
        NM_DEC: _dec_to_dec_long}
    try:
        dec = _NM_CHECK_FUNCT[notation](nm, check=True)
    except ValueError:
        return False
    if dec in _NETMASKS_VALUES:
        return True
    return False

Function internally used to check if the given netmask is of the specified notation .


def is_bits_nm(nm):
    """Return true if the netmask is in bits notatation."""
    try:
        bits = int(str(nm))
    except ValueError:
        return False
    if bits > 32 or bits < 0:
        return False
    return True

def is_bits_nm ( nm ) : try : bits = int ( str ( nm ) ) except ValueError : return False if bits > 32 or bits < 0 : return False return True

Return true if the netmask is in bits notatation .


def is_wildcard_nm(nm):
    """Return true if the netmask is in wildcard bits notatation."""
    try:
        dec = 0xFFFFFFFF - _dot_to_dec(nm, check=True)
    except ValueError:
        return False
    if dec in _NETMASKS_VALUES:
        return True
    return False

def is_wildcard_nm ( nm ) : try : dec = 0xFFFFFFFF - _dot_to_dec ( nm , check = True ) except ValueError : return False if dec in _NETMASKS_VALUES : return True return False

Return true if the netmask is in wildcard bits notatation .


def _dot_to_dec(ip, check=True):
    """Dotted decimal notation to decimal conversion."""
    if check and not is_dot(ip):
        raise ValueError('_dot_to_dec: invalid IP: "%s"' % ip)
    octets = str(ip).split('.')
    dec = 0
    dec |= int(octets[0]) << 24
    dec |= int(octets[1]) << 16
    dec |= int(octets[2]) << 8
    dec |= int(octets[3])
    return dec

def
_dot_to_dec
(
ip
,
check
=
True
)
:
if
check
and
not
is_dot
(
ip
)
:
raise
ValueError
(
'_dot_to_dec: invalid IP: "%s"'
%
ip
)
octets
=
str
(
ip
)
.
split
(
'.'
)
dec
=
0
dec
|=
int
(
octets
[
0
]
)
<<
24
dec
|=
int
(
octets
[
1
]
)
<<
16
dec
|=
int
(
octets
[
2
]
)
<<
8
dec
|=
int
(
octets
[
3
]
)
return
dec

Dotted decimal notation to decimal conversion .


def _dec_to_dot(ip):
    """Decimal to dotted decimal notation conversion."""
    first = int((ip >> 24) & 255)
    second = int((ip >> 16) & 255)
    third = int((ip >> 8) & 255)
    fourth = int(ip & 255)
    return '%d.%d.%d.%d' % (first, second, third, fourth)

def _dec_to_dot ( ip ) : first = int ( ( ip >> 24 ) & 255 ) second = int ( ( ip >> 16 ) & 255 ) third = int ( ( ip >> 8 ) & 255 ) fourth = int ( ip & 255 ) return '%d.%d.%d.%d' % ( first , second , third , fourth )

Decimal to dotted decimal notation conversion .


def _hex_to_dec(ip, check=True):
    """Hexadecimal to decimal conversion."""
    if check and not is_hex(ip):
        raise ValueError('_hex_to_dec: invalid IP: "%s"' % ip)
    if isinstance(ip, int):
        ip = hex(ip)
    return int(str(ip), 16)

def
_hex_to_dec
(
ip
,
check
=
True
)
:
if
check
and
not
is_hex
(
ip
)
:
raise
ValueError
(
'_hex_to_dec: invalid IP: "%s"'
%
ip
)
if
isinstance
(
ip
,
int
)
:
ip
=
hex
(
ip
)
return
int
(
str
(
ip
)
,
16
)


def _oct_to_dec(ip, check=True):
    """Octal to decimal conversion."""
    if check and not is_oct(ip):
        raise ValueError('_oct_to_dec: invalid IP: "%s"' % ip)
    if isinstance(ip, int):
        ip = oct(ip)
    return int(str(ip), 8)

def
_oct_to_dec
(
ip
,
check
=
True
)
:
if
check
and
not
is_oct
(
ip
)
:
raise
ValueError
(
'_oct_to_dec: invalid IP: "%s"'
%
ip
)
if
isinstance
(
ip
,
int
)
:
ip
=
oct
(
ip
)
return
int
(
str
(
ip
)
,
8
)


def _bin_to_dec(ip, check=True):
    """Binary to decimal conversion."""
    if check and not is_bin(ip):
        raise ValueError('_bin_to_dec: invalid IP: "%s"' % ip)
    if isinstance(ip, int):
        ip = str(ip)
    return int(str(ip), 2)

def
_bin_to_dec
(
ip
,
check
=
True
)
:
if
check
and
not
is_bin
(
ip
)
:
raise
ValueError
(
'_bin_to_dec: invalid IP: "%s"'
%
ip
)
if
isinstance
(
ip
,
int
)
:
ip
=
str
(
ip
)
return
int
(
str
(
ip
)
,
2
)


def _BYTES_TO_BITS():
    """Generate a table to convert a whole byte to binary.
    This code was taken from the Python Cookbook, 2nd edition - O'Reilly."""
    the_table = 256*[None]
    bits_per_byte = list(range(7, -1, -1))
    for n in range(256):
        l = n
        bits = 8*[None]
        for i in bits_per_byte:
            bits[i] = '01'[n & 1]
            n >>= 1
        the_table[l] = ''.join(bits)
    return the_table

def _BYTES_TO_BITS ( ) : the_table = 256 * [ None ] bits_per_byte = list ( range ( 7 , - 1 , - 1 ) ) for n in range ( 256 ) : l = n bits = 8 * [ None ] for i in bits_per_byte : bits [ i ] = '01' [ n & 1 ] n >>= 1 the_table [ l ] = '' . join ( bits ) return the_table

Generate a table to convert a whole byte to binary . This code was taken from the Python Cookbook 2nd edition - O Reilly .


def _dec_to_bin(ip):
    """Decimal to binary conversion."""
    bits = []
    while ip:
        bits.append(_BYTES_TO_BITS[ip & 255])
        ip >>= 8
    bits.reverse()
    return ''.join(bits) or 32*'0'

def _dec_to_bin ( ip ) : bits = [ ] while ip : bits . append ( _BYTES_TO_BITS [ ip & 255 ] ) ip >>= 8 bits . reverse ( ) return '' . join ( bits ) or 32 * '0'


def _bits_to_dec(nm, check=True):
    """Bits to decimal conversion."""
    if check and not is_bits_nm(nm):
        raise ValueError('_bits_to_dec: invalid netmask: "%s"' % nm)
    bits = int(str(nm))
    return VALID_NETMASKS[bits]

def
_bits_to_dec
(
nm
,
check
=
True
)
:
if
check
and
not
is_bits_nm
(
nm
)
:
raise
ValueError
(
'_bits_to_dec: invalid netmask: "%s"'
%
nm
)
bits
=
int
(
str
(
nm
)
)
return
VALID_NETMASKS
[
bits
]


def _wildcard_to_dec(nm, check=False):
    """Wildcard bits to decimal conversion."""
    if check and not is_wildcard_nm(nm):
        raise ValueError('_wildcard_to_dec: invalid netmask: "%s"' % nm)
    return 0xFFFFFFFF - _dot_to_dec(nm, check=False)


def _detect(ip, _isnm):
    """Function internally used to detect the notation of the
    given IP or netmask."""
    ip = str(ip)
    if len(ip) > 1:
        if ip[0:2] == '0x':
            if _CHECK_FUNCT[IP_HEX][_isnm](ip):
                return IP_HEX
        elif ip[0] == '0':
            if _CHECK_FUNCT[IP_OCT][_isnm](ip):
                return IP_OCT
    if _CHECK_FUNCT[IP_DOT][_isnm](ip):
        return IP_DOT
    elif _isnm and _CHECK_FUNCT[NM_BITS][_isnm](ip):
        return NM_BITS
    elif _CHECK_FUNCT[IP_DEC][_isnm](ip):
        return IP_DEC
    elif _isnm and _CHECK_FUNCT[NM_WILDCARD][_isnm](ip):
        return NM_WILDCARD
    elif _CHECK_FUNCT[IP_BIN][_isnm](ip):
        return IP_BIN
    return IP_UNKNOWN

def _detect ( ip , _isnm ) : ip = str ( ip ) if len ( ip ) > 1 : if ip [ 0 : 2 ] == '0x' : if _CHECK_FUNCT [ IP_HEX ] [ _isnm ] ( ip ) : return IP_HEX elif ip [ 0 ] == '0' : if _CHECK_FUNCT [ IP_OCT ] [ _isnm ] ( ip ) : return IP_OCT if _CHECK_FUNCT [ IP_DOT ] [ _isnm ] ( ip ) : return IP_DOT elif _isnm and _CHECK_FUNCT [ NM_BITS ] [ _isnm ] ( ip ) : return NM_BITS elif _CHECK_FUNCT [ IP_DEC ] [ _isnm ] ( ip ) : return IP_DEC elif _isnm and _CHECK_FUNCT [ NM_WILDCARD ] [ _isnm ] ( ip ) : return NM_WILDCARD elif _CHECK_FUNCT [ IP_BIN ] [ _isnm ] ( ip ) : return IP_BIN return IP_UNKNOWN

Function internally used to detect the notation of the given IP or netmask .


def _convert(ip, notation, inotation, _check, _isnm):
    """Internally used to convert IPs and netmasks to other notations."""
    inotation_orig = inotation
    notation_orig = notation
    inotation = _get_notation(inotation)
    notation = _get_notation(notation)
    if inotation is None:
        raise ValueError('_convert: unknown input notation: "%s"' % inotation_orig)
    if notation is None:
        raise ValueError('_convert: unknown output notation: "%s"' % notation_orig)
    docheck = _check or False
    if inotation == IP_UNKNOWN:
        inotation = _detect(ip, _isnm)
        if inotation == IP_UNKNOWN:
            raise ValueError('_convert: unable to guess input notation or invalid value')
        if _check is None:
            docheck = True
    # We _always_ check this case later.
    if _isnm:
        docheck = False
    dec = 0
    if inotation == IP_DOT:
        dec = _dot_to_dec(ip, docheck)
    elif inotation == IP_HEX:
        dec = _hex_to_dec(ip, docheck)
    elif inotation == IP_BIN:
        dec = _bin_to_dec(ip, docheck)
    elif inotation == IP_OCT:
        dec = _oct_to_dec(ip, docheck)
    elif inotation == IP_DEC:
        dec = _dec_to_dec_long(ip, docheck)
    elif _isnm and inotation == NM_BITS:
        dec = _bits_to_dec(ip, docheck)
    elif _isnm and inotation == NM_WILDCARD:
        dec = _wildcard_to_dec(ip, docheck)
    else:
        raise ValueError('_convert: unknown IP/netmask notation: "%s"' % inotation_orig)
    # Ensure this is a valid netmask.
    if _isnm and dec not in _NETMASKS_VALUES:
        raise ValueError('_convert: invalid netmask: "%s"' % ip)
    if notation == IP_DOT:
        return _dec_to_dot(dec)
    elif notation == IP_HEX:
        return _dec_to_hex(dec)
    elif notation == IP_BIN:
        return _dec_to_bin(dec)
    elif notation == IP_OCT:
        return _dec_to_oct(dec)
    elif notation == IP_DEC:
        return _dec_to_dec_str(dec)
    elif _isnm and notation == NM_BITS:
        return _dec_to_bits(dec)
    elif _isnm and notation == NM_WILDCARD:
        return _dec_to_wildcard(dec)
    else:
        raise ValueError('convert: unknown notation: "%s"' % notation_orig)

Internally used to convert IPs and netmasks to other notations .


def convert(ip, notation=IP_DOT, inotation=IP_UNKNOWN, check=True):
    """Convert among IP address notations.

    Given an IP address, this function returns the address
    in another notation.

    @param ip: the IP address.
    @type ip: integers, strings or object with an appropriate __str()__ method.

    @param notation: the notation of the output (default: IP_DOT).
    @type notation: one of the IP_* constants, or the equivalent strings.

    @param inotation: force the input to be considered in the given notation
                    (default the notation of the input is autodetected).
    @type inotation: one of the IP_* constants, or the equivalent strings.

    @param check: force the notation check on the input.
    @type check: True force the check, False force not to check and None
                do the check only if the inotation is unknown.

    @return: a string representing the IP in the selected notation.

    @raise ValueError: raised when the input is in unknown notation."""
    return _convert(ip, notation, inotation, _check=check, _isnm=False)

def convert ( ip , notation = IP_DOT , inotation = IP_UNKNOWN , check = True ) : return _convert ( ip , notation , inotation , _check = check , _isnm = False )


def convert_nm(nm, notation=IP_DOT, inotation=IP_UNKNOWN, check=True):
    """Convert a netmask to another notation."""
    return _convert(nm, notation, inotation, _check=check, _isnm=True)

def convert_nm ( nm , notation = IP_DOT , inotation = IP_UNKNOWN , check = True ) : return _convert ( nm , notation , inotation , _check = check , _isnm = True )


def _add(self, other):
        """Sum two IP addresses."""
        if isinstance(other, self.__class__):
            sum_ = self._ip_dec + other._ip_dec
        elif isinstance(other, int):
            sum_ = self._ip_dec + other
        else:
            other = self.__class__(other)
            sum_ = self._ip_dec + other._ip_dec
        return sum_

def _add ( self , other ) : if isinstance ( other , self . __class__ ) : sum_ = self . _ip_dec + other . _ip_dec elif isinstance ( other , int ) : sum_ = self . _ip_dec + other else : other = self . __class__ ( other ) sum_ = self . _ip_dec + other . _ip_dec return sum_


def _sub(self, other):
        """Subtract two IP addresses."""
        if isinstance(other, self.__class__):
            sub = self._ip_dec - other._ip_dec
        if isinstance(other, int):
            sub = self._ip_dec - other
        else:
            other = self.__class__(other)
            sub = self._ip_dec - other._ip_dec
        return sub

def _sub ( self , other ) : if isinstance ( other , self . __class__ ) : sub = self . _ip_dec - other . _ip_dec if isinstance ( other , int ) : sub = self . _ip_dec - other else : other = self . __class__ ( other ) sub = self . _ip_dec - other . _ip_dec return sub


def get_bits(self):
        """Return the bits notation of the netmask."""
        return _convert(self._ip, notation=NM_BITS,
                        inotation=IP_DOT, _check=False, _isnm=self._isnm)

def get_bits ( self ) : return _convert ( self . _ip , notation = NM_BITS , inotation = IP_DOT , _check = False , _isnm = self . _isnm )

Return the bits notation of the netmask .


def get_wildcard(self):
        """Return the wildcard bits notation of the netmask."""
        return _convert(self._ip, notation=NM_WILDCARD,
                        inotation=IP_DOT, _check=False, _isnm=self._isnm)

def get_wildcard ( self ) : return _convert ( self . _ip , notation = NM_WILDCARD , inotation = IP_DOT , _check = False , _isnm = self . _isnm )

Return the wildcard bits notation of the netmask .


def set(self, ip, netmask=None):
        """Set the IP address and the netmask."""
        if isinstance(ip, str) and netmask is None:
            ipnm = ip.split('/')
            if len(ipnm) != 2:
                raise ValueError('set: invalid CIDR: "%s"' % ip)
            ip = ipnm[0]
            netmask = ipnm[1]
        if isinstance(ip, IPv4Address):
            self._ip = ip
        else:
            self._ip = IPv4Address(ip)
        if isinstance(netmask, IPv4NetMask):
            self._nm = netmask
        else:
            self._nm = IPv4NetMask(netmask)
        ipl = int(self._ip)
        nml = int(self._nm)
        base_add = ipl & nml
        self._ip_num = 0xFFFFFFFF - 1 - nml
        # NOTE: quite a mess.
        #      This's here to handle /32 (-1) and /31 (0) netmasks.
        if self._ip_num in (-1, 0):
            if self._ip_num == -1:
                self._ip_num = 1
            else:
                self._ip_num = 2
            self._net_ip = None
            self._bc_ip = None
            self._first_ip_dec = base_add
            self._first_ip = IPv4Address(self._first_ip_dec, notation=IP_DEC)
            if self._ip_num == 1:
                last_ip_dec = self._first_ip_dec
            else:
                last_ip_dec = self._first_ip_dec + 1
            self._last_ip = IPv4Address(last_ip_dec, notation=IP_DEC)
            return
        self._net_ip = IPv4Address(base_add, notation=IP_DEC)
        self._bc_ip = IPv4Address(base_add + self._ip_num + 1, notation=IP_DEC)
        self._first_ip_dec = base_add + 1
        self._first_ip = IPv4Address(self._first_ip_dec, notation=IP_DEC)
        self._last_ip = IPv4Address(base_add + self._ip_num, notation=IP_DEC)


def set_ip(self, ip):
        """Change the current IP."""
        self.set(ip=ip, netmask=self._nm)

def set_ip ( self , ip ) : self . set ( ip = ip , netmask = self . _nm )


def set_netmask(self, netmask):
        """Change the current netmask."""
        self.set(ip=self._ip, netmask=netmask)

def set_netmask ( self , netmask ) : self . set ( ip = self . _ip , netmask = netmask )


def is_valid_ip(self, ip):
        """Return true if the given address in amongst the usable addresses,
        or if the given CIDR is contained in this one."""
        if not isinstance(ip, (IPv4Address, CIDR)):
            if str(ip).find('/') == -1:
                ip = IPv4Address(ip)
            else:
                # Support for CIDR strings/objects, an idea of Nicola Novello.
                ip = CIDR(ip)
        if isinstance(ip, IPv4Address):
            if ip < self._first_ip or ip > self._last_ip:
                return False
        elif isinstance(ip, CIDR):
            # NOTE: manage /31 networks; 127.0.0.1/31 is considered to
            #       be included in 127.0.0.1/8.
            if ip._nm._ip_dec == 0xFFFFFFFE \
                    and self._nm._ip_dec != 0xFFFFFFFE:
                compare_to_first = self._net_ip._ip_dec
                compare_to_last = self._bc_ip._ip_dec
            else:
                compare_to_first = self._first_ip._ip_dec
                compare_to_last = self._last_ip._ip_dec
            if ip._first_ip._ip_dec < compare_to_first or \
                    ip._last_ip._ip_dec > compare_to_last:
                return False
        return True

Return true if the given address in amongst the usable addresses or if the given CIDR is contained in this one .


async def upload_file(self, bucket, file, uploadpath=None, key=None,
                          ContentType=None, **kw):
        """Upload a file to S3 possibly using the multi-part uploader
        Return the key uploaded
        """
        is_filename = False

        if hasattr(file, 'read'):
            if hasattr(file, 'seek'):
                file.seek(0)
            file = file.read()
            size = len(file)
        elif key:
            size = len(file)
        else:
            is_filename = True
            size = os.stat(file).st_size
            key = os.path.basename(file)

        assert key, 'key not available'

        if not ContentType:
            ContentType, _ = mimetypes.guess_type(key)

        if uploadpath:
            if not uploadpath.endswith('/'):
                uploadpath = '%s/' % uploadpath
            key = '%s%s' % (uploadpath, key)

        params = dict(Bucket=bucket, Key=key)

        if not ContentType:
            ContentType = 'application/octet-stream'

        params['ContentType'] = ContentType

        if size > MULTI_PART_SIZE and is_filename:
            resp = await _multipart(self, file, params)
        elif is_filename:
            with open(file, 'rb') as fp:
                params['Body'] = fp.read()
            resp = await self.put_object(**params)
        else:
            params['Body'] = file
            resp = await self.put_object(**params)
        if 'Key' not in resp:
            resp['Key'] = key
        if 'Bucket' not in resp:
            resp['Bucket'] = bucket
        return resp

async
def
upload_file
(
self
,
bucket
,
file
,
uploadpath
=
,
key
=
,
ContentType
=
,
*
*
kw
)
:
is_filename
=
False
if
hasattr
(
file
,
'read'
)
:
if
hasattr
(
file
,
'seek'
)
:
file
.
seek
(
0
)
file
=
file
.
read
(
)
size
=
len
(
file
)
elif
key
:
size
=
len
(
file
)
else
:
is_filename
=
True
size
=
os
.
stat
(
file
)
.
st_size
key
=
os
.
path
.
basename
(
file
)
assert
key
,
'key not available'
if
not
ContentType
:
ContentType
,
_
=
mimetypes
.
guess_type
(
key
)
if
uploadpath
:
if
not
uploadpath
.
endswith
(
'/'
)
:
uploadpath
=
'%s/'
%
uploadpath
key
=
'%s%s'
%
(
uploadpath
,
key
)
params
=
dict
(
Bucket
=
bucket
,
Key
=
key
)
if
not
ContentType
:
ContentType
=
'application/octet-stream'
params
[
'ContentType'
]
=
ContentType
if
size
>
MULTI_PART_SIZE
and
is_filename
:
resp
=
await
_multipart
(
self
,
file
,
params
)
elif
is_filename
:
with
open
(
file
,
'rb'
)
as
fp
:
params
[
'Body'
]
=
fp
.
read
(
)
resp
=
await
self
.
put_object
(
*
*
params
)
else
:
params
[
'Body'
]
=
file
resp
=
await
self
.
put_object
(
*
*
params
)
if
'Key'
not
in
resp
:
resp
[
'Key'
]
=
key
if
'Bucket'
not
in
resp
:
resp
[
'Bucket'
]
=
bucket
return
resp

Upload a file to S3 possibly using the multi - part uploader Return the key uploaded


async def copy_storage_object(self, source_bucket, source_key,
                                  bucket, key):
        """Copy a file from one bucket into another
        """
        info = await self.head_object(Bucket=source_bucket, Key=source_key)
        size = info['ContentLength']

        if size > MULTI_PART_SIZE:
            result = await _multipart_copy(self, source_bucket, source_key,
                                           bucket, key, size)
        else:
            result = await self.copy_object(
                Bucket=bucket, Key=key,
                CopySource=_source_string(source_bucket, source_key)
            )
        return result

async def copy_storage_object ( self , source_bucket , source_key , bucket , key ) : info = await self . head_object ( Bucket = source_bucket , Key = source_key ) size = info [ 'ContentLength' ] if size > MULTI_PART_SIZE : result = await _multipart_copy ( self , source_bucket , source_key , bucket , key , size ) else : result = await self . copy_object ( Bucket = bucket , Key = key , CopySource = _source_string ( source_bucket , source_key ) ) return result


def upload_folder(self, bucket, folder, key=None, skip=None,
                      content_types=None):
        """Recursively upload a ``folder`` into a backet.

        :param bucket: bucket where to upload the folder to
        :param folder: the folder location in the local file system
        :param key: Optional key where the folder is uploaded
        :param skip: Optional list of files to skip
        :param content_types: Optional dictionary mapping suffixes to
            content types
        :return: a coroutine
        """
        uploader = FolderUploader(self, bucket, folder, key, skip,
                                  content_types)
        return uploader.start()

def upload_folder ( self , bucket , folder , key = None , skip = None , content_types = None ) : uploader = FolderUploader ( self , bucket , folder , key , skip , content_types ) return uploader . start ( )

Recursively upload a folder into a backet .


async def _upload_file(self, full_path):
        """Coroutine for uploading a single file
        """
        rel_path = os.path.relpath(full_path, self.folder)
        key = s3_key(os.path.join(self.key, rel_path))
        ct = self.content_types.get(key.split('.')[-1])
        with open(full_path, 'rb') as fp:
            file = fp.read()
        try:
            await self.botocore.upload_file(self.bucket, file, key=key,
                                            ContentType=ct)
        except Exception as exc:
            LOGGER.error('Could not upload "%s": %s', key, exc)
            self.failures[key] = self.all.pop(full_path)
            return
        size = self.all.pop(full_path)
        self.success[key] = size
        self.total_size += size
        percentage = 100*(1 - len(self.all)/self.total_files)
        message = '{0:.0f}% completed - uploaded "{1}" - {2}'.format(
            percentage, key, convert_bytes(size))
        LOGGER.info(message)


async def trigger(self, event, data=None, socket_id=None):
        '''Trigger an ``event`` on this channel
        '''
        json_data = json.dumps(data, cls=self.pusher.encoder)
        query_string = self.signed_query(event, json_data, socket_id)
        signed_path = "%s?%s" % (self.path, query_string)
        pusher = self.pusher
        absolute_url = pusher.get_absolute_path(signed_path)
        response = await pusher.http.post(
            absolute_url, data=json_data,
            headers=[('Content-Type', 'application/json')])
        response.raise_for_status()
        return response.status_code == 202

async def trigger ( self , event , data = None , socket_id = None ) : json_data = json . dumps ( data , cls = self . pusher . encoder ) query_string = self . signed_query ( event , json_data , socket_id ) signed_path = "%s?%s" % ( self . path , query_string ) pusher = self . pusher absolute_url = pusher . get_absolute_path ( signed_path ) response = await pusher . http . post ( absolute_url , data = json_data , headers = [ ( 'Content-Type' , 'application/json' ) ] ) response . raise_for_status ( ) return response . status_code == 202


async def connect(self):
        '''Connect to a Pusher websocket
        '''
        if not self._consumer:
            waiter = self._waiter = asyncio.Future()
            try:
                address = self._websocket_host()
                self.logger.info('Connect to %s', address)
                self._consumer = await self.http.get(address)
                if self._consumer.status_code != 101:
                    raise PusherError("Could not connect to websocket")
            except Exception as exc:
                waiter.set_exception(exc)
                raise
            else:
                await waiter
        return self._consumer

async
def
connect
(
self
)
:
if
not
self
.
_consumer
:
waiter
=
self
.
_waiter
=
asyncio
.
Future
(
)
try
:
address
=
self
.
_websocket_host
(
)
self
.
logger
.
info
(
'Connect to %s'
,
address
)
self
.
_consumer
=
await
self
.
http
.
get
(
address
)
if
self
.
_consumer
.
status_code
!=
101
:
raise
PusherError
(
"Could not connect to websocket"
)
except
Exception
as
exc
:
waiter
.
set_exception
(
exc
)
raise
else
:
await
waiter
return
self
.
_consumer


def on_message(self, websocket, message):
        '''Handle websocket incoming messages
        '''
        waiter = self._waiter
        self._waiter = None
        encoded = json.loads(message)
        event = encoded.get('event')
        channel = encoded.get('channel')
        data = json.loads(encoded.get('data'))
        try:
            if event == PUSHER_ERROR:
                raise PusherError(data['message'], data['code'])
            elif event == PUSHER_CONNECTION:
                self.socket_id = data.get('socket_id')
                self.logger.info('Succesfully connected on socket %s',
                                 self.socket_id)
                waiter.set_result(self.socket_id)
            elif event == PUSHER_SUBSCRIBED:
                self.logger.info('Succesfully subscribed to %s',
                                 encoded.get('channel'))
            elif channel:
                self[channel]._event(event, data)
        except Exception as exc:
            if waiter:
                waiter.set_exception(exc)
            else:
                self.logger.exception('pusher error')

def
on_message
(
self
,
websocket
,
message
)
:
waiter
=
self
.
_waiter
self
.
_waiter
=
encoded
=
json
.
loads
(
message
)
event
=
encoded
.
get
(
'event'
)
channel
=
encoded
.
get
(
'channel'
)
data
=
json
.
loads
(
encoded
.
get
(
'data'
)
)
try
:
if
event
==
PUSHER_ERROR
:
raise
PusherError
(
data
[
'message'
]
,
data
[
'code'
]
)
elif
event
==
PUSHER_CONNECTION
:
self
.
socket_id
=
data
.
get
(
'socket_id'
)
self
.
logger
.
info
(
'Succesfully connected on socket %s'
,
self
.
socket_id
)
waiter
.
set_result
(
self
.
socket_id
)
elif
event
==
PUSHER_SUBSCRIBED
:
self
.
logger
.
info
(
'Succesfully subscribed to %s'
,
encoded
.
get
(
'channel'
)
)
elif
channel
:
self
[
channel
]
.
_event
(
event
,
data
)
except
Exception
as
exc
:
if
waiter
:
waiter
.
set_exception
(
exc
)
else
:
self
.
logger
.
exception
(
'pusher error'
)


def const_equal(str_a, str_b):
    '''Constant time string comparison'''

    if len(str_a) != len(str_b):
        return False

    result = True
    for i in range(len(str_a)):
        result &= (str_a[i] == str_b[i])

    return result

def const_equal ( str_a , str_b ) : if len ( str_a ) != len ( str_b ) : return False result = True for i in range ( len ( str_a ) ) : result &= ( str_a [ i ] == str_b [ i ] ) return result


def decode_html_entities(html):
    """
    Decodes a limited set of HTML entities.
    """
    if not html:
        return html

    for entity, char in six.iteritems(html_entity_map):
        html = html.replace(entity, char)

    return html

def decode_html_entities ( html ) : if not html : return html for entity , char in six . iteritems ( html_entity_map ) : html = html . replace ( entity , char ) return html

Decodes a limited set of HTML entities .


def set_signature_passphrases(self, signature_passphrases):
        '''Set signature passphrases'''
        self.signature_passphrases = self._update_dict(signature_passphrases,
                                                       {}, replace_data=True)

def set_signature_passphrases ( self , signature_passphrases ) : self . signature_passphrases = self . _update_dict ( signature_passphrases , { } , replace_data = True )


def set_encryption_passphrases(self, encryption_passphrases):
        '''Set encryption passphrases'''
        self.encryption_passphrases = self._update_dict(encryption_passphrases,
                                                        {}, replace_data=True)

def set_encryption_passphrases ( self , encryption_passphrases ) : self . encryption_passphrases = self . _update_dict ( encryption_passphrases , { } , replace_data = True )


def set_algorithms(self, signature=None, encryption=None,
                       serialization=None, compression=None):
        '''Set algorithms used for sealing. Defaults can not be overridden.'''

        self.signature_algorithms = \
            self._update_dict(signature, self.DEFAULT_SIGNATURE)

        self.encryption_algorithms = \
            self._update_dict(encryption, self.DEFAULT_ENCRYPTION)

        self.serialization_algorithms = \
            self._update_dict(serialization, self.DEFAULT_SERIALIZATION)

        self.compression_algorithms = \
            self._update_dict(compression, self.DEFAULT_COMPRESSION)

def set_algorithms ( self , signature = None , encryption = None , serialization = None , compression = None ) : self . signature_algorithms = self . _update_dict ( signature , self . DEFAULT_SIGNATURE ) self . encryption_algorithms = self . _update_dict ( encryption , self . DEFAULT_ENCRYPTION ) self . serialization_algorithms = self . _update_dict ( serialization , self . DEFAULT_SERIALIZATION ) self . compression_algorithms = self . _update_dict ( compression , self . DEFAULT_COMPRESSION )

Set algorithms used for sealing . Defaults can not be overridden .


def get_algorithms(self):
        '''Get algorithms used for sealing'''

        return {
            'signature': self.signature_algorithms,
            'encryption': self.encryption_algorithms,
            'serialization': self.serialization_algorithms,
            'compression': self.compression_algorithms,
        }

def get_algorithms ( self ) : return { 'signature' : self . signature_algorithms , 'encryption' : self . encryption_algorithms , 'serialization' : self . serialization_algorithms , 'compression' : self . compression_algorithms , }


def _set_options(self, options):
        '''Private function for setting options used for sealing'''
        if not options:
            return self.options.copy()

        options = options.copy()

        if 'magic' in options:
            self.set_magic(options['magic'])
            del(options['magic'])

        if 'flags' in options:
            flags = options['flags']
            del(options['flags'])
            for key, value in flags.iteritems():
                if not isinstance(value, bool):
                    raise TypeError('Invalid flag type for: %s' % key)
        else:
            flags = self.options['flags']

        if 'info' in options:
            del(options['info'])

        for key, value in options.iteritems():
            if not isinstance(value, int):
                raise TypeError('Invalid option type for: %s' % key)
            if value < 0 or value > 255:
                raise ValueError('Option value out of range for: %s' % key)

        new_options = self.options.copy()
        new_options.update(options)
        new_options['flags'].update(flags)

        return new_options

def
_set_options
(
self
,
options
)
:
if
not
options
:
return
self
.
options
.
copy
(
)
options
=
options
.
copy
(
)
if
'magic'
in
options
:
self
.
set_magic
(
options
[
'magic'
]
)
del
(
options
[
'magic'
]
)
if
'flags'
in
options
:
flags
=
options
[
'flags'
]
del
(
options
[
'flags'
]
)
for
key
,
value
in
flags
.
iteritems
(
)
:
if
not
isinstance
(
value
,
bool
)
:
raise
TypeError
(
'Invalid flag type for: %s'
%
key
)
else
:
flags
=
self
.
options
[
'flags'
]
if
'info'
in
options
:
del
(
options
[
'info'
]
)
for
key
,
value
in
options
.
iteritems
(
)
:
if
not
isinstance
(
value
,
int
)
:
raise
TypeError
(
'Invalid option type for: %s'
%
key
)
if
value
<
0
or
value
>
255
:
raise
ValueError
(
'Option value out of range for: %s'
%
key
)
new_options
=
self
.
options
.
copy
(
)
new_options
.
update
(
options
)
new_options
[
'flags'
]
.
update
(
flags
)
return
new_options


def verify_signature(self, data):
        '''Verify sealed data signature'''

        data = self._remove_magic(data)
        data = urlsafe_nopadding_b64decode(data)
        options = self._read_header(data)
        data = self._add_magic(data)
        self._unsign_data(data, options)

def verify_signature ( self , data ) : data = self . _remove_magic ( data ) data = urlsafe_nopadding_b64decode ( data ) options = self . _read_header ( data ) data = self . _add_magic ( data ) self . _unsign_data ( data , options )


def _encode(self, data, algorithm, key=None):
        '''Encode data with specific algorithm'''

        if algorithm['type'] == 'hmac':
            return data + self._hmac_generate(data, algorithm, key)
        elif algorithm['type'] == 'aes':
            return self._aes_encrypt(data, algorithm, key)
        elif algorithm['type'] == 'no-serialization':
            return data
        elif algorithm['type'] == 'json':
            return json.dumps(data)
        elif algorithm['type'] == 'no-compression':
            return data
        elif algorithm['type'] == 'gzip':
            return self._zlib_compress(data, algorithm)
        else:
            raise Exception('Algorithm not supported: %s' % algorithm['type'])

def
_encode
(
self
,
data
,
algorithm
,
key
=
)
:
if
algorithm
[
'type'
]
==
'hmac'
:
return
data
+
self
.
_hmac_generate
(
data
,
algorithm
,
key
)
elif
algorithm
[
'type'
]
==
'aes'
:
return
self
.
_aes_encrypt
(
data
,
algorithm
,
key
)
elif
algorithm
[
'type'
]
==
'no-serialization'
:
return
data
elif
algorithm
[
'type'
]
==
'json'
:
return
json
.
dumps
(
data
)
elif
algorithm
[
'type'
]
==
'no-compression'
:
return
data
elif
algorithm
[
'type'
]
==
'gzip'
:
return
self
.
_zlib_compress
(
data
,
algorithm
)
else
:
raise
Exception
(
'Algorithm not supported: %s'
%
algorithm
[
'type'
]
)


def _decode(self, data, algorithm, key=None):
        '''Decode data with specific algorithm'''

        if algorithm['type'] == 'hmac':
            verify_signature = data[-algorithm['hash_size']:]
            data = data[:-algorithm['hash_size']]
            signature = self._hmac_generate(data, algorithm, key)
            if not const_equal(verify_signature, signature):
                raise Exception('Invalid signature')
            return data
        elif algorithm['type'] == 'aes':
            return self._aes_decrypt(data, algorithm, key)
        elif algorithm['type'] == 'no-serialization':
            return data
        elif algorithm['type'] == 'json':
            return json.loads(data)
        elif algorithm['type'] == 'no-compression':
            return data
        elif algorithm['type'] == 'gzip':
            return self._zlib_decompress(data, algorithm)
        else:
            raise Exception('Algorithm not supported: %s' % algorithm['type'])

def
_decode
(
self
,
data
,
algorithm
,
key
=
)
:
if
algorithm
[
'type'
]
==
'hmac'
:
verify_signature
=
data
[
algorithm
[
'hash_size'
]
:
]
data
=
data
[
:
algorithm
[
'hash_size'
]
]
signature
=
self
.
_hmac_generate
(
data
,
algorithm
,
key
)
if
not
const_equal
(
verify_signature
,
signature
)
:
raise
Exception
(
'Invalid signature'
)
return
data
elif
algorithm
[
'type'
]
==
'aes'
:
return
self
.
_aes_decrypt
(
data
,
algorithm
,
key
)
elif
algorithm
[
'type'
]
==
'no-serialization'
:
return
data
elif
algorithm
[
'type'
]
==
'json'
:
return
json
.
loads
(
data
)
elif
algorithm
[
'type'
]
==
'no-compression'
:
return
data
elif
algorithm
[
'type'
]
==
'gzip'
:
return
self
.
_zlib_decompress
(
data
,
algorithm
)
else
:
raise
Exception
(
'Algorithm not supported: %s'
%
algorithm
[
'type'
]
)


def _sign_data(self, data, options):
        '''Add signature to data'''

        if options['signature_algorithm_id'] not in self.signature_algorithms:
            raise Exception('Unknown signature algorithm id: %d'
                            % options['signature_algorithm_id'])

        signature_algorithm = \
            self.signature_algorithms[options['signature_algorithm_id']]

        algorithm = self._get_algorithm_info(signature_algorithm)

        key_salt = get_random_bytes(algorithm['salt_size'])
        key = self._generate_key(options['signature_passphrase_id'],
                            self.signature_passphrases, key_salt, algorithm)

        data = self._encode(data, algorithm, key)

        return data + key_salt

def
_sign_data
(
self
,
data
,
options
)
:
if
options
[
'signature_algorithm_id'
]
not
in
self
.
signature_algorithms
:
raise
Exception
(
'Unknown signature algorithm id: %d'
%
options
[
'signature_algorithm_id'
]
)
signature_algorithm
=
self
.
signature_algorithms
[
options
[
'signature_algorithm_id'
]
]
algorithm
=
self
.
_get_algorithm_info
(
signature_algorithm
)
key_salt
=
get_random_bytes
(
algorithm
[
'salt_size'
]
)
key
=
self
.
_generate_key
(
options
[
'signature_passphrase_id'
]
,
self
.
signature_passphrases
,
key_salt
,
algorithm
)
data
=
self
.
_encode
(
data
,
algorithm
,
key
)
return
data
+
key_salt


def _unsign_data(self, data, options):
        '''Verify and remove signature'''

        if options['signature_algorithm_id'] not in self.signature_algorithms:
            raise Exception('Unknown signature algorithm id: %d'
                            % options['signature_algorithm_id'])

        signature_algorithm = \
            self.signature_algorithms[options['signature_algorithm_id']]

        algorithm = self._get_algorithm_info(signature_algorithm)

        key_salt = ''
        if algorithm['salt_size']:
            key_salt = data[-algorithm['salt_size']:]
            data = data[:-algorithm['salt_size']]

        key = self._generate_key(options['signature_passphrase_id'],
                            self.signature_passphrases, key_salt, algorithm)

        data = self._decode(data, algorithm, key)

        return data

def
_unsign_data
(
self
,
data
,
options
)
:
if
options
[
'signature_algorithm_id'
]
not
in
self
.
signature_algorithms
:
raise
Exception
(
'Unknown signature algorithm id: %d'
%
options
[
'signature_algorithm_id'
]
)
signature_algorithm
=
self
.
signature_algorithms
[
options
[
'signature_algorithm_id'
]
]
algorithm
=
self
.
_get_algorithm_info
(
signature_algorithm
)
key_salt
=
''
if
algorithm
[
'salt_size'
]
:
key_salt
=
data
[
algorithm
[
'salt_size'
]
:
]
data
=
data
[
:
algorithm
[
'salt_size'
]
]
key
=
self
.
_generate_key
(
options
[
'signature_passphrase_id'
]
,
self
.
signature_passphrases
,
key_salt
,
algorithm
)
data
=
self
.
_decode
(
data
,
algorithm
,
key
)
return
data


def _remove_magic(self, data):
        '''Verify and remove magic'''

        if not self.magic:
            return data

        magic_size = len(self.magic)
        magic = data[:magic_size]
        if magic != self.magic:
            raise Exception('Invalid magic')
        data = data[magic_size:]

        return data

def
_remove_magic
(
self
,
data
)
:
if
not
self
.
magic
:
return
data
magic_size
=
len
(
self
.
magic
)
magic
=
data
[
:
magic_size
]
if
magic
!=
self
.
magic
:
raise
Exception
(
'Invalid magic'
)
data
=
data
[
magic_size
:
]
return
data


def _add_header(self, data, options):
        '''Add header to data'''

        # pylint: disable=W0142

        version_info = self._get_version_info(options['version'])

        flags = options['flags']

        header_flags = dict(
            (i, str(int(j))) for i, j in options['flags'].iteritems())
        header_flags = ''.join(version_info['flags'](**header_flags))
        header_flags = int(header_flags, 2)
        options['flags'] = header_flags

        header = version_info['header']
        header = header(**options)
        header = pack(version_info['header_format'], *header)

        if 'timestamp' in flags and flags['timestamp']:
            timestamp = long(time())
            timestamp = pack(version_info['timestamp_format'], timestamp)
            header = header + timestamp

        return header + data

def
_add_header
(
self
,
data
,
options
)
:
# pylint: disable=W0142
version_info
=
self
.
_get_version_info
(
options
[
'version'
]
)
flags
=
options
[
'flags'
]
header_flags
=
dict
(
(
i
,
str
(
int
(
j
)
)
)
for
i
,
j
in
options
[
'flags'
]
.
iteritems
(
)
)
header_flags
=
''
.
join
(
version_info
[
'flags'
]
(
*
*
header_flags
)
)
header_flags
=
int
(
header_flags
,
2
)
options
[
'flags'
]
=
header_flags
header
=
version_info
[
'header'
]
header
=
header
(
*
*
options
)
header
=
pack
(
version_info
[
'header_format'
]
,
*
header
)
if
'timestamp'
in
flags
and
flags
[
'timestamp'
]
:
timestamp
=
long
(
time
(
)
)
timestamp
=
pack
(
version_info
[
'timestamp_format'
]
,
timestamp
)
header
=
header
+
timestamp
return
header
+
data


def _read_header(self, data):
        '''Read header from data'''

        # pylint: disable=W0212

        version = self._read_version(data)
        version_info = self._get_version_info(version)
        header_data = data[:version_info['header_size']]
        header = version_info['header']
        header = header._make(
            unpack(version_info['header_format'], header_data))
        header = dict(header._asdict())

        flags = list("{0:0>8b}".format(header['flags']))
        flags = dict(version_info['flags']._make(flags)._asdict())
        flags = dict((i, bool(int(j))) for i, j in flags.iteritems())
        header['flags'] = flags

        timestamp = None
        if flags['timestamp']:
            ts_start = version_info['header_size']
            ts_end = ts_start + version_info['timestamp_size']
            timestamp_data = data[ts_start:ts_end]
            timestamp = unpack(
                version_info['timestamp_format'], timestamp_data)[0]
        header['info'] = {'timestamp': timestamp}

        return header

def
_read_header
(
self
,
data
)
:
# pylint: disable=W0212
version
=
self
.
_read_version
(
data
)
version_info
=
self
.
_get_version_info
(
version
)
header_data
=
data
[
:
version_info
[
'header_size'
]
]
header
=
version_info
[
'header'
]
header
=
header
.
_make
(
unpack
(
version_info
[
'header_format'
]
,
header_data
)
)
header
=
dict
(
header
.
_asdict
(
)
)
flags
=
list
(
"{0:0>8b}"
.
format
(
header
[
'flags'
]
)
)
flags
=
dict
(
version_info
[
'flags'
]
.
_make
(
flags
)
.
_asdict
(
)
)
flags
=
dict
(
(
i
,
bool
(
int
(
j
)
)
)
for
i
,
j
in
flags
.
iteritems
(
)
)
header
[
'flags'
]
=
flags
timestamp
=
if
flags
[
'timestamp'
]
:
ts_start
=
version_info
[
'header_size'
]
ts_end
=
ts_start
+
version_info
[
'timestamp_size'
]
timestamp_data
=
data
[
ts_start
:
ts_end
]
timestamp
=
unpack
(
version_info
[
'timestamp_format'
]
,
timestamp_data
)
[
0
]
header
[
'info'
]
=
{
'timestamp'
:
timestamp
}
return
header


def _remove_header(self, data, options):
        '''Remove header from data'''

        version_info = self._get_version_info(options['version'])
        header_size = version_info['header_size']

        if options['flags']['timestamp']:
            header_size += version_info['timestamp_size']

        data = data[header_size:]

        return data

def _remove_header ( self , data , options ) : version_info = self . _get_version_info ( options [ 'version' ] ) header_size = version_info [ 'header_size' ] if options [ 'flags' ] [ 'timestamp' ] : header_size += version_info [ 'timestamp_size' ] data = data [ header_size : ] return data


def _read_version(self, data):
        '''Read header version from data'''

        version = ord(data[0])
        if version not in self.VERSIONS:
            raise Exception('Version not defined: %d' % version)
        return version

def
_read_version
(
self
,
data
)
:
version
=
ord
(
data
[
0
]
)
if
version
not
in
self
.
VERSIONS
:
raise
Exception
(
'Version not defined: %d'
%
version
)
return
version


def _get_algorithm_info(self, algorithm_info):
        '''Get algorithm info'''

        if algorithm_info['algorithm'] not in self.ALGORITHMS:
            raise Exception('Algorithm not supported: %s'
                            % algorithm_info['algorithm'])

        algorithm = self.ALGORITHMS[algorithm_info['algorithm']]
        algorithm_info.update(algorithm)

        return algorithm_info

def
_get_algorithm_info
(
self
,
algorithm_info
)
:
if
algorithm_info
[
'algorithm'
]
not
in
self
.
ALGORITHMS
:
raise
Exception
(
'Algorithm not supported: %s'
%
algorithm_info
[
'algorithm'
]
)
algorithm
=
self
.
ALGORITHMS
[
algorithm_info
[
'algorithm'
]
]
algorithm_info
.
update
(
algorithm
)
return
algorithm_info


def _generate_key(pass_id, passphrases, salt, algorithm):
        '''Generate and return PBKDF2 key'''

        if pass_id not in passphrases:
            raise Exception('Passphrase not defined for id: %d' % pass_id)

        passphrase = passphrases[pass_id]

        if len(passphrase) < 32:
            raise Exception('Passphrase less than 32 characters long')

        digestmod = EncryptedPickle._get_hashlib(algorithm['pbkdf2_algorithm'])

        encoder = PBKDF2(passphrase, salt,
                         iterations=algorithm['pbkdf2_iterations'],
                         digestmodule=digestmod)

        return encoder.read(algorithm['key_size'])


def _update_dict(data, default_data, replace_data=False):
        '''Update algorithm definition type dictionaries'''

        if not data:
            data = default_data.copy()
            return data

        if not isinstance(data, dict):
            raise TypeError('Value not dict type')
        if len(data) > 255:
            raise ValueError('More than 255 values defined')
        for i in data.keys():
            if not isinstance(i, int):
                raise TypeError('Index not int type')
            if i < 0 or i > 255:
                raise ValueError('Index value out of range')

        if not replace_data:
            data.update(default_data)

        return data

def
_update_dict
(
data
,
default_data
,
replace_data
=
False
)
:
if
not
data
:
data
=
default_data
.
copy
(
)
return
data
if
not
isinstance
(
data
,
dict
)
:
raise
TypeError
(
'Value not dict type'
)
if
len
(
data
)
>
255
:
raise
ValueError
(
'More than 255 values defined'
)
for
i
in
data
.
keys
(
)
:
if
not
isinstance
(
i
,
int
)
:
raise
TypeError
(
'Index not int type'
)
if
i
<
0
or
i
>
255
:
raise
ValueError
(
'Index value out of range'
)
if
not
replace_data
:
data
.
update
(
default_data
)
return
data


def getTableOfContents(self):
        """
        This function populates the internal tableOfContents list with the contents
        of the zip file TOC. If the server does not support ranged requests, this will raise
        and exception. It will also throw an exception if the TOC cannot be found.
        """

        self.directory_size = self.getDirectorySize()
        if self.directory_size > 65536:
            self.directory_size += 2
            self.requestContentDirectory()

        # and find the offset from start of file where it can be found
        directory_start = unpack("i", self.raw_bytes[self.directory_end + 16: self.directory_end + 20])[0]

        # find the data in the raw_bytes
        self.raw_bytes = self.raw_bytes
        current_start = directory_start - self.start
        filestart = 0
        compressedsize = 0
        tableOfContents = []

        try:
            while True:
                # get file name size (n), extra len (m) and comm len (k)
                zip_n = unpack("H", self.raw_bytes[current_start + 28: current_start + 28 + 2])[0]
                zip_m = unpack("H", self.raw_bytes[current_start + 30: current_start + 30 + 2])[0]
                zip_k = unpack("H", self.raw_bytes[current_start + 32: current_start + 32 + 2])[0]

                filename = self.raw_bytes[current_start + 46: current_start + 46 + zip_n]

                # check if this is the index file
                filestart = unpack("I", self.raw_bytes[current_start + 42: current_start + 42 + 4])[0]
                compressedsize = unpack("I", self.raw_bytes[current_start + 20: current_start + 20 + 4])[0]
                uncompressedsize = unpack("I", self.raw_bytes[current_start + 24: current_start + 24 + 4])[0]
                tableItem = {
                    'filename': filename,
                    'compressedsize': compressedsize,
                    'uncompressedsize': uncompressedsize,
                    'filestart': filestart
                }
                tableOfContents.append(tableItem)

                # not this file, move along
                current_start = current_start + 46 + zip_n + zip_m + zip_k
        except:
            pass

        self.tableOfContents = tableOfContents
        return tableOfContents

This function populates the internal tableOfContents list with the contents of the zip file TOC . If the server does not support ranged requests this will raise and exception . It will also throw an exception if the TOC cannot be found .


def extractFile(self, filename):
        """
        This function will extract a single file from the remote zip without downloading
        the entire zip file. The filename argument should match whatever is in the 'filename'
        key of the tableOfContents.
        """
        files = [x for x in self.tableOfContents if x['filename'] == filename]
        if len(files) == 0:
            raise FileNotFoundException()

        fileRecord = files[0]

        # got here? need to fetch the file size
        metaheadroom = 1024  # should be enough
        request = urllib2.Request(self.zipURI)
        start = fileRecord['filestart']
        end = fileRecord['filestart'] + fileRecord['compressedsize'] + metaheadroom
        request.headers['Range'] = "bytes=%s-%s" % (start, end)
        handle = urllib2.urlopen(request)

        # make sure the response is ranged
        return_range = handle.headers.get('Content-Range')
        if return_range != "bytes %d-%d/%s" % (start, end, self.filesize):
            raise Exception("Ranged requests are not supported for this URI")

        filedata = handle.read()

        # find start of raw file data
        zip_n = unpack("H", filedata[26:28])[0]
        zip_m = unpack("H", filedata[28:30])[0]

        # check compressed size
        has_data_descriptor = bool(unpack("H", filedata[6:8])[0] & 8)
        comp_size = unpack("I", filedata[18:22])[0]
        if comp_size == 0 and has_data_descriptor:
            # assume compressed size in the Central Directory is correct
            comp_size = fileRecord['compressedsize']
        elif comp_size != fileRecord['compressedsize']:
            raise Exception("Something went wrong. Directory and file header disagree of compressed file size")

        raw_zip_data = filedata[30 + zip_n + zip_m: 30 + zip_n + zip_m + comp_size]
        uncompressed_data = ""

        # can't decompress if stored without compression
        compression_method = unpack("H", filedata[8:10])[0]
        if compression_method == 0:
          return raw_zip_data

        dec = zlib.decompressobj(-zlib.MAX_WBITS)
        for chunk in raw_zip_data:
            rv = dec.decompress(chunk)
            if rv:
                uncompressed_data = uncompressed_data + rv

        return uncompressed_data

This function will extract a single file from the remote zip without downloading the entire zip file . The filename argument should match whatever is in the filename key of the tableOfContents .


def do_photometry(self):
        """
        Does photometry and estimates uncertainties by calculating the scatter around a linear fit to the data
        in each orientation. This function is called by other functions and generally the user will not need
        to interact with it directly.
        """

        std_f = np.zeros(4)
        data_save = np.zeros_like(self.postcard)
        self.obs_flux = np.zeros_like(self.reference_flux)

        for i in range(4):
            g = np.where(self.qs == i)[0]
            wh = np.where(self.times[g] > 54947)

            data_save[g] = np.roll(self.postcard[g], int(self.roll_best[i,0]), axis=1)
            data_save[g] = np.roll(data_save[g], int(self.roll_best[i,1]), axis=2)

            self.target_flux_pixels = data_save[:,self.targets == 1]
            self.target_flux = np.sum(self.target_flux_pixels, axis=1)

            self.obs_flux[g] = self.target_flux[g] / self.reference_flux[g]
            self.obs_flux[g] /= np.median(self.obs_flux[g[wh]])

            fitline = np.polyfit(self.times[g][wh], self.obs_flux[g][wh], 1)
            std_f[i] = np.max([np.std(self.obs_flux[g][wh]/(fitline[0]*self.times[g][wh]+fitline[1])), 0.001])

        self.flux_uncert = std_f

def do_photometry ( self ) : std_f = np . zeros ( 4 ) data_save = np . zeros_like ( self . postcard ) self . obs_flux = np . zeros_like ( self . reference_flux ) for i in range ( 4 ) : g = np . where ( self . qs == i ) [ 0 ] wh = np . where ( self . times [ g ] > 54947 ) data_save [ g ] = np . roll ( self . postcard [ g ] , int ( self . roll_best [ i , 0 ] ) , axis = 1 ) data_save [ g ] = np . roll ( data_save [ g ] , int ( self . roll_best [ i , 1 ] ) , axis = 2 ) self . target_flux_pixels = data_save [ : , self . targets == 1 ] self . target_flux = np . sum ( self . target_flux_pixels , axis = 1 ) self . obs_flux [ g ] = self . target_flux [ g ] / self . reference_flux [ g ] self . obs_flux [ g ] /= np . median ( self . obs_flux [ g [ wh ] ] ) fitline = np . polyfit ( self . times [ g ] [ wh ] , self . obs_flux [ g ] [ wh ] , 1 ) std_f [ i ] = np . max ( [ np . std ( self . obs_flux [ g ] [ wh ] / ( fitline [ 0 ] * self . times [ g ] [ wh ] + fitline [ 1 ] ) ) , 0.001 ] ) self . flux_uncert = std_f

Does photometry and estimates uncertainties by calculating the scatter around a linear fit to the data in each orientation . This function is called by other functions and generally the user will not need to interact with it directly .


def generate_panel(self, img):
        """
        Creates the figure shown in ``adjust_aperture`` for visualization purposes. Called by other functions
        and generally not called by the user directly.

        Args:
            img: The data frame to be passed through to be plotted. A cutout of the ``integrated_postcard``

        """
        plt.figure(figsize=(14,6))
        ax = plt.gca()
        fig = plt.gcf()
        plt.subplot(122)

        data_save = np.zeros_like(self.postcard)
        self.roll_best = np.zeros((4,2))

        for i in range(4):
            g = np.where(self.qs == i)[0]
            wh = np.where(self.times[g] > 54947)

            self.roll_best[i] = self.do_rolltest(g, wh)

        self.do_photometry()
        for i in range(4):
            g = np.where(self.qs == i)[0]
            plt.errorbar(self.times[g], self.obs_flux[g], yerr=self.flux_uncert[i], fmt=fmt[i])

        plt.xlabel('Time', fontsize=20)
        plt.ylabel('Relative Flux', fontsize=20)

        plt.subplot(121)
        implot = plt.imshow(img, interpolation='nearest', cmap='gray', vmin=98000*52, vmax=104000*52)
        cid = fig.canvas.mpl_connect('button_press_event', self.onclick)

        plt.show(block=True)

def
generate_panel
(
self
,
img
)
:
plt
.
figure
(
figsize
=
(
14
,
6
)
)
ax
=
plt
.
gca
(
)
fig
=
plt
.
gcf
(
)
plt
.
subplot
(
122
)
data_save
=
np
.
zeros_like
(
self
.
postcard
)
self
.
roll_best
=
np
.
zeros
(
(
4
,
2
)
)
for
i
in
range
(
4
)
:
g
=
np
.
where
(
self
.
qs
==
i
)
[
0
]
wh
=
np
.
where
(
self
.
times
[
g
]
>
54947
)
self
.
roll_best
[
i
]
=
self
.
do_rolltest
(
g
,
wh
)
self
.
do_photometry
(
)
for
i
in
range
(
4
)
:
g
=
np
.
where
(
self
.
qs
==
i
)
[
0
]
plt
.
errorbar
(
self
.
times
[
g
]
,
self
.
obs_flux
[
g
]
,
yerr
=
self
.
flux_uncert
[
i
]
,
fmt
=
fmt
[
i
]
)
plt
.
xlabel
(
'Time'
,
fontsize
=
20
)
plt
.
ylabel
(
'Relative Flux'
,
fontsize
=
20
)
plt
.
subplot
(
121
)
implot
=
plt
.
imshow
(
img
,
interpolation
=
'nearest'
,
cmap
=
'gray'
,
vmin
=
98000
*
52
,
vmax
=
104000
*
52
)
cid
=
fig
.
canvas
.
mpl_connect
(
'button_press_event'
,
self
.
onclick
)
plt
.
show
(
block
=
True
)

Creates the figure shown in adjust_aperture for visualization purposes . Called by other functions and generally not called by the user directly .


def calc_centroids(self):
        """
        Identify the centroid positions for the target star at all epochs. Useful for verifying that there is
        no correlation between flux and position, as might be expected for high proper motion stars.
        """
        self.cm = np.zeros((len(self.postcard), 2))
        for i in range(len(self.postcard)):
            target = self.postcard[i]
            target[self.targets != 1] = 0.0
            self.cm[i] = center_of_mass(target)

def calc_centroids ( self ) : self . cm = np . zeros ( ( len ( self . postcard ) , 2 ) ) for i in range ( len ( self . postcard ) ) : target = self . postcard [ i ] target [ self . targets != 1 ] = 0.0 self . cm [ i ] = center_of_mass ( target )

Identify the centroid positions for the target star at all epochs . Useful for verifying that there is no correlation between flux and position as might be expected for high proper motion stars .


def define_spotsignal(self):
        """
        Identify the "expected" flux value at the time of each observation based on the
        Kepler long-cadence data, to ensure variations observed are not the effects of a single
        large starspot. Only works if the target star was targeted for long or short cadence
        observations during the primary mission.
        """
        client = kplr.API()
        star = client.star(self.kic)

        lcs = star.get_light_curves(short_cadence=False)
        time, flux, ferr, qual = [], [], [], []
        for lc in lcs:
            with lc.open() as f:
                hdu_data = f[1].data
                time.append(hdu_data["time"])
                flux.append(hdu_data["pdcsap_flux"])
                ferr.append(hdu_data["pdcsap_flux_err"])
                qual.append(hdu_data["sap_quality"])
            tout = np.array([])
            fout = np.array([])
            eout = np.array([])
            for i in range(len(flux)):
                t = time[i][qual[i] == 0]
                f = flux[i][qual[i] == 0]
                e = ferr[i][qual[i] == 0]

                t = t[np.isfinite(f)]
                e = e[np.isfinite(f)]
                f = f[np.isfinite(f)]

                e /= np.median(f)
                f /= np.median(f)
                tout = np.append(tout, t[50:]+54833)
                fout = np.append(fout, f[50:])
                eout = np.append(eout, e[50:])

            self.spot_signal = np.zeros(52)

            for i in range(len(self.times)):
                if self.times[i] < 55000:
                    self.spot_signal[i] = 1.0
                else:
                    self.spot_signal[i] = fout[np.abs(self.times[i] - tout) == np.min(np.abs(self.times[i] - tout))]

def define_spotsignal ( self ) : client = kplr . API ( ) star = client . star ( self . kic ) lcs = star . get_light_curves ( short_cadence = False ) time , flux , ferr , qual = [ ] , [ ] , [ ] , [ ] for lc in lcs : with lc . open ( ) as f : hdu_data = f [ 1 ] . data time . append ( hdu_data [ "time" ] ) flux . append ( hdu_data [ "pdcsap_flux" ] ) ferr . append ( hdu_data [ "pdcsap_flux_err" ] ) qual . append ( hdu_data [ "sap_quality" ] ) tout = np . array ( [ ] ) fout = np . array ( [ ] ) eout = np . array ( [ ] ) for i in range ( len ( flux ) ) : t = time [ i ] [ qual [ i ] == 0 ] f = flux [ i ] [ qual [ i ] == 0 ] e = ferr [ i ] [ qual [ i ] == 0 ] t = t [ np . isfinite ( f ) ] e = e [ np . isfinite ( f ) ] f = f [ np . isfinite ( f ) ] e /= np . median ( f ) f /= np . median ( f ) tout = np . append ( tout , t [ 50 : ] + 54833 ) fout = np . append ( fout , f [ 50 : ] ) eout = np . append ( eout , e [ 50 : ] ) self . spot_signal = np . zeros ( 52 ) for i in range ( len ( self . times ) ) : if self . times [ i ] < 55000 : self . spot_signal [ i ] = 1.0 else : self . spot_signal [ i ] = fout [ np . abs ( self . times [ i ] - tout ) == np . min ( np . abs ( self . times [ i ] - tout ) ) ]

Identify the expected flux value at the time of each observation based on the Kepler long - cadence data to ensure variations observed are not the effects of a single large starspot . Only works if the target star was targeted for long or short cadence observations during the primary mission .


def model_uncert(self):
        """
        Estimate the photometric uncertainties on each data point following Equation A.2 of The Paper.
        Based on the kepcal package of Dan Foreman-Mackey.
        """
        Y = self.photometry_array.T
        Y /= np.median(Y, axis=1)[:, None]
        C = np.median(Y, axis=0)

        nstars, nobs = np.shape(Y)

        Z = np.empty((nstars, 4))

        qs = self.qs.astype(int)

        for s in range(4):
            Z[:, s] = np.median((Y / C)[:, qs == s], axis=1)

        resid2 = (Y - Z[:, qs] * C)**2
        z = Z[:, qs]
        trend = z * C[None, :]

        lnS = np.log(np.nanmedian(resid2, axis=0))
        jitter = np.log(0.1*np.nanmedian(np.abs(np.diff(Y, axis=1))))

        cal_ferr = np.sqrt(np.exp(2*(jitter/trend))+z**2*np.exp(lnS)[None, :])

        self.modeled_uncert = cal_ferr
        self.target_uncert = cal_ferr[0]

def model_uncert ( self ) : Y = self . photometry_array . T Y /= np . median ( Y , axis = 1 ) [ : , None ] C = np . median ( Y , axis = 0 ) nstars , nobs = np . shape ( Y ) Z = np . empty ( ( nstars , 4 ) ) qs = self . qs . astype ( int ) for s in range ( 4 ) : Z [ : , s ] = np . median ( ( Y / C ) [ : , qs == s ] , axis = 1 ) resid2 = ( Y - Z [ : , qs ] * C ) ** 2 z = Z [ : , qs ] trend = z * C [ None , : ] lnS = np . log ( np . nanmedian ( resid2 , axis = 0 ) ) jitter = np . log ( 0.1 * np . nanmedian ( np . abs ( np . diff ( Y , axis = 1 ) ) ) ) cal_ferr = np . sqrt ( np . exp ( 2 * ( jitter / trend ) ) + z ** 2 * np . exp ( lnS ) [ None , : ] ) self . modeled_uncert = cal_ferr self . target_uncert = cal_ferr [ 0 ]

Estimate the photometric uncertainties on each data point following Equation A . 2 of The Paper . Based on the kepcal package of Dan Foreman - Mackey .


def _dump_field(self, fd):
        """Dump single field.
        """
        v = {}
        v['label'] = Pbd.LABELS[fd.label]
        v['type'] = fd.type_name if len(fd.type_name) > 0 else Pbd.TYPES[fd.type]
        v['name'] = fd.name
        v['number'] = fd.number
        v['default'] = '[default = {}]'.format(fd.default_value) if len(fd.default_value) > 0 else ''

        f = '{label} {type} {name} = {number} {default};'.format(**v)
        f = ' '.join(f.split())
        self._print(f)

        if len(fd.type_name) > 0:
            self.uses.append(fd.type_name)


def disassemble(self):
        """Disassemble serialized protocol buffers file.
        """
        ser_pb = open(self.input_file, 'rb').read()  # Read serialized pb file

        fd = FileDescriptorProto()
        fd.ParseFromString(ser_pb)
        self.name = fd.name

        self._print('// Reversed by pbd (https://github.com/rsc-dev/pbd)')
        self._print('syntax = "proto2";')
        self._print('')

        if len(fd.package) > 0:
            self._print('package {};'.format(fd.package))
            self.package = fd.package
        else:
            self._print('// Package not defined')

        self._walk(fd)

Disassemble serialized protocol buffers file .


def find_imports(self, pbds):
        """Find all missing imports in list of Pbd instances.
        """
        # List of types used, but not defined
        imports = list(set(self.uses).difference(set(self.defines)))

        # Clumpsy, but enought for now
        for imp in imports:
            for p in pbds:
                if imp in p.defines:
                    self.imports.append(p.name)
                    break

        self.imports = list(set(self.imports))

        for import_file in self.imports:
            self.lines.insert(2, 'import "{}";'.format(import_file))

def
find_imports
(
self
,
pbds
)
:
# List of types used, but not defined
imports
=
list
(
set
(
self
.
uses
)
.
difference
(
set
(
self
.
defines
)
)
)
# Clumpsy, but enought for now
for
imp
in
imports
:
for
p
in
pbds
:
if
imp
in
p
.
defines
:
self
.
imports
.
append
(
p
.
name
)
break
self
.
imports
=
list
(
set
(
self
.
imports
)
)
for
import_file
in
self
.
imports
:
self
.
lines
.
insert
(
2
,
'import "{}";'
.
format
(
import_file
)
)

Find all missing imports in list of Pbd instances .


def fasta_dict_to_file(fasta_dict, fasta_file, line_char_limit=None):
    """Write fasta_dict to fasta_file

    :param fasta_dict: returned by fasta_file_to_dict
    :param fasta_file: output file can be a string path or a file object
    :param line_char_limit: None = no limit (default)
    :return: None
    """
    fasta_fp = fasta_file
    if isinstance(fasta_file, str):
        fasta_fp = open(fasta_file, 'wb')

    for key in fasta_dict:
        seq = fasta_dict[key]['seq']
        if line_char_limit:
            seq = '\n'.join([seq[i:i+line_char_limit] for i in range(0, len(seq), line_char_limit)])
        fasta_fp.write(u'{0:s}\n{1:s}\n'.format(fasta_dict[key]['header'], seq))

def fasta_dict_to_file ( fasta_dict , fasta_file , line_char_limit = None ) : fasta_fp = fasta_file if isinstance ( fasta_file , str ) : fasta_fp = open ( fasta_file , 'wb' ) for key in fasta_dict : seq = fasta_dict [ key ] [ 'seq' ] if line_char_limit : seq = '\n' . join ( [ seq [ i : i + line_char_limit ] for i in range ( 0 , len ( seq ) , line_char_limit ) ] ) fasta_fp . write ( u'{0:s}\n{1:s}\n' . format ( fasta_dict [ key ] [ 'header' ] , seq ) )


def add_line_error(self, line_data, error_info, log_level=logging.ERROR):
        """Helper function to record and log an error message

        :param line_data: dict
        :param error_info: dict
        :param logger:
        :param log_level: int
        :return:
        """
        if not error_info: return
        try:
            line_data['line_errors'].append(error_info)
        except KeyError:
            line_data['line_errors'] = [error_info]
        except TypeError: # no line_data
            pass
        try:
            self.logger.log(log_level, Gff3.error_format.format(current_line_num=line_data['line_index'] + 1, error_type=error_info['error_type'], message=error_info['message'], line=line_data['line_raw'].rstrip()))
        except AttributeError: # no logger
            pass

def
add_line_error
(
self
,
line_data
,
error_info
,
log_level
=
logging
.
ERROR
)
:
if
not
error_info
:
return
try
:
line_data
[
'line_errors'
]
.
append
(
error_info
)
except
KeyError
:
line_data
[
'line_errors'
]
=
[
error_info
]
except
TypeError
:
# no line_data
pass
try
:
self
.
logger
.
log
(
log_level
,
Gff3
.
error_format
.
format
(
current_line_num
=
line_data
[
'line_index'
]
+
1
,
error_type
=
error_info
[
'error_type'
]
,
message
=
error_info
[
'message'
]
,
line
=
line_data
[
'line_raw'
]
.
rstrip
(
)
)
)
except
AttributeError
:
# no logger
pass


def check_parent_boundary(self):
        """
        checks whether child features are within the coordinate boundaries of parent features

        :return:
        """
        for line in self.lines:
            for parent_feature in line['parents']:
                ok = False
                for parent_line in parent_feature:
                    if parent_line['start'] <= line['start'] and line['end'] <= parent_line['end']:
                        ok = True
                        break
                if not ok:
                    self.add_line_error(line, {'message': 'This feature is not contained within the feature boundaries of parent: {0:s}: {1:s}'.format(
                        parent_feature[0]['attributes']['ID'],
                        ','.join(['({0:s}, {1:d}, {2:d})'.format(line['seqid'], line['start'], line['end']) for line in parent_feature])
                    ), 'error_type': 'BOUNDS', 'location': 'parent_boundary'})


def check_phase(self):
        """
        1. get a list of CDS with the same parent
        2. sort according to strand
        3. calculate and validate phase
        """
        plus_minus = set(['+', '-'])
        for k, g in groupby(sorted([line for line in self.lines if  line['line_type'] == 'feature' and line['type'] == 'CDS' and 'Parent' in line['attributes']], key=lambda x: x['attributes']['Parent']), key=lambda x: x['attributes']['Parent']):
            cds_list = list(g)
            strand_set = list(set([line['strand'] for line in cds_list]))
            if len(strand_set) != 1:
                for line in cds_list:
                    self.add_line_error(line, {'message': 'Inconsistent CDS strand with parent: {0:s}'.format(k), 'error_type': 'STRAND'})
                continue
            if len(cds_list) == 1:
                if cds_list[0]['phase'] != 0:
                    self.add_line_error(cds_list[0], {'message': 'Wrong phase {0:d}, should be {1:d}'.format(cds_list[0]['phase'], 0), 'error_type': 'PHASE'})
                continue
            strand = strand_set[0]
            if strand not in plus_minus:
                # don't process unknown strands
                continue
            if strand == '-':
                # sort end descending
                sorted_cds_list = sorted(cds_list, key=lambda x: x['end'], reverse=True)
            else:
                sorted_cds_list = sorted(cds_list, key=lambda x: x['start'])
            phase = 0
            for line in sorted_cds_list:
                if line['phase'] != phase:
                    self.add_line_error(line, {'message': 'Wrong phase {0:d}, should be {1:d}'.format(line['phase'], phase), 'error_type': 'PHASE'})
                phase = (3 - ((line['end'] - line['start'] + 1 - phase) % 3)) % 3

1 . get a list of CDS with the same parent 2 . sort according to strand 3 . calculate and validate phase


def adopt(self, old_parent, new_parent):
        """
        Transfer children from old_parent to new_parent

        :param old_parent: feature_id(str) or line_index(int) or line_data(dict) or feature
        :param new_parent: feature_id(str) or line_index(int) or line_data(dict)
        :return: List of children transferred
        """
        try: # assume line_data(dict)
            old_id = old_parent['attributes']['ID']
        except TypeError:
            try: # assume line_index(int)
                old_id = self.lines[old_parent]['attributes']['ID']
            except TypeError: # assume feature_id(str)
                old_id = old_parent
        old_feature = self.features[old_id]
        old_indexes = [ld['line_index'] for ld in old_feature]
        try: # assume line_data(dict)
            new_id = new_parent['attributes']['ID']
        except TypeError:
            try: # assume line_index(int)
                new_id = self.lines[new_parent]['attributes']['ID']
            except TypeError: # assume feature_id(str)
                new_id = new_parent
        new_feature = self.features[new_id]
        new_indexes = [ld['line_index'] for ld in new_feature]
        # build a list of children to be moved
        # add the child to the new parent's children list if its not already there
        # update the child's parent list and parent attribute
        # finally remove the old parent's children list
        children = old_feature[0]['children']
        new_parent_children_set = set([ld['line_index'] for ld in new_feature[0]['children']])
        for child in children:
            if child['line_index'] not in new_parent_children_set:
                new_parent_children_set.add(child['line_index'])
                for new_ld in new_feature:
                    new_ld['children'].append(child)
                child['parents'].append(new_feature)
                child['attributes']['Parent'].append(new_id)
            # remove multiple, list.remove() only removes 1
            child['parents'] = [f for f in child['parents'] if f[0]['attributes']['ID'] != old_id]
            child['attributes']['Parent'] = [d for d in child['attributes']['Parent'] if d != old_id]
        for old_ld in old_feature:
            old_ld['children'] = []
        return children


def remove(self, line_data, root_type=None):
        """
        Marks line_data and all of its associated feature's 'line_status' as 'removed', does not actually remove the line_data from the data structure.
        The write function checks the 'line_status' when writing the gff file.
        Find the root parent of line_data of type root_type, remove all of its descendants.
        If the root parent has a parent with no children after the remove, remove the root parent's parent recursively.

        :param line_data:
        :param root_type:
        :return:
        """
        roots = [ld for ld in self.ancestors(line_data) if (root_type and ld['line_type'] == root_type) or (not root_type and not ld['parents'])] or [line_data]
        for root in roots:
            root['line_status'] = 'removed'
            root_descendants = self.descendants(root)
            for root_descendant in root_descendants:
                root_descendant['line_status'] = 'removed'
            root_ancestors = self.ancestors(root) # BFS, so we will process closer ancestors first
            for root_ancestor in root_ancestors:
                if len([ld for ld in root_ancestor['children'] if ld['line_status'] != 'removed']) == 0: # if all children of a root_ancestor is removed
                    # remove this root_ancestor
                    root_ancestor['line_status'] = 'removed'

Marks line_data and all of its associated feature s line_status as removed does not actually remove the line_data from the data structure . The write function checks the line_status when writing the gff file . Find the root parent of line_data of type root_type remove all of its descendants . If the root parent has a parent with no children after the remove remove the root parent s parent recursively .


def abfIDfromFname(fname):
    """given a filename, return the ABFs ID string."""
    fname=os.path.abspath(fname)
    basename=os.path.basename(fname)
    return os.path.splitext(basename)[0]

def abfIDfromFname ( fname ) : fname = os . path . abspath ( fname ) basename = os . path . basename ( fname ) return os . path . splitext ( basename ) [ 0 ]

given a filename return the ABFs ID string .


def abfProtocol(fname):
    """Determine the protocol used to record an ABF file"""
    f=open(fname,'rb')
    raw=f.read(30*1000) #it should be in the first 30k of the file
    f.close()
    raw=raw.decode("utf-8","ignore")
    raw=raw.split("Clampex")[1].split(".pro")[0]
    protocol = os.path.basename(raw) # the whole protocol filename
    protocolID = protocol.split(" ")[0] # just the first number
    return protocolID


def headerHTML(header,fname):
        """given the bytestring ABF header, make and launch HTML."""
        html=""
        html+="
%s
"%(fname)
        html+=pprint.pformat(header, indent=1)
        html=html.replace("\n",'
').replace(" "," ")
        html=html.replace(r"\x00","")
        html+=""
        print("saving header file:",fname)
        f=open(fname,'w')
        f.write(html)
        f.close()
        webbrowser.open(fname)

def
headerHTML
(
header
,
fname
)
:
html
=
""
html
+=
"
%s
"
%
(
fname
)
html
+=
pprint
.
pformat
(
header
,
indent
=
1
)
html
=
html
.
replace
(
"\n"
,
'
'
)
.
replace
(
" "
,
" "
)
html
=
html
.
replace
(
r"\x00"
,
""
)
html
+=
""
print
(
"saving header file:"
,
fname
)
f
=
open
(
fname
,
'w'
)
f
.
write
(
html
)
f
.
close
(
)
webbrowser
.
open
(
fname
)

given the bytestring ABF header make and launch HTML .


def setsweeps(self):
        """iterate over every sweep"""
        for sweep in range(self.sweeps):
            self.setsweep(sweep)
            yield self.sweep

def setsweeps ( self ) : for sweep in range ( self . sweeps ) : self . setsweep ( sweep ) yield self . sweep


def comments_load(self):
        """read the header and populate self with information about comments"""
        self.comment_times,self.comment_sweeps,self.comment_tags=[],[],[]
        self.comments=0 # will be >0 if comments exist
        self.comment_text=""

        try:
            # this used to work
            self.comment_tags = list(self.ABFblock.segments[0].eventarrays[0].annotations['comments'])
            self.comment_times = list(self.ABFblock.segments[0].eventarrays[0].times/self.trace.itemsize)
            self.comment_sweeps = list(self.comment_times)
        except:
            # now this notation seems to work
            for events in self.ABFblock.segments[0].events: # this should only happen once actually
                self.comment_tags = events.annotations['comments'].tolist()
                self.comment_times = np.array(events.times.magnitude/self.trace.itemsize)
                self.comment_sweeps = self.comment_times/self.sweepInterval

        for i,c in enumerate(self.comment_tags):
            self.comment_tags[i]=c.decode("utf-8")

def
comments_load
(
self
)
:
self
.
comment_times
,
self
.
comment_sweeps
,
self
.
comment_tags
=
[
]
,
[
]
,
[
]
self
.
comments
=
0
# will be >0 if comments exist
self
.
comment_text
=
""
try
:
# this used to work
self
.
comment_tags
=
list
(
self
.
ABFblock
.
segments
[
0
]
.
eventarrays
[
0
]
.
annotations
[
'comments'
]
)
self
.
comment_times
=
list
(
self
.
ABFblock
.
segments
[
0
]
.
eventarrays
[
0
]
.
times
/
self
.
trace
.
itemsize
)
self
.
comment_sweeps
=
list
(
self
.
comment_times
)
except
:
# now this notation seems to work
for
events
in
self
.
ABFblock
.
segments
[
0
]
.
events
:
# this should only happen once actually
self
.
comment_tags
=
events
.
annotations
[
'comments'
]
.
tolist
(
)
self
.
comment_times
=
np
.
array
(
events
.
times
.
magnitude
/
self
.
trace
.
itemsize
)
self
.
comment_sweeps
=
self
.
comment_times
/
self
.
sweepInterval
for
i
,
c
in
enumerate
(
self
.
comment_tags
)
:
self
.
comment_tags
[
i
]
=
c
.
decode
(
"utf-8"
)


def get_protocol_sequence(self,sweep):
        """
        given a sweep, return the protocol as condensed sequence.
        This is better for comparing similarities and determining steps.
        There should be no duplicate numbers.
        """
        self.setsweep(sweep)
        return list(self.protoSeqX),list(self.protoSeqY)

def get_protocol_sequence ( self , sweep ) : self . setsweep ( sweep ) return list ( self . protoSeqX ) , list ( self . protoSeqY )

given a sweep return the protocol as condensed sequence . This is better for comparing similarities and determining steps . There should be no duplicate numbers .


def average(self,t1=0,t2=None,setsweep=False):
        """return the average of part of the current sweep."""
        if setsweep:
            self.setsweep(setsweep)
        if t2 is None or t2>self.sweepLength:
            t2=self.sweepLength
            self.log.debug("resetting t2 to [%f]",t2)
        t1=max(t1,0)
        if t1>t2:
            self.log.error("t1 cannot be larger than t2")
            return False
        I1,I2=int(t1*self.pointsPerSec),int(t2*self.pointsPerSec)
        if I1==I2:
            return np.nan
        return np.average(self.sweepY[I1:I2])

def
average
(
self
,
t1
=
0
,
t2
=
,
setsweep
=
False
)
:
if
setsweep
:
self
.
setsweep
(
setsweep
)
if
t2
is
or
t2
>
self
.
sweepLength
:
t2
=
self
.
sweepLength
self
.
log
.
debug
(
"resetting t2 to [%f]"
,
t2
)
t1
=
max
(
t1
,
0
)
if
t1
>
t2
:
self
.
log
.
error
(
"t1 cannot be larger than t2"
)
return
False
I1
,
I2
=
int
(
t1
*
self
.
pointsPerSec
)
,
int
(
t2
*
self
.
pointsPerSec
)
if
I1
==
I2
:
return
np
.
return
np
.
average
(
self
.
sweepY
[
I1
:
I2
]
)

return the average of part of the current sweep .


def averageSweep(self,sweepFirst=0,sweepLast=None):
        """
        Return a sweep which is the average of multiple sweeps.
        For now, standard deviation is lost.
        """
        if sweepLast is None:
            sweepLast=self.sweeps-1
        nSweeps=sweepLast-sweepFirst+1
        runningSum=np.zeros(len(self.sweepY))
        self.log.debug("averaging sweep %d to %d",sweepFirst,sweepLast)
        for sweep in np.arange(nSweeps)+sweepFirst:
            self.setsweep(sweep)
            runningSum+=self.sweepY.flatten()
        average=runningSum/nSweeps
        #TODO: standard deviation?
        return average

def
averageSweep
(
self
,
sweepFirst
=
0
,
sweepLast
=
)
:
if
sweepLast
is
:
sweepLast
=
self
.
sweeps
1
nSweeps
=
sweepLast
sweepFirst
+
1
runningSum
=
np
.
zeros
(
len
(
self
.
sweepY
)
)
self
.
log
.
debug
(
"averaging sweep %d to %d"
,
sweepFirst
,
sweepLast
)
for
sweep
in
np
.
arange
(
nSweeps
)
+
sweepFirst
:
self
.
setsweep
(
sweep
)
runningSum
+=
self
.
sweepY
.
flatten
(
)
average
=
runningSum
/
nSweeps
#TODO: standard deviation?
return
average

Return a sweep which is the average of multiple sweeps . For now standard deviation is lost .


def kernel_gaussian(self, sizeMS, sigmaMS=None, forwardOnly=False):
        """create kernel based on this ABF info."""
        sigmaMS=sizeMS/10 if sigmaMS is None else sigmaMS
        size,sigma=sizeMS*self.pointsPerMs,sigmaMS*self.pointsPerMs
        self.kernel=swhlab.common.kernel_gaussian(size,sigma,forwardOnly)
        return self.kernel

def kernel_gaussian ( self , sizeMS , sigmaMS = None , forwardOnly = False ) : sigmaMS = sizeMS / 10 if sigmaMS is None else sigmaMS size , sigma = sizeMS * self . pointsPerMs , sigmaMS * self . pointsPerMs self . kernel = swhlab . common . kernel_gaussian ( size , sigma , forwardOnly ) return self . kernel


def sweepYfiltered(self):
        """
        Get the filtered sweepY of the current sweep.
        Only works if self.kernel has been generated.
        """
        assert self.kernel is not None
        return swhlab.common.convolve(self.sweepY,self.kernel)

def sweepYfiltered ( self ) : assert self . kernel is not None return swhlab . common . convolve ( self . sweepY , self . kernel )

Get the filtered sweepY of the current sweep . Only works if self . kernel has been generated .


def dictFlat(l):
    """Given a list of list of dicts, return just the dicts."""
    if type(l) is dict:
        return [l]
    if "numpy" in str(type(l)):
        return l
    dicts=[]
    for item in l:
        if type(item)==dict:
            dicts.append(item)
        elif type(item)==list:
            for item2 in item:
                dicts.append(item2)
    return dicts

def dictFlat ( l ) : if type ( l ) is dict : return [ l ] if "numpy" in str ( type ( l ) ) : return l dicts = [ ] for item in l : if type ( item ) == dict : dicts . append ( item ) elif type ( item ) == list : for item2 in item : dicts . append ( item2 ) return dicts

Given a list of list of dicts return just the dicts .


def matrixValues(matrix,key):
    """given a key, return a list of values from the matrix with that key."""
    assert key in matrix.dtype.names
    col=matrix.dtype.names.index(key)
    values=np.empty(len(matrix))*np.nan
    for i in range(len(matrix)):
        values[i]=matrix[i][col]
    return values

def matrixValues ( matrix , key ) : assert key in matrix . dtype . names col = matrix . dtype . names . index ( key ) values = np . empty ( len ( matrix ) ) * np . nan for i in range ( len ( matrix ) ) : values [ i ] = matrix [ i ] [ col ] return values

given a key return a list of values from the matrix with that key .


def matrixToDicts(data):
    """given a recarray, return it as a list of dicts."""

    # 1D array
    if "float" in str(type(data[0])):
        d={}
        for x in range(len(data)):
            d[data.dtype.names[x]]=data[x]
        return d

    # 2D array
    l=[]
    for y in range(len(data)):
        d={}
        for x in range(len(data[y])):
            d[data.dtype.names[x]]=data[y][x]
        l.append(d)
    return l

def
matrixToDicts
(
data
)
:
# 1D array
if
"float"
in
str
(
type
(
data
[
0
]
)
)
:
d
=
{
}
for
x
in
range
(
len
(
data
)
)
:
d
[
data
.
dtype
.
names
[
x
]
]
=
data
[
x
]
return
d
# 2D array
l
=
[
]
for
y
in
range
(
len
(
data
)
)
:
d
=
{
}
for
x
in
range
(
len
(
data
[
y
]
)
)
:
d
[
data
.
dtype
.
names
[
x
]
]
=
data
[
y
]
[
x
]
l
.
append
(
d
)
return
l

given a recarray return it as a list of dicts .


def html_temp_launch(html):
    """given text, make it a temporary HTML file and launch it."""
    fname = tempfile.gettempdir()+"/swhlab/temp.html"
    with open(fname,'w') as f:
        f.write(html)
    webbrowser.open(fname)

def html_temp_launch ( html ) : fname = tempfile . gettempdir ( ) + "/swhlab/temp.html" with open ( fname , 'w' ) as f : f . write ( html ) webbrowser . open ( fname )

given text make it a temporary HTML file and launch it .


def checkOut(thing,html=True):
    """show everything we can about an object's projects and methods."""
    msg=""
    for name in sorted(dir(thing)):
        if not "__" in name:
            msg+="%s\n"%name
            try:
                msg+=" ^-VALUE: %s\n"%getattr(thing,name)()
            except:
                pass
    if html:
        html=''+msg+''
        html=html.replace(" "," ").replace("\n","
")
        fname = tempfile.gettempdir()+"/swhlab/checkout.html"
        with open(fname,'w') as f:
            f.write(html)
        webbrowser.open(fname)
    print(msg.replace('','').replace('',''))

def
checkOut
(
thing
,
html
=
True
)
:
msg
=
""
for
name
in
sorted
(
dir
(
thing
)
)
:
if
not
"__"
in
name
:
msg
+=
"%s\n"
%
name
try
:
msg
+=
" ^-VALUE: %s\n"
%
getattr
(
thing
,
name
)
(
)
except
:
pass
if
html
:
html
=
''
+
msg
+
''
html
=
html
.
replace
(
" "
,
" "
)
.
replace
(
"\n"
,
"
"
)
fname
=
tempfile
.
gettempdir
(
)
+
"/swhlab/checkout.html"
with
open
(
fname
,
'w'
)
as
f
:
f
.
write
(
html
)
webbrowser
.
open
(
fname
)
print
(
msg
.
replace
(
''
,
''
)
.
replace
(
''
,
''
)
)

show everything we can about an object s projects and methods .


def matrixToHTML(data,names=None,units=None,bookName=None,sheetName=None,xCol=None):
    """Put 2d numpy data into a temporary HTML file."""
    if not names:
        names=[""]*len(data[0])
        if data.dtype.names:
            names=list(data.dtype.names)
    if not units:
        units=[""]*len(data[0])
        for i in range(len(units)):
            if names[i] in UNITS.keys():
                units[i]=UNITS[names[i]]
    if 'recarray' in str(type(data)): #make it a regular array
        data=data.view(float).reshape(data.shape + (-1,))
    if xCol and xCol in names:
        xCol=names.index(xCol)
        names.insert(0,names[xCol])
        units.insert(0,units[xCol])
        data=np.insert(data,0,data[:,xCol],1)

    htmlFname = tempfile.gettempdir()+"/swhlab/WKS-%s.%s.html"%(bookName,sheetName)
    html="""

    """
    html+="
FauxRigin
"
    if bookName or sheetName:
        html+='%s / %s

'%(bookName,sheetName)
    html+="
"
    #cols=list(range(len(names)))
    colNames=['']
    for i in range(len(units)):
        label="%s (%d)"%(chr(i+ord('A')),i)
        colNames.append(label)
    html+=htmlListToTR(colNames,'labelCol','labelCol')
    html+=htmlListToTR(['Long Name']+list(names),'name',td1Class='labelRow')
    html+=htmlListToTR(['Units']+list(units),'units',td1Class='labelRow')
    cutOff=False
    for y in range(len(data)):
        html+=htmlListToTR([y+1]+list(data[y]),trClass='data%d'%(y%2),td1Class='labelRow')
        if y>=200:
            cutOff=True
            break
    html+="
"
    html=html.replace(">nan<",">--<")
    html=html.replace(">None<","><")
    if cutOff:
        html+="
... showing only %d of %d rows ...
"%(y,len(data))
    html+=""
    with open(htmlFname,'w') as f:
        f.write(html)
    webbrowser.open(htmlFname)
    return

"""<body>
    <style>
    body {
          background-color: #ababab;
          padding:20px;
          }
    table {
           font-size:12px;
           border-spacing: 0;
           border-collapse: collapse;
           //border:2px solid #000000;
           }
    .name {background-color:#fafac8;text-align:center;}
    .units {background-color:#fafac8;text-align:center;}
    .data0 {background-color:#FFFFFF;font-family: monospace;text-align:center;}
    .data1 {background-color:#FAFAFA;font-family: monospace;text-align:center;}
    .labelRow {background-color:#e0dfe4; text-align:right;border:1px solid #000000;}
    .labelCol {background-color:#e0dfe4; text-align:center;border:1px solid #000000;}
    td {
        border:1px solid #c0c0c0; padding:5px;
        //font-family: Verdana, Geneva, sans-serif;
        font-family: Arial, Helvetica, sans-serif
        }
    </style>
    <html>"""

"<h3>... showing only %d of %d rows ...</h3>"

Put 2d numpy data into a temporary HTML file .


def XMLtoPython(xmlStr=r"C:\Apps\pythonModules\GSTemp.xml"):
    """
    given a string or a path to an XML file, return an XML object.
    """
    #TODO: this absolute file path crazy stuff needs to stop!
    if os.path.exists(xmlStr):
        with open(xmlStr) as f:
            xmlStr=f.read()
    print(xmlStr)
    print("DONE")
    return

given a string or a path to an XML file return an XML object .


def algo_exp(x, m, t, b):
    """mono-exponential curve."""
    return m*np.exp(-t*x)+b

def algo_exp ( x , m , t , b ) : return m * np . exp ( - t * x ) + b


def where_cross(data,threshold):
    """return a list of Is where the data first crosses above threshold."""
    Is=np.where(data>threshold)[0]
    Is=np.concatenate(([0],Is))
    Ds=Is[:-1]-Is[1:]+1
    return Is[np.where(Ds)[0]+1]

def where_cross ( data , threshold ) : Is = np . where ( data > threshold ) [ 0 ] Is = np . concatenate ( ( [ 0 ] , Is ) ) Ds = Is [ : - 1 ] - Is [ 1 : ] + 1 return Is [ np . where ( Ds ) [ 0 ] + 1 ]

return a list of Is where the data first crosses above threshold .


def originFormat(thing):
    """Try to format anything as a 2D matrix with column names."""
    if type(thing) is list and type(thing[0]) is dict:
        return originFormat_listOfDicts(thing)
    if type(thing) is list and type(thing[0]) is list:
        return originFormat_listOfDicts(dictFlat(thing))
    else:
        print(" !! I don't know how to format this object!")
        print(thing)

Try to format anything as a 2D matrix with column names .


def pickle_save(thing,fname):
    """save something to a pickle file"""
    pickle.dump(thing, open(fname,"wb"),pickle.HIGHEST_PROTOCOL)
    return thing

def pickle_save ( thing , fname ) : pickle . dump ( thing , open ( fname , "wb" ) , pickle . HIGHEST_PROTOCOL ) return thing


def msgDict(d,matching=None,sep1="=",sep2="\n",sort=True,cantEndWith=None):
    """convert a dictionary to a pretty formatted string."""
    msg=""
    if "record" in str(type(d)):
        keys=d.dtype.names
    else:
        keys=d.keys()
    if sort:
        keys=sorted(keys)
    for key in keys:
        if key[0]=="_":
            continue
        if matching:
            if not key in matching:
                continue
        if cantEndWith and key[-len(cantEndWith)]==cantEndWith:
            continue
        if 'float' in str(type(d[key])):
            s="%.02f"%d[key]
        else:
            s=str(d[key])
        if "object" in s:
            s='<object>'
        msg+=key+sep1+s+sep2
    return msg.strip()

def msgDict ( d , matching = None , sep1 = "=" , sep2 = "\n" , sort = True , cantEndWith = None ) : msg = "" if "record" in str ( type ( d ) ) : keys = d . dtype . names else : keys = d . keys ( ) if sort : keys = sorted ( keys ) for key in keys : if key [ 0 ] == "_" : continue if matching : if not key in matching : continue if cantEndWith and key [ - len ( cantEndWith ) ] == cantEndWith : continue if 'float' in str ( type ( d [ key ] ) ) : s = "%.02f" % d [ key ] else : s = str ( d [ key ] ) if "object" in s : s = '<object>' msg += key + sep1 + s + sep2 return msg . strip ( )

convert a dictionary to a pretty formatted string .


def determineProtocol(fname):
    """determine the comment cooked in the protocol."""
    f=open(fname,'rb')
    raw=f.read(5000) #it should be in the first 5k of the file
    f.close()
    protoComment="unknown"
    if b"SWHLab4[" in raw:
        protoComment=raw.split(b"SWHLab4[")[1].split(b"]",1)[0]
    elif b"SWH[" in raw:
        protoComment=raw.split(b"SWH[")[1].split(b"]",1)[0]
    else:
        protoComment="?"
    if not type(protoComment) is str:
        protoComment=protoComment.decode("utf-8")
    return protoComment

determine the comment cooked in the protocol .


def scanABFfolder(abfFolder):
    """
    scan an ABF directory and subdirectory. Try to do this just once.
    Returns ABF files, SWHLab files, and groups.
    """
    assert os.path.isdir(abfFolder)
    filesABF=forwardSlash(sorted(glob.glob(abfFolder+"/*.*")))
    filesSWH=[]
    if os.path.exists(abfFolder+"/swhlab4/"):
        filesSWH=forwardSlash(sorted(glob.glob(abfFolder+"/swhlab4/*.*")))
    groups=getABFgroups(filesABF)
    return filesABF,filesSWH,groups

def scanABFfolder ( abfFolder ) : assert os . path . isdir ( abfFolder ) filesABF = forwardSlash ( sorted ( glob . glob ( abfFolder + "/*.*" ) ) ) filesSWH = [ ] if os . path . exists ( abfFolder + "/swhlab4/" ) : filesSWH = forwardSlash ( sorted ( glob . glob ( abfFolder + "/swhlab4/*.*" ) ) ) groups = getABFgroups ( filesABF ) return filesABF , filesSWH , groups

scan an ABF directory and subdirectory . Try to do this just once . Returns ABF files SWHLab files and groups .


def getParent(abfFname):
    """given an ABF file name, return the ABF of its parent."""
    child=os.path.abspath(abfFname)
    files=sorted(glob.glob(os.path.dirname(child)+"/*.*"))
    parentID=abfFname #its own parent
    for fname in files:
        if fname.endswith(".abf") and fname.replace(".abf",".TIF") in files:
            parentID=os.path.basename(fname).replace(".abf","")
        if os.path.basename(child) in fname:
            break
    return parentID

def
getParent
(
abfFname
)
:
child
=
os
.
path
.
abspath
(
abfFname
)
files
=
sorted
(
glob
.
glob
(
os
.
path
.
dirname
(
child
)
+
"/*.*"
)
)
parentID
=
abfFname
#its own parent
for
fname
in
files
:
if
fname
.
endswith
(
".abf"
)
and
fname
.
replace
(
".abf"
,
".TIF"
)
in
files
:
parentID
=
os
.
path
.
basename
(
fname
)
.
replace
(
".abf"
,
""
)
if
os
.
path
.
basename
(
child
)
in
fname
:
break
return
parentID

given an ABF file name return the ABF of its parent .


def getParent2(abfFname,groups):
    """given an ABF and the groups dict, return the ID of its parent."""
    if ".abf" in abfFname:
        abfFname=os.path.basename(abfFname).replace(".abf","")
    for parentID in groups.keys():
        if abfFname in groups[parentID]:
            return parentID
    return abfFname

def getParent2 ( abfFname , groups ) : if ".abf" in abfFname : abfFname = os . path . basename ( abfFname ) . replace ( ".abf" , "" ) for parentID in groups . keys ( ) : if abfFname in groups [ parentID ] : return parentID return abfFname

given an ABF and the groups dict return the ID of its parent .


def getNotesForABF(abfFile):
    """given an ABF, find the parent, return that line of experiments.txt"""
    parent=getParent(abfFile)
    parent=os.path.basename(parent).replace(".abf","")
    expFile=os.path.dirname(abfFile)+"/experiment.txt"
    if not os.path.exists(expFile):
        return "no experiment file"
    with open(expFile) as f:
        raw=f.readlines()
    for line in raw:
        if line[0]=='~':
            line=line[1:].strip()
            if line.startswith(parent):
                while "\t\t" in line:
                    line=line.replace("\t\t","\t")
                line=line.replace("\t","\n")
                return line
    return "experiment.txt found, but didn't contain %s"%parent

given an ABF find the parent return that line of experiments . txt


def getIDsFromFiles(files):
    """given a path or list of files, return ABF IDs."""
    if type(files) is str:
        files=glob.glob(files+"/*.*")
    IDs=[]
    for fname in files:
        if fname[-4:].lower()=='.abf':
            ext=fname.split('.')[-1]
            IDs.append(os.path.basename(fname).replace('.'+ext,''))
    return sorted(IDs)

def getIDsFromFiles ( files ) : if type ( files ) is str : files = glob . glob ( files + "/*.*" ) IDs = [ ] for fname in files : if fname [ - 4 : ] . lower ( ) == '.abf' : ext = fname . split ( '.' ) [ - 1 ] IDs . append ( os . path . basename ( fname ) . replace ( '.' + ext , '' ) ) return sorted ( IDs )

given a path or list of files return ABF IDs .