def get_ipaddress(self, name, retries=3, delay=3):
    '''get the ip_address of an inserted instance. Will try three times with
       delay to give the instance time to start up.

       Parameters
       ==========
       name: the name of the instance to get the ip address for.
       retries: the number of retries before giving up
       delay: the delay between retry

       Note from @vsoch: this function is pretty nasty.

    '''
    for rr in range(retries):

        # Retrieve list of instances
        instances = self._get_instances()

        for instance in instances['items']:
            if instance['name'] == name:

                # Iterate through network interfaces
                for network in instance['networkInterfaces']:
                    if network['name'] == 'nic0':

                        # Access configurations
                        for subnet in network['accessConfigs']:
                            if subnet['name'] == 'External NAT':
                                if 'natIP' in subnet:
                                    return subnet['natIP']

        sleep(delay)
    bot.warning('Did not find IP address, check Cloud Console!')

get the ip_address of an inserted instance . Will try three times with delay to give the instance time to start up .


def run_build(self, config):
    '''run a build, meaning inserting an instance. Retry if there is failure

       Parameters
       ==========
       config: the configuration dictionary generated by setup_build

    '''
    project = self._get_project()
    zone = self._get_zone()

    bot.custom(prefix='INSTANCE', message=config['name'], color="CYAN")
    bot.info(config['description'])

    response = self._compute_service.instances().insert(project=project,
                                                        zone=zone,
                                                        body=config).execute()

    # Direct the user to the web portal with log
    ipaddress = self._get_ipaddress(config['name'])
    bot.info('Robot Logger: http://%s' %ipaddress)
    bot.info('Allow a few minutes for web server install, beepboop!')
    return response

run a build meaning inserting an instance . Retry if there is failure


def list_containers(self):
    '''return a list of containers, determined by finding the metadata field
       "type" with value "container." We alert the user to no containers
       if results is empty, and exit

       {'metadata': {'items':
                              [
                               {'key': 'type', 'value': 'container'}, ...
                              ]
                    }
       }

    '''
    results = []
    for image in self._bucket.list_blobs():
        if image.metadata is not None:
            if "type" in image.metadata:
                if image.metadata['type'] == "container":
                    results.append(image)

    if len(results) == 0:
        bot.info("No containers found, based on metadata type:container")

    return results

return a list of containers determined by finding the metadata field type with value container . We alert the user to no containers if results is empty and exit


def search_all(self):
    '''a "list all" search that doesn't require a query. Here we return to
       the user all objects that have custom metadata value of "container"

       IMPORTANT: the upload function adds this metadata. For a container to
       be found by the client, it must have the type as container in metadata.
    '''

    results = self._list_containers()

    bot.info("[gs://%s] Containers" %self._bucket_name)

    rows = []
    for i in results:
        size = round(i.size / (1024*1024.0))
        size = ("%s MB" %size).rjust(10)
        rows.append([size, i.metadata['name']])

    bot.table(rows)
    return rows

def
search_all
(
self
)
:
results
=
self
.
_list_containers
(
)
bot
.
info
(
"[gs://%s] Containers"
%
self
.
_bucket_name
)
rows
=
[
]
for
i
in
results
:
size
=
round
(
i
.
size
/
(
1024
*
1024.0
)
)
size
=
(
"%s MB"
%
size
)
.
rjust
(
10
)
rows
.
append
(
[
size
,
i
.
metadata
[
'name'
]
]
)
bot
.
table
(
rows
)
return
rows

a list all search that doesn t require a query . Here we return to the user all objects that have custom metadata value of container


def main(args,parser,subparser):
    '''the list command corresponds with listing images for an external
       resource. This is different from listing images that are local to the
       database, which should be done with "images"
    '''
    from sregistry.main import get_client
    cli = get_client(quiet=args.quiet)

    for query in args.query:
        if query in ['','*']:
            query = None

        cli.ls(query=query)

def main ( args , parser , subparser ) : from sregistry . main import get_client cli = get_client ( quiet = args . quiet ) for query in args . query : if query in [ '' , '*' ] : query = None cli . ls ( query = query )

the list command corresponds with listing images for an external resource . This is different from listing images that are local to the database which should be done with images


def main(args, parser, subparser):
    '''sharing an image means sending a remote share from an image you
       control to a contact, usually an email.
    '''
    from sregistry.main import get_client
    images = args.image

    if not isinstance(images,list):
        images = [images]

    for image in images:
        print(image)

        # Detect any uri, and refresh client if necessary
        cli = get_client(image, quiet=args.quiet)
        cli.announce(args.command)
        cli.share(image, share_to=args.share_to)

sharing an image means sending a remote share from an image you control to a contact usually an email .


def init_db(self, db_path):
    '''initialize the database, with the default database path or custom of

       the format sqlite:////scif/data/expfactory.db

    The custom path can be set with the environment variable SREGISTRY_DATABASE
    when a user creates the client, we must initialize this db
    the database should use the .singularity cache folder to cache
    layers and images, and .singularity/sregistry.db as a database
    '''

    # Database Setup, use default if uri not provided
    self.database = 'sqlite:///%s' % db_path
    self.storage = SREGISTRY_STORAGE

    bot.debug("Database located at %s" % self.database)
    self.engine = create_engine(self.database, convert_unicode=True)
    self.session = scoped_session(sessionmaker(autocommit=False,
                                               autoflush=False,
                                               bind=self.engine))

    Base.query = self.session.query_property()

    # import all modules here that might define models so that
    # they will be registered properly on the metadata.  Otherwise
    # you will have to import them first before calling init_db()
    Base.metadata.create_all(bind=self.engine)
    self.Base = Base

initialize the database with the default database path or custom of


def get_build_template():
    '''get default build template.
    '''
    base = get_installdir()
    name = "%s/main/templates/build/singularity-cloudbuild.json" % base

    if os.path.exists(name):
        bot.debug("Found template %s" %name)
        return read_json(name)

    bot.warning("Template %s not found." % name)


def search(self, query=None, args=None):
    '''query will show images determined by the extension of img
       or simg.

       Parameters
       ==========
       query: the container name (path) or uri to search for
       args.endpoint: can be an endpoint id and optional path, e.g.:

             --endpoint  6881ae2e-db26-11e5-9772-22000b9da45e:.singularity'
             --endpoint  6881ae2e-db26-11e5-9772-22000b9da45e'

       if not defined, we show the user endpoints to choose from

       Usage
       =====
       If endpoint is defined with a query, then we search the given endpoint
       for a container of interested (designated by ending in .img or .simg

       If no endpoint is provided but instead just a query, we use the query
       to search endpoints.

    '''

    # No query is defined
    if query is None:

        # Option 1: No query or endpoints lists all shared and personal
        if args.endpoint is None:
            bot.info('Listing shared endpoints. Add query to expand search.')
            return self._list_endpoints()

        # Option 2: An endpoint without query will just list containers there
        else:
            return self._list_endpoint(args.endpoint)

    # Option 3: A query without an endpoint will search endpoints for it
    if args.endpoint is None:
        bot.info('You must specify an endpoint id to query!')
        return self._list_endpoints(query)

    # Option 4: A query with an endpoint will search the endpoint for pattern
    return self._list_endpoint(endpoint=args.endpoint,
                               query=query)

query will show images determined by the extension of img or simg .


def list_endpoints(self, query=None):
    '''list all endpoints, providing a list of endpoints to the user to
       better filter the search. This function takes no arguments,
       as the user has not provided an endpoint id or query.
    '''
    bot.info('Please select an endpoint id to query from')

    endpoints = self._get_endpoints(query)

    # Iterate through endpoints to provide user a list

    bot.custom(prefix="Globus", message="Endpoints", color="CYAN")
    rows = []
    for kind,eps in endpoints.items():
        for epid,epmeta in eps.items():
            rows.append([epid, '[%s]' %kind, epmeta['name']])

    bot.table(rows)
    return rows

list all endpoints providing a list of endpoints to the user to better filter the search . This function takes no arguments as the user has not provided an endpoint id or query .


def list_endpoint(self, endpoint, query=None):
    '''An endpoint is required here to list files within. Optionally, we can
       take a path relative to the endpoint root.

       Parameters
       ==========
       endpoint: a single endpoint ID or an endpoint id and relative path.
                 If no path is provided, we use '', which defaults to scratch.

       query: if defined, limit files to those that have query match

    '''
    if not hasattr(self, 'transfer_client'):
        self._init_transfer_client()

    # Separate endpoint id from the desired path

    endpoint, path = self._parse_endpoint_name(endpoint)

    # Get a list of files at endpoint, under specific path
    try:
        result = self.transfer_client.operation_ls(endpoint, path=path)
    except TransferAPIError as err:

        # Tell the user what went wrong!
        bot.custom(prefix='ERROR', message=err, color='RED')
        sys.exit(1)

    rows = []

    for filey in result:

        # Highlight container contenders with purple
        name = filey['name']
        if query is None or query in name:
            if name.endswith('img'):
                name = bot.addColor('PURPLE',name)

            rows.append([filey['type'],
                         filey['permissions'],
                         str(filey['size']),
                         name ])

    if len(rows) > 0:
        rows = [["type","[perm]","[size]","[name]"]] + rows
        bot.custom(prefix="Endpoint Listing %s" %path, message='', color="CYAN")
        bot.table(rows)
    else:
        bot.info('No content was found at the selected endpoint.')
    return rows

An endpoint is required here to list files within . Optionally we can take a path relative to the endpoint root .


def share(self, query, share_to=None):
    '''share will use the client to get a shareable link for an image of choice.
       the functions returns a url of choice to send to a recipient.
    '''

    names = parse_image_name(remove_uri(query))

    # Dropbox path is the path in storage with a slash
    dropbox_path = '/%s' % names['storage']

    # First ensure that exists
    if self.exists(dropbox_path) is True:

        # Create new shared link
        try:
            share = self.dbx.sharing_create_shared_link_with_settings(dropbox_path)

        # Already exists!
        except ApiError as err:
            share = self.dbx.sharing_create_shared_link(dropbox_path)

        bot.info(share.url)
    return share.url

share will use the client to get a shareable link for an image of choice . the functions returns a url of choice to send to a recipient .


def read_client_secrets():
    '''for private or protected registries, a client secrets file is required
       to be located at .sregistry. If no secrets are found, we use default
       of Singularity Hub, and return a dummy secrets.
    '''
    client_secrets = _default_client_secrets()

    # If token file not provided, check environment
    secrets = get_secrets_file()

    # If exists, load
    if secrets is not None:
        client_secrets = read_json(secrets)

    # Otherwise, initialize
    else:
        from sregistry.defaults import SREGISTRY_CLIENT_SECRETS
        write_json(client_secrets, SREGISTRY_CLIENT_SECRETS)

    return client_secrets

for private or protected registries a client secrets file is required to be located at . sregistry . If no secrets are found we use default of Singularity Hub and return a dummy secrets .


def _get_services(self, version='v1'):
        '''get version 1 of the google compute and storage service

        Parameters
        ==========
        version: version to use (default is v1)
        '''
        self._bucket_service = storage.Client()
        creds = GoogleCredentials.get_application_default()
        self._storage_service = discovery_build('storage', version, credentials=creds)
        self._compute_service = discovery_build('compute', version, credentials=creds)

def _get_services ( self , version = 'v1' ) : self . _bucket_service = storage . Client ( ) creds = GoogleCredentials . get_application_default ( ) self . _storage_service = discovery_build ( 'storage' , version , credentials = creds ) self . _compute_service = discovery_build ( 'compute' , version , credentials = creds )


def delete_object(service, bucket_name, object_name):
    '''delete object will delete a file from a bucket

       Parameters
       ==========
       storage_service: the service obtained with get_storage_service
       bucket_name: the name of the bucket
       object_name: the "name" parameter of the object.

    '''
    try:
        operation = service.objects().delete(bucket=bucket_name,
                                             object=object_name).execute()
    except HttpError as e:
        pass
        operation = e
    return operation

def delete_object ( service , bucket_name , object_name ) : try : operation = service . objects ( ) . delete ( bucket = bucket_name , object = object_name ) . execute ( ) except HttpError as e : pass operation = e return operation


def delete(self, name):
    '''delete an image from Google Storage.

       Parameters
       ==========
       name: the name of the file (or image) to delete

    '''

    bot.debug("DELETE %s" % name)

    for file_object in files:
        if isinstance(file_object, dict):
            if "kind" in file_object:
                if file_object['kind'] == "storage#object":
                    object_name = "/".join(file_object['id'].split('/')[:-1])
                    object_name = re.sub('%s/' %self._bucket['name'],'', object_name,1)

                    delete_object(service=self._bucket_service,
                                  bucket_name=bucket['name'],
                                  object_name=object_name)

def
delete
(
self
,
name
)
:
bot
.
debug
(
"DELETE %s"
%
name
)
for
file_object
in
files
:
if
isinstance
(
file_object
,
dict
)
:
if
"kind"
in
file_object
:
if
file_object
[
'kind'
]
==
"storage#object"
:
object_name
=
"/"
.
join
(
file_object
[
'id'
]
.
split
(
'/'
)
[
:
1
]
)
object_name
=
re
.
sub
(
'%s/'
%
self
.
_bucket
[
'name'
]
,
''
,
object_name
,
1
)
delete_object
(
service
=
self
.
_bucket_service
,
bucket_name
=
bucket
[
'name'
]
,
object_name
=
object_name
)


def destroy(self, name):
    '''destroy an instance, meaning take down the instance and stop the build.

       Parameters
       ==========
       name: the name of the instance to stop building.
    '''

    instances = self._get_instances()
    project = self._get_project()
    zone = self._get_zone()
    found = False

    if 'items' in instances:
        for instance in instances['items']:
            if instance['name'] == name:
                found = True
                break

    if found:
        bot.info('Killing instance %s' %name)
        return self._compute_service.instances().delete(project=project,
                                                        zone=zone,
                                                        instance=name).execute()

def
destroy
(
self
,
name
)
:
instances
=
self
.
_get_instances
(
)
project
=
self
.
_get_project
(
)
zone
=
self
.
_get_zone
(
)
found
=
False
if
'items'
in
instances
:
for
instance
in
instances
[
'items'
]
:
if
instance
[
'name'
]
==
name
:
found
=
True
break
if
found
:
bot
.
info
(
'Killing instance %s'
%
name
)
return
self
.
_compute_service
.
instances
(
)
.
delete
(
project
=
project
,
zone
=
zone
,
instance
=
name
)
.
execute
(
)

destroy an instance meaning take down the instance and stop the build .


def get_subparsers(parser):
    '''get_subparser will get a dictionary of subparsers, to help with printing help
    '''

    actions = [action for action in parser._actions
               if isinstance(action, argparse._SubParsersAction)]

    subparsers = dict()
    for action in actions:
        # get all subparsers and print help
        for choice, subparser in action.choices.items():
            subparsers[choice] = subparser

    return subparsers

def
get_subparsers
(
parser
)
:
actions
=
[
action
for
action
in
parser
.
_actions
if
isinstance
(
action
,
argparse
.
_SubParsersAction
)
]
subparsers
=
dict
(
)
for
action
in
actions
:
# get all subparsers and print help
for
choice
,
subparser
in
action
.
choices
.
items
(
)
:
subparsers
[
choice
]
=
subparser
return
subparsers

get_subparser will get a dictionary of subparsers to help with printing help


def generate(self, delim='-', length=4, chars='0123456789'):
        '''
        Generate a robot name. Inspiration from Haikunator, but much more
                 poorly implemented ;)

        Parameters
        ==========
        delim: Delimiter
        length: TokenLength
        chars: TokenChars
        '''

        descriptor = self._select(self._descriptors)
        noun = self._select(self._nouns)
        numbers = ''.join((self._select(chars) for _ in range(length)))
        return delim.join([descriptor, noun, numbers])

def generate ( self , delim = '-' , length = 4 , chars = '0123456789' ) : descriptor = self . _select ( self . _descriptors ) noun = self . _select ( self . _nouns ) numbers = '' . join ( ( self . _select ( chars ) for _ in range ( length ) ) ) return delim . join ( [ descriptor , noun , numbers ] )

Generate a robot name . Inspiration from Haikunator but much more poorly implemented ; )


def get_tmpdir(requested_tmpdir=None, prefix="", create=True):
    '''get a temporary directory for an operation. If SREGISTRY_TMPDIR
       is set, return that. Otherwise, return the output of tempfile.mkdtemp

       Parameters
       ==========
       requested_tmpdir: an optional requested temporary directory, first
       priority as is coming from calling function.
       prefix: Given a need for a sandbox (or similar), we will need to
       create a subfolder *within* the SREGISTRY_TMPDIR.
       create: boolean to determine if we should create folder (True)
    '''
    from sregistry.defaults import SREGISTRY_TMPDIR

    # First priority for the base goes to the user requested.
    tmpdir = requested_tmpdir or SREGISTRY_TMPDIR

    prefix = prefix or "sregistry-tmp"
    prefix = "%s.%s" %(prefix, next(tempfile._get_candidate_names()))
    tmpdir = os.path.join(tmpdir, prefix)

    if not os.path.exists(tmpdir) and create is True:
        os.mkdir(tmpdir)

    return tmpdir

get a temporary directory for an operation . If SREGISTRY_TMPDIR is set return that . Otherwise return the output of tempfile . mkdtemp


def extract_tar(archive, output_folder, handle_whiteout=False):
    '''extract a tar archive to a specified output folder

        Parameters
        ==========
        archive: the archive file to extract
        output_folder: the output folder to extract to
        handle_whiteout: use docker2oci variation to handle whiteout files

    '''
    from .terminal import run_command

    # Do we want to remove whiteout files?
    if handle_whiteout is True:
        return _extract_tar(archive, output_folder)

    # If extension is .tar.gz, use -xzf
    args = '-xf'
    if archive.endswith(".tar.gz"):
        args = '-xzf'

    # Just use command line, more succinct.
    command = ["tar", args, archive, "-C", output_folder, "--exclude=dev/*"]
    if not bot.is_quiet():
        print("Extracting %s" % archive)

    return run_command(command)

def
extract_tar
(
archive
,
output_folder
,
handle_whiteout
=
False
)
:
from
.
terminal
import
run_command
# Do we want to remove whiteout files?
if
handle_whiteout
is
True
:
return
_extract_tar
(
archive
,
output_folder
)
# If extension is .tar.gz, use -xzf
args
=
'-xf'
if
archive
.
endswith
(
".tar.gz"
)
:
args
=
'-xzf'
# Just use command line, more succinct.
command
=
[
"tar"
,
args
,
archive
,
"-C"
,
output_folder
,
"--exclude=dev/*"
]
if
not
bot
.
is_quiet
(
)
:
print
(
"Extracting %s"
%
archive
)
return
run_command
(
command
)


def _extract_tar(archive, output_folder):
    '''use blob2oci to handle whiteout files for extraction. Credit for this
       script goes to docker2oci by Olivier Freyermouth, and see script
       folder for license.

       Parameters
       ==========
       archive: the archive to extract
       output_folder the output folder (sandbox) to extract to

    '''
    from .terminal import ( run_command, which )

    result = which('blob2oci')
    if result['return_code'] != 0:
        bot.error('Cannot find blob2oci script on path, exiting.')
        sys.exit(1)

    script = result['message']
    command = ['exec' ,script, '--layer', archive, '--extract', output_folder]

    if not bot.is_quiet():
        print("Extracting %s" % archive)

    return run_command(command)

use blob2oci to handle whiteout files for extraction . Credit for this script goes to docker2oci by Olivier Freyermouth and see script folder for license .


def get_file_hash(filename):
    '''find the SHA256 hash string of a file
    '''
    hasher = hashlib.sha256()
    with open(filename, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hasher.update(chunk)
    return hasher.hexdigest()

def get_file_hash ( filename ) : hasher = hashlib . sha256 ( ) with open ( filename , "rb" ) as f : for chunk in iter ( lambda : f . read ( 4096 ) , b"" ) : hasher . update ( chunk ) return hasher . hexdigest ( )


def read_file(filename, mode="r", readlines=True):
    '''write_file will open a file, "filename" and write content, "content"
       and properly close the file
    '''
    with open(filename, mode) as filey:
        if readlines is True:
            content = filey.readlines()
        else:
            content = filey.read()
    return content

def read_file ( filename , mode = "r" , readlines = True ) : with open ( filename , mode ) as filey : if readlines is True : content = filey . readlines ( ) else : content = filey . read ( ) return content

write_file will open a file filename and write content content and properly close the file


def read_json(filename, mode='r'):
    '''read_json reads in a json file and returns
       the data structure as dict.
    '''
    with open(filename, mode) as filey:
        data = json.load(filey)
    return data

def read_json ( filename , mode = 'r' ) : with open ( filename , mode ) as filey : data = json . load ( filey ) return data

read_json reads in a json file and returns the data structure as dict .


def clean_up(files):
    '''clean up will delete a list of files, only if they exist
    '''
    if not isinstance(files, list):
        files = [files]

    for f in files:
        if os.path.exists(f):
            bot.verbose3("Cleaning up %s" % f)
            os.remove(f)

def
clean_up
(
files
)
:
if
not
isinstance
(
files
,
list
)
:
files
=
[
files
]
for
f
in
files
:
if
os
.
path
.
exists
(
f
)
:
bot
.
verbose3
(
"Cleaning up %s"
%
f
)
os
.
remove
(
f
)

clean up will delete a list of files only if they exist


def push(self, path, name, tag=None):
    '''push an image to an S3 endpoint'''

    path = os.path.abspath(path)
    image = os.path.basename(path)
    bot.debug("PUSH %s" % path)

    if not os.path.exists(path):
        bot.error('%s does not exist.' %path)
        sys.exit(1)

    # Extract the metadata
    names = parse_image_name(remove_uri(name), tag=tag)
    image_size = os.path.getsize(path) >> 20

    # Create extra metadata, this is how we identify the image later
    # *important* bug in boto3 will return these capitalized
    # see https://github.com/boto/boto3/issues/1709
    metadata = {'sizemb': "%s" % image_size,
                'client': 'sregistry' }

    self.bucket.upload_file(path, names['storage_uri'], {"Metadata": metadata })


def get_or_create_collection(self, name):
    '''get a collection if it exists. If it doesn't exist, create it first.

    Parameters
    ==========
    name: the collection name, usually parsed from get_image_names()['name']

    '''
    from sregistry.database.models import Collection
    collection = self.get_collection(name)

    # If it doesn't exist, create it
    if collection is None:
        collection = Collection(name=name)
        self.session.add(collection)
        self.session.commit()

    return collection

def
get_or_create_collection
(
self
,
name
)
:
from
sregistry
.
database
.
models
import
Collection
collection
=
self
.
get_collection
(
name
)
# If it doesn't exist, create it
if
collection
is
:
collection
=
Collection
(
name
=
name
)
self
.
session
.
add
(
collection
)
self
.
session
.
commit
(
)
return
collection

get a collection if it exists . If it doesn t exist create it first .


def get_collection(self, name):
    '''get a collection, if it exists, otherwise return None.
    '''
    from sregistry.database.models import Collection
    return Collection.query.filter(Collection.name == name).first()

def get_collection ( self , name ) : from sregistry . database . models import Collection return Collection . query . filter ( Collection . name == name ) . first ( )

get a collection if it exists otherwise return None .


def get_container(self, name, collection_id, tag="latest", version=None):
    '''get a container, otherwise return None.
    '''
    from sregistry.database.models import Container
    if version is None:
        container = Container.query.filter_by(collection_id = collection_id,
                                              name = name,
                                              tag = tag).first()
    else:
        container = Container.query.filter_by(collection_id = collection_id,
                                              name = name,
                                              tag = tag,
                                              version = version).first()
    return container

def get_container ( self , name , collection_id , tag = "latest" , version = None ) : from sregistry . database . models import Container if version is None : container = Container . query . filter_by ( collection_id = collection_id , name = name , tag = tag ) . first ( ) else : container = Container . query . filter_by ( collection_id = collection_id , name = name , tag = tag , version = version ) . first ( ) return container


def images(self, query=None):
    '''List local images in the database, optionally with a query.

       Paramters
       =========
       query: a string to search for in the container or collection name|tag|uri

    '''
    from sregistry.database.models import Collection, Container

    rows = []
    if query is not None:
        like = "%" + query + "%"
        containers = Container.query.filter(or_(Container.name == query,
                                                Container.tag.like(like),
                                                Container.uri.like(like),
                                                Container.name.like(like))).all()
    else:
        containers = Container.query.all()

    if len(containers) > 0:
        message = "  [date]   [client]\t[uri]"
        bot.custom(prefix='Containers:', message=message, color="RED")
        for c in containers:
            uri = c.get_uri()
            created_at = c.created_at.strftime('%B %d, %Y')
            rows.append([created_at, "   [%s]" %c.client, uri])
        bot.table(rows)
    return containers

def
images
(
self
,
query
=
)
:
from
sregistry
.
database
.
models
import
Collection
,
Container
rows
=
[
]
if
query
is
not
:
like
=
"%"
+
query
+
"%"
containers
=
Container
.
query
.
filter
(
or_
(
Container
.
name
==
query
,
Container
.
tag
.
like
(
like
)
,
Container
.
uri
.
like
(
like
)
,
Container
.
name
.
like
(
like
)
)
)
.
all
(
)
else
:
containers
=
Container
.
query
.
all
(
)
if
len
(
containers
)
>
0
:
message
=
"  [date]   [client]\t[uri]"
bot
.
custom
(
prefix
=
'Containers:'
,
message
=
message
,
color
=
"RED"
)
for
c
in
containers
:
uri
=
c
.
get_uri
(
)
created_at
=
c
.
created_at
.
strftime
(
'%B %d, %Y'
)
rows
.
append
(
[
created_at
,
"   [%s]"
%
c
.
client
,
uri
]
)
bot
.
table
(
rows
)
return
containers

List local images in the database optionally with a query .


def inspect(self, name):
    '''Inspect a local image in the database, which typically includes the
       basic fields in the model.

    '''
    print(name)
    container = self.get(name)
    if container is not None:
        collection = container.collection.name
        fields = container.__dict__.copy()
        fields['collection'] = collection
        fields['metrics'] = json.loads(fields['metrics'])
        del fields['_sa_instance_state']
        fields['created_at'] = str(fields['created_at'])
        print(json.dumps(fields, indent=4, sort_keys=True))
        return fields

def inspect ( self , name ) : print ( name ) container = self . get ( name ) if container is not None : collection = container . collection . name fields = container . __dict__ . copy ( ) fields [ 'collection' ] = collection fields [ 'metrics' ] = json . loads ( fields [ 'metrics' ] ) del fields [ '_sa_instance_state' ] fields [ 'created_at' ] = str ( fields [ 'created_at' ] ) print ( json . dumps ( fields , indent = 4 , sort_keys = True ) ) return fields

Inspect a local image in the database which typically includes the basic fields in the model .


def rename(self, image_name, path):
    '''rename performs a move, but ensures the path is maintained in storage

       Parameters
       ==========
       image_name: the image name (uri) to rename to.
       path: the name to rename (basename is taken)

    '''
    container = self.get(image_name, quiet=True)

    if container is not None:
        if container.image is not None:

            # The original directory for the container stays the same
            dirname = os.path.dirname(container.image)

            # But we derive a new filename and uri

            names = parse_image_name( remove_uri (path) )
            storage = os.path.join( self.storage,
                                    os.path.dirname(names['storage']) )

            # This is the collection folder

            if not os.path.exists(storage):
                os.mkdir(storage)

            # Here we get the new full path, rename the container file

            fullpath = os.path.abspath(os.path.join(dirname, names['storage']))
            container = self.cp(move_to=fullpath,
                                container=container,
                                command="rename")

            # On successful rename of file, update the uri

            if container is not None:
                container.uri = names['uri']
                self.session.commit()
                return container

    bot.warning('%s not found' %(image_name))

rename performs a move but ensures the path is maintained in storage


def mv(self, image_name, path):
    '''Move an image from it's current location to a new path.
       Removing the image from organized storage is not the recommended approach
       however is still a function wanted by some.

       Parameters
       ==========
       image_name: the parsed image name.
       path: the location to move the image to

    '''

    container = self.get(image_name, quiet=True)

    if container is not None:

        name = container.uri or container.get_uri()
        image = container.image or ''

        # Only continue if image file exists
        if os.path.exists(image):

            # Default assume directory, use image name and path fully
            filename = os.path.basename(image)
            filedir = os.path.abspath(path)

            # If it's a file, use filename provided
            if not os.path.isdir(path):
                filename = os.path.basename(path)
                filedir = os.path.dirname(path)

            # If directory is empty, assume $PWD
            if filedir == '':
                filedir = os.getcwd()

            # Copy to the fullpath from the storage
            fullpath = os.path.abspath(os.path.join(filedir,filename))
            return self.cp(move_to=fullpath,
                           container=container,
                           command="move")

    bot.warning('%s not found' %(image_name))

Move an image from it s current location to a new path . Removing the image from organized storage is not the recommended approach however is still a function wanted by some .


def rmi(self, image_name):
    '''Remove an image from the database and filesystem.
    '''
    container = self.rm(image_name, delete=True)
    if container is not None:
        bot.info("[rmi] %s" % container)

def
rmi
(
self
,
image_name
)
:
container
=
self
.
rm
(
image_name
,
delete
=
True
)
if
container
is
not
:
bot
.
info
(
"[rmi] %s"
%
container
)

Remove an image from the database and filesystem .


def add(self, image_path=None,
              image_uri=None,
              image_name=None,
              url=None,
              metadata=None,
              save=True,
              copy=False):

    '''get or create a container, including the collection to add it to.
    This function can be used from a file on the local system, or via a URL
    that has been downloaded. Either way, if one of url, version, or image_file
    is not provided, the model is created without it. If a version is not
    provided but a file path is, then the file hash is used.

    Parameters
    ==========
    image_path: full path to image file
    image_name: if defined, the user wants a custom name (and not based on uri)
    metadata: any extra metadata to keep for the image (dict)
    save: if True, move the image to the cache if it's not there
    copy: If True, copy the image instead of moving it.

    image_name: a uri that gets parsed into a names object that looks like:

    {'collection': 'vsoch',
     'image': 'hello-world',
     'storage': 'vsoch/hello-world-latest.img',
     'tag': 'latest',
     'version': '12345'
     'uri': 'vsoch/hello-world:latest@12345'}

    After running add, the user will take some image in a working
    directory, add it to the database, and have it available for search
    and use under SREGISTRY_STORAGE/<collection>/<container>

    If the container was retrieved from a webby place, it should have version
    If no version is found, the file hash is used.
    '''

    from sregistry.database.models import (
        Container,
        Collection
    )

    # We can only save if the image is provided
    if image_path is not None:
        if not os.path.exists(image_path) and save is True:
            bot.error('Cannot find %s' %image_path)
            sys.exit(1)

    # An image uri is required for version, tag, etc.
    if image_uri is None:
        bot.error('You must provide an image uri <collection>/<namespace>')
        sys.exit(1)

    names = parse_image_name( remove_uri(image_uri) )
    bot.debug('Adding %s to registry' % names['uri'])

    # If Singularity is installed, inspect image for metadata
    metadata = self.get_metadata(image_path, names=names)
    collection = self.get_or_create_collection(names['collection'])

    # Get a hash of the file for the version, or use provided
    version = names.get('version')
    if version == None:
        if image_path != None:
            version = get_image_hash(image_path)
        else:
            version = ''  # we can't determine a version, not in API/no file
        names = parse_image_name( remove_uri(image_uri), version=version )

    # If save, move to registry storage first
    if save is True and image_path is not None:

        # If the user hasn't defined a custom name
        if image_name is None:
            image_name = self._get_storage_name(names)

        if copy is True:
            copyfile(image_path, image_name)
        else:
            shutil.move(image_path, image_name)

        image_path = image_name

    # Just in case the client didn't provide it, see if we have in metadata
    if url is None and "url" in metadata:
        url = metadata['url']

    # First check that we don't have one already!
    container = self.get_container(name=names['image'],
                                   collection_id=collection.id,
                                   tag=names['tag'],
                                   version=version)

    # The container did not exist, create it
    if container is None:
        action = "new"
        container = Container(metrics=json.dumps(metadata),
                              name=names['image'],
                              image=image_path,
                              client=self.client_name,
                              tag=names['tag'],
                              version=version,
                              url=url,
                              uri=names['uri'],
                              collection_id=collection.id)

        self.session.add(container)
        collection.containers.append(container)

    # The container existed, update it.
    else:
        action="update"
        metrics=json.loads(container.metrics)
        metrics.update(metadata)
        container.url= url
        container.client=self.client_name
        if image_path is not None:
            container.image=image_path
        container.metrics=json.dumps(metrics)

    self.session.commit()
    bot.info("[container][%s] %s" % (action,names['uri']))
    return container

get or create a container including the collection to add it to . This function can be used from a file on the local system or via a URL that has been downloaded . Either way if one of url version or image_file is not provided the model is created without it . If a version is not provided but a file path is then the file hash is used .


def push(self, path, name, tag=None):
    '''push an image to Singularity Registry'''

    path = os.path.abspath(path)
    image = os.path.basename(path)
    bot.debug("PUSH %s" % path)

    if not os.path.exists(path):
        bot.error('%s does not exist.' %path)
        sys.exit(1)

    # Interaction with a registry requires secrets
    self.require_secrets()

    # Extract the metadata
    names = parse_image_name(remove_uri(name), tag=tag)
    image_size = os.path.getsize(path) >> 20

# COLLECTION ###################################################################

    # If the registry is provided in the uri, use it
    if names['registry'] == None:
        names['registry'] = self.base

    # If the base doesn't start with http or https, add it
    names = self._add_https(names)

    # Prepare push request, this will return a collection ID if permission
    url = '%s/push/' % names['registry']
    auth_url = '%s/upload/chunked_upload' % names['registry']
    SREGISTRY_EVENT = self.authorize(request_type="push",
                                     names=names)

    # Data fields for collection
    fields = { 'collection': names['collection'],
               'name':names['image'],
               'tag': names['tag']}

    headers = { 'Authorization': SREGISTRY_EVENT }

    r = requests.post(auth_url, json=fields, headers=headers)

    # Always tell the user what's going on!
    message = self._read_response(r)
    print('\n[1. Collection return status {0} {1}]'.format(r.status_code, message))

    # Get the collection id, if created, and continue with upload
    if r.status_code != 200:
        sys.exit(1)

# UPLOAD #######################################################################

    url = '%s/upload' % names['registry'].replace('/api','')
    bot.debug('Seting upload URL to {0}'.format(url))

    cid = r.json()['cid']
    upload_to = os.path.basename(names['storage'])

    SREGISTRY_EVENT = self.authorize(request_type="upload",
                                     names=names)

    encoder = MultipartEncoder(fields={'SREGISTRY_EVENT': SREGISTRY_EVENT,
                                       'name': names['image'],
                                       'collection': str(cid),
                                       'tag': names['tag'],
                                       'file1': (upload_to, open(path, 'rb'), 'text/plain')})

    progress_callback = create_callback(encoder, self.quiet)
    monitor = MultipartEncoderMonitor(encoder, progress_callback)
    headers = {'Content-Type': monitor.content_type,
               'Authorization': SREGISTRY_EVENT }

    try:
        r = requests.post(url, data=monitor, headers=headers)
        r.raise_for_status()
        message = r.json()['message']
        print('\n[Return status {0} {1}]'.format(r.status_code, message))
    except requests.HTTPError as e:
        print('\nUpload failed: {0}.'.format(e))
    except KeyboardInterrupt:
        print('\nUpload cancelled.')
    except Exception as e:
        print(e)


def parse_header(recipe, header="from", remove_header=True):
    '''take a recipe, and return the complete header, line. If
       remove_header is True, only return the value.

       Parameters
       ==========
       recipe: the recipe file
       headers: the header key to find and parse
       remove_header: if true, remove the key

    '''
    parsed_header = None
    fromline = [x for x in recipe.split('\n') if "%s:" %header in x.lower()]

    # Case 1: We did not find the fromline
    if len(fromline) == 0:
        return ""

    # Case 2: We found it!
    if len(fromline) > 0:
        fromline = fromline[0]
        parsed_header = fromline.strip()

    # Does the user want to clean it up?
    if remove_header is True:
        parsed_header = fromline.split(':', 1)[-1].strip()
    return parsed_header

def
parse_header
(
recipe
,
header
=
"from"
,
remove_header
=
True
)
:
parsed_header
=
fromline
=
[
x
for
x
in
recipe
.
split
(
'\n'
)
if
"%s:"
%
header
in
x
.
lower
(
)
]
# Case 1: We did not find the fromline
if
len
(
fromline
)
==
0
:
return
""
# Case 2: We found it!
if
len
(
fromline
)
>
0
:
fromline
=
fromline
[
0
]
parsed_header
=
fromline
.
strip
(
)
# Does the user want to clean it up?
if
remove_header
is
True
:
parsed_header
=
fromline
.
split
(
':'
,
1
)
[
1
]
.
strip
(
)
return
parsed_header

take a recipe and return the complete header line . If remove_header is True only return the value .


def find_single_recipe(filename, pattern="Singularity", manifest=None):
    '''find_single_recipe will parse a single file, and if valid,
       return an updated manifest

       Parameters
       ==========
       filename: the filename to assess for a recipe
       pattern: a default pattern to search for
       manifest: an already started manifest

    '''

    if pattern is None:
        pattern = "Singularity*"

    recipe = None
    file_basename = os.path.basename(filename)
    if fnmatch.fnmatch(file_basename, pattern):
        recipe = {'path': os.path.abspath(filename),
                  'modified':os.path.getmtime(filename)}

    # If we already have the recipe, only add if more recent
    if manifest is not None and recipe is not None:
        container_uri = '/'.join(filename.split('/')[-2:])
        if container_uri in manifest:
            if manifest[container_uri]['modified'] < os.path.getmtime(filename):
                manifest[container_uri] = recipe
        else:
            manifest[container_uri] = recipe
        return manifest

    return recipe

find_single_recipe will parse a single file and if valid return an updated manifest


def create_build_package(package_files):
    '''given a list of files, copy them to a temporary folder,
       compress into a .tar.gz, and rename based on the file hash.
       Return the full path to the .tar.gz in the temporary folder.

       Parameters
       ==========
       package_files: a list of files to include in the tar.gz

    '''
    # Ensure package files all exist
    for package_file in package_files:
        if not os.path.exists(package_file):
            bot.exit('Cannot find %s.' % package_file)

    bot.log('Generating build package for %s files...' % len(package_files))
    build_dir = get_tmpdir(prefix="sregistry-build")
    build_tar = '%s/build.tar.gz' % build_dir
    tar = tarfile.open(build_tar, "w:gz")

    # Create the tar.gz
    for package_file in package_files:
        tar.add(package_file)
    tar.close()

    # Get hash (sha256), and rename file
    sha256 = get_file_hash(build_tar)
    hash_tar =  "%s/%s.tar.gz" %(build_dir, sha256)
    shutil.move(build_tar, hash_tar)
    return hash_tar

given a list of files copy them to a temporary folder compress into a . tar . gz and rename based on the file hash . Return the full path to the . tar . gz in the temporary folder .


def run_build(self, config, bucket, names):
    '''run a build, meaning creating a build. Retry if there is failure
    '''

    project = self._get_project()

    #          prefix,    message, color
    bot.custom('PROJECT', project, "CYAN")
    bot.custom('BUILD  ', config['steps'][0]['name'], "CYAN")

    response = self._build_service.projects().builds().create(body=config,
                                              projectId=project).execute()

    build_id = response['metadata']['build']['id']
    status = response['metadata']['build']['status']
    bot.log("build %s: %s" % (build_id, status))

    start = time.time()
    while status not in ['COMPLETE', 'FAILURE', 'SUCCESS']:
        time.sleep(15)
        response = self._build_service.projects().builds().get(id=build_id,
                                                  projectId=project).execute()

        build_id = response['id']
        status = response['status']
        bot.log("build %s: %s" % (build_id, status))

    end = time.time()
    bot.log('Total build time: %s seconds' % (round(end - start, 2)))

    # If successful, update blob metadata and visibility
    if status == 'SUCCESS':

        # Does the user want to keep the container private?
        env = 'SREGISTRY_GOOGLE_STORAGE_PRIVATE'
        blob = bucket.blob(response['artifacts']['objects']['paths'][0])

        # Make Public, if desired
        if self._get_and_update_setting(env) == None:
            blob.make_public()
            response['public_url'] = blob.public_url

        # Add the metadata directly to the object
        update_blob_metadata(blob, response, config, bucket, names)
        response['media_link'] = blob.media_link
        response['size'] = blob.size
        response['file_hash'] = blob.md5_hash

    return response

run a build meaning creating a build . Retry if there is failure


def update_blob_metadata(blob, response, config, bucket, names):
    '''a specific function to take a blob, along with a SUCCESS response
       from Google build, the original config, and update the blob
       metadata with the artifact file name, dependencies, and image hash.
    '''

    manifest = os.path.basename(response['results']['artifactManifest'])
    manifest = json.loads(bucket.blob(manifest).download_as_string())

    metadata = {'file_hash': manifest['file_hash'][0]['file_hash'][0]['value'],
                'artifactManifest': response['results']['artifactManifest'],
                'location': manifest['location'],
                'storageSourceBucket': config['source']['storageSource']['bucket'],
                'storageSourceObject': config['source']['storageSource']['object'],
                'buildCommand': ' '.join(config['steps'][0]['args']),
                'builder': config['steps'][0]['name'],
                'media_link': blob.media_link,
                'self_link': blob.self_link,
                'size': blob.size,
                'name': names['tag_uri'],
                'type': "container"} # identifier that the blob is a container

    blob.metadata = metadata
    blob._properties['metadata'] = metadata
    blob.patch()

a specific function to take a blob along with a SUCCESS response from Google build the original config and update the blob metadata with the artifact file name dependencies and image hash .


def format_container_name(name, special_characters=None):
    '''format_container_name will take a name supplied by the user,
    remove all special characters (except for those defined by "special-characters"
    and return the new image name.
    '''
    if special_characters is None:
        special_characters = []
    return ''.join(e.lower()
                   for e in name if e.isalnum() or e in special_characters)

def format_container_name ( name , special_characters = None ) : if special_characters is None : special_characters = [ ] return '' . join ( e . lower ( ) for e in name if e . isalnum ( ) or e in special_characters )

format_container_name will take a name supplied by the user remove all special characters ( except for those defined by special - characters and return the new image name .


def useColor(self):
        '''useColor will determine if color should be added
        to a print. Will check if being run in a terminal, and
        if has support for asci'''
        COLORIZE = get_user_color_preference()
        if COLORIZE is not None:
            return COLORIZE
        streams = [self.errorStream, self.outputStream]
        for stream in streams:
            if not hasattr(stream, 'isatty'):
                return False
            if not stream.isatty():
                return False
        return True

def useColor ( self ) : COLORIZE = get_user_color_preference ( ) if COLORIZE is not None : return COLORIZE streams = [ self . errorStream , self . outputStream ] for stream in streams : if not hasattr ( stream , 'isatty' ) : return False if not stream . isatty ( ) : return False return True

useColor will determine if color should be added to a print . Will check if being run in a terminal and if has support for asci


def emitError(self, level):
        '''determine if a level should print to
        stderr, includes all levels but INFO and QUIET'''
        if level in [ABORT,
                     ERROR,
                     WARNING,
                     VERBOSE,
                     VERBOSE1,
                     VERBOSE2,
                     VERBOSE3,
                     DEBUG]:
            return True
        return False

def emitError ( self , level ) : if level in [ ABORT , ERROR , WARNING , VERBOSE , VERBOSE1 , VERBOSE2 , VERBOSE3 , DEBUG ] : return True return False

determine if a level should print to stderr includes all levels but INFO and QUIET


def write(self, stream, message):
        '''write will write a message to a stream,
        first checking the encoding
        '''
        if isinstance(message, bytes):
            message = message.decode('utf-8')
        stream.write(message)

def write ( self , stream , message ) : if isinstance ( message , bytes ) : message = message . decode ( 'utf-8' ) stream . write ( message )

write will write a message to a stream first checking the encoding


def table(self, rows, col_width=2):
        '''table will print a table of entries. If the rows is
        a dictionary, the keys are interpreted as column names. if
        not, a numbered list is used.
        '''

        labels = [str(x) for x in range(1,len(rows)+1)]
        if isinstance(rows, dict):
            labels = list(rows.keys())
            rows = list(rows.values())

        for row in rows:
            label = labels.pop(0)
            label = label.ljust(col_width)
            message = "\t".join(row)
            self.custom(prefix=label,
                        message=message)

def table ( self , rows , col_width = 2 ) : labels = [ str ( x ) for x in range ( 1 , len ( rows ) + 1 ) ] if isinstance ( rows , dict ) : labels = list ( rows . keys ( ) ) rows = list ( rows . values ( ) ) for row in rows : label = labels . pop ( 0 ) label = label . ljust ( col_width ) message = "\t" . join ( row ) self . custom ( prefix = label , message = message )

table will print a table of entries . If the rows is a dictionary the keys are interpreted as column names . if not a numbered list is used .


def push(self, path, name, tag=None):
    '''push an image to Globus endpoint. In this case, the name is the
       globus endpoint id and path.

       --name <endpointid>:/path/for/image

    '''

    # Split the name into endpoint and rest

    endpoint, remote = self._parse_endpoint_name(name)

    path = os.path.abspath(path)
    image = os.path.basename(path)
    bot.debug("PUSH %s" % path)

    # Flatten image uri into image name

    q = parse_image_name(image)

    if not os.path.exists(path):
        bot.error('%s does not exist.' %path)
        sys.exit(1)

    # Ensure we have a transfer client
    if not hasattr(self, 'transfer_client'):
        self._init_transfer_client()

    # The user must have a personal endpoint

    endpoints = self._get_endpoints()

    if len(endpoints['my-endpoints']) == 0:
        bot.error('You must have a personal endpoint to transfer the container')
        sys.exit(1)

    # Take the first endpoint that is active

    source_endpoint = None
    for eid,contender in endpoints['my-endpoints'].items():
       if contender['gcp_connected'] is True:
           source_endpoint = contender
           break

    # Exit if none are active, required!

    if source_endpoint is None:
        bot.error('No activated local endpoints online! Go online to transfer')
        sys.exit(1)

    # The destination endpoint should have an .singularity/shub folder set
    self._create_endpoint_cache(endpoint)

    # SREGISTRY_STORAGE must be an endpoint
    # if the image isn't already there, add it first

    added = self.add(image_path=path,
                     image_uri=q['uri'],
                     copy=True)

    label = "Singularity Registry Transfer for %s" %added.name
    tdata = globus_sdk.TransferData(self.transfer_client,
                                    source_endpoint['id'],
                                    endpoint,
                                    label=label,
                                    sync_level="checksum")
    image = ".singularity/shub/%s" %image
    tdata.add_item(added.image, image)
    bot.info('Requesting transfer from local %s to %s:%s' %(SREGISTRY_STORAGE,
                                                            endpoint, image))
    transfer_result = self.transfer_client.submit_transfer(tdata)
    bot.info(transfer_result['message'])
    return transfer_result

push an image to Globus endpoint . In this case the name is the globus endpoint id and path .


def get_template(name):
    '''return a default template for some function in sregistry
       If there is no template, None is returned.

       Parameters
       ==========
       name: the name of the template to retrieve

    '''
    name = name.lower()
    templates = dict()

    templates['tarinfo'] = {"gid": 0,
                            "uid": 0,
                            "uname": "root",
                            "gname": "root",
                            "mode": 493}

    if name in templates:
        bot.debug("Found template for %s" % (name))
        return templates[name]
    else:
        bot.warning("Cannot find template %s" % (name))

def
get_template
(
name
)
:
name
=
name
.
lower
(
)
templates
=
dict
(
)
templates
[
'tarinfo'
]
=
{
"gid"
:
0
,
"uid"
:
0
,
"uname"
:
"root"
,
"gname"
:
"root"
,
"mode"
:
493
}
if
name
in
templates
:
bot
.
debug
(
"Found template for %s"
%
(
name
)
)
return
templates
[
name
]
else
:
bot
.
warning
(
"Cannot find template %s"
%
(
name
)
)

return a default template for some function in sregistry If there is no template None is returned .


def get_manifest(self, repo_name, tag):
    '''return the image manifest via the aws client, saved in self.manifest
    '''

    image = None
    repo = self.aws.describe_images(repositoryName=repo_name)
    if 'imageDetails' in repo:
        for contender in repo.get('imageDetails'):
            if tag in contender['imageTags']:
                image = contender
                break

    # if the image isn't found, we need to exit
    if image is None:
        bot.exit('Cannot find %s:%s, is the uri correct?' %(repo_name, digest))

    digest = image['imageDigest']
    digests = self.aws.batch_get_image(repositoryName=repo_name,
                                       imageIds=[{"imageDigest": digest,
                                                  "imageTag": tag}])

    self.manifest = json.loads(digests['images'][0]['imageManifest'])
    return self.manifest

return the image manifest via the aws client saved in self . manifest


def get_build_template(name=None, manager='apt'):
    '''get a particular build template, by default we return templates
       that are based on package managers.

       Parameters
       ==========
       name: the full path of the template file to use.
       manager: the package manager to use in the template (yum or apt)

    '''
    base = get_installdir()
    if name is None:
        name = "%s/main/templates/build/singularity-builder-%s.sh" %(base,
                                                                     manager)

    if os.path.exists(name):
        bot.debug("Found template %s" %name)
        return ''.join(read_file(name))

    bot.warning("Template %s not found." %name)

get a particular build template by default we return templates that are based on package managers .


def _update_secrets(self):
        '''update secrets will take a secrets credential file
           either located at .sregistry or the environment variable
           SREGISTRY_CLIENT_SECRETS and update the current client
           secrets as well as the associated API base. This is where you
           should do any customization of the secrets flie, or using
           it to update your client, if needed.
        '''
        # Get a setting for client myclient and some variable name VAR.
        # returns None if not set
        setting = self._get_setting('SREGISTRY_MYCLIENT_VAR')

        # Get (and if found in environment (1) settings (2) update the variable
        # It will still return None if not set
        setting = self._get_and_update_setting('SREGISTRY_MYCLIENT_VAR')

        # If you have a setting that is required and not found, you should exit.

        # Here is how to read all client secrets
        self.secrets = read_client_secrets()

        # If you don't want to use the shared settings file, you have your own.
        # Here is how to get if the user has a cache for you enabled, this
        # returns a path (enabled) or None (disabled) that you should honor
        # You can use this as a file path or folder and for both cases, you
        # need to create the file or folder
        if self._credential_cache is not None:
            bot.info("credential cache set to %s" %self._credential_cache)

update secrets will take a secrets credential file either located at . sregistry or the environment variable SREGISTRY_CLIENT_SECRETS and update the current client secrets as well as the associated API base . This is where you should do any customization of the secrets flie or using it to update your client if needed .


def _make_repr(class_name, *args, **kwargs):
    """
    Generate a repr string.

    Positional arguments should be the positional arguments used to
    construct the class. Keyword arguments should consist of tuples of
    the attribute value and default. If the value is the default, then
    it won't be rendered in the output.

    Here's an example::

        def __repr__(self):
            return make_repr('MyClass', 'foo', name=(self.name, None))

    The output of this would be something line ``MyClass('foo',
    name='Will')``.

    """
    arguments = [repr(arg) for arg in args]
    arguments.extend(
        "{}={!r}".format(name, value)
        for name, (value, default) in sorted(kwargs.items())
        if value != default
    )
    return "{}({})".format(class_name, ", ".join(arguments))

def
_make_repr
(
class_name
,
*
args
,
*
*
kwargs
)
:
arguments
=
[
repr
(
arg
)
for
arg
in
args
]
arguments
.
extend
(
"{}={!r}"
.
format
(
name
,
value
)
for
name
,
(
value
,
default
)
in
sorted
(
kwargs
.
items
(
)
)
if
value
!=
default
)
return
"{}({})"
.
format
(
class_name
,
", "
.
join
(
arguments
)
)


def s3errors(path):
    """Translate S3 errors to FSErrors."""
    try:
        yield
    except ClientError as error:
        _error = error.response.get("Error", {})
        error_code = _error.get("Code", None)
        response_meta = error.response.get("ResponseMetadata", {})
        http_status = response_meta.get("HTTPStatusCode", 200)
        error_msg = _error.get("Message", None)
        if error_code == "NoSuchBucket":
            raise errors.ResourceError(path, exc=error, msg=error_msg)
        if http_status == 404:
            raise errors.ResourceNotFound(path)
        elif http_status == 403:
            raise errors.PermissionDenied(path=path, msg=error_msg)
        else:
            raise errors.OperationFailed(path=path, exc=error)
    except SSLError as error:
        raise errors.OperationFailed(path, exc=error)
    except EndpointConnectionError as error:
        raise errors.RemoteConnectionError(path, exc=error, msg="{}".format(error))

def s3errors ( path ) : try : yield except ClientError as error : _error = error . response . get ( "Error" , { } ) error_code = _error . get ( "Code" , None ) response_meta = error . response . get ( "ResponseMetadata" , { } ) http_status = response_meta . get ( "HTTPStatusCode" , 200 ) error_msg = _error . get ( "Message" , None ) if error_code == "NoSuchBucket" : raise errors . ResourceError ( path , exc = error , msg = error_msg ) if http_status == 404 : raise errors . ResourceNotFound ( path ) elif http_status == 403 : raise errors . PermissionDenied ( path = path , msg = error_msg ) else : raise errors . OperationFailed ( path = path , exc = error ) except SSLError as error : raise errors . OperationFailed ( path , exc = error ) except EndpointConnectionError as error : raise errors . RemoteConnectionError ( path , exc = error , msg = "{}" . format ( error ) )


def factory(cls, filename, mode, on_close):
        """Create a S3File backed with a temporary file."""
        _temp_file = tempfile.TemporaryFile()
        proxy = cls(_temp_file, filename, mode, on_close=on_close)
        return proxy

def factory ( cls , filename , mode , on_close ) : _temp_file = tempfile . TemporaryFile ( ) proxy = cls ( _temp_file , filename , mode , on_close = on_close ) return proxy

Create a S3File backed with a temporary file .


def gravatar_url(user_or_email, size=GRAVATAR_DEFAULT_SIZE):
    """ Builds a gravatar url from an user or email """
    if hasattr(user_or_email, 'email'):
        email = user_or_email.email
    else:
        email = user_or_email

    try:
        return escape(get_gravatar_url(email=email, size=size))
    except:
        return ''

def gravatar_url ( user_or_email , size = GRAVATAR_DEFAULT_SIZE ) : if hasattr ( user_or_email , 'email' ) : email = user_or_email . email else : email = user_or_email try : return escape ( get_gravatar_url ( email = email , size = size ) ) except : return ''


def get_gravatar_url(email, size=GRAVATAR_DEFAULT_SIZE, default=GRAVATAR_DEFAULT_IMAGE,
        rating=GRAVATAR_DEFAULT_RATING, secure=GRAVATAR_DEFAULT_SECURE):
    """
    Builds a url to a gravatar from an email address.

    :param email: The email to fetch the gravatar for
    :param size: The size (in pixels) of the gravatar to fetch
    :param default: What type of default image to use if the gravatar does not exist
    :param rating: Used to filter the allowed gravatar ratings
    :param secure: If True use https, otherwise plain http
    """
    if secure:
        url_base = GRAVATAR_SECURE_URL
    else:
        url_base = GRAVATAR_URL

    # Calculate the email hash
    email_hash = calculate_gravatar_hash(email)

    # Build querystring
    query_string = urlencode({
        's': str(size),
        'd': default,
        'r': rating,
    })

    # Build url
    url = '{base}avatar/{hash}.jpg?{qs}'.format(base=url_base,
            hash=email_hash, qs=query_string)

    return url

def
get_gravatar_url
(
email
,
size
=
GRAVATAR_DEFAULT_SIZE
,
default
=
GRAVATAR_DEFAULT_IMAGE
,
rating
=
GRAVATAR_DEFAULT_RATING
,
secure
=
GRAVATAR_DEFAULT_SECURE
)
:
if
secure
:
url_base
=
GRAVATAR_SECURE_URL
else
:
url_base
=
GRAVATAR_URL
# Calculate the email hash
email_hash
=
calculate_gravatar_hash
(
email
)
# Build querystring
query_string
=
urlencode
(
{
's'
:
str
(
size
)
,
'd'
:
default
,
'r'
:
rating
,
}
)
# Build url
url
=
'{base}avatar/{hash}.jpg?{qs}'
.
format
(
base
=
url_base
,
hash
=
email_hash
,
qs
=
query_string
)
return
url

Builds a url to a gravatar from an email address .


def has_gravatar(email):
    """
    Returns True if the user has a gravatar, False if otherwise
    """
    # Request a 404 response if the gravatar does not exist
    url = get_gravatar_url(email, default=GRAVATAR_DEFAULT_IMAGE_404)

    # Verify an OK response was received
    try:
        request = Request(url)
        request.get_method = lambda: 'HEAD'
        return 200 == urlopen(request).code
    except (HTTPError, URLError):
        return False

Returns True if the user has a gravatar False if otherwise


def get_gravatar_profile_url(email, secure=GRAVATAR_DEFAULT_SECURE):
    """
    Builds a url to a gravatar profile from an email address.

    :param email: The email to fetch the gravatar for
    :param secure: If True use https, otherwise plain http
    """
    if secure:
        url_base = GRAVATAR_SECURE_URL
    else:
        url_base = GRAVATAR_URL

    # Calculate the email hash
    email_hash = calculate_gravatar_hash(email)

    # Build url
    url = '{base}{hash}'.format(base=url_base, hash=email_hash)

    return url

def
get_gravatar_profile_url
(
email
,
secure
=
GRAVATAR_DEFAULT_SECURE
)
:
if
secure
:
url_base
=
GRAVATAR_SECURE_URL
else
:
url_base
=
GRAVATAR_URL
# Calculate the email hash
email_hash
=
calculate_gravatar_hash
(
email
)
# Build url
url
=
'{base}{hash}'
.
format
(
base
=
url_base
,
hash
=
email_hash
)
return
url

Builds a url to a gravatar profile from an email address .


def chimera_blocks(M=16, N=16, L=4):
    """
    Generator for blocks for a chimera block quotient
    """
    for x in xrange(M):
        for y in xrange(N):
            for u in (0, 1):
                yield tuple((x, y, u, k) for k in xrange(L))

def chimera_blocks ( M = 16 , N = 16 , L = 4 ) : for x in xrange ( M ) : for y in xrange ( N ) : for u in ( 0 , 1 ) : yield tuple ( ( x , y , u , k ) for k in xrange ( L ) )


def chimera_block_quotient(G, blocks):
    """
    Extract the blocks from a graph, and returns a
    block-quotient graph according to the acceptability
    functions block_good and eblock_good

    Inputs:
        G: a networkx graph
        blocks: a tuple of tuples

    """
    from networkx import Graph
    from itertools import product

    BG = Graph()
    blockid = {}
    for i, b in enumerate(blocks):
        BG.add_node(i)
        if not b or not all(G.has_node(x) for x in b):
            continue
        for q in b:
            if q in blockid:
                raise(RuntimeError, "two blocks overlap")
            blockid[q] = i

    for q, u in blockid.items():
        ublock = blocks[u]
        for p in G[q]:
            if p not in blockid:
                continue
            v = blockid[p]
            if BG.has_edge(u, v) or u == v:
                continue
            vblock = blocks[v]

            if ublock[0][2] == vblock[0][2]:
                block_edges = zip(ublock, vblock)
            else:
                block_edges = product(ublock, vblock)

            if all(G.has_edge(x, y) for x, y in block_edges):
                BG.add_edge(u, v)

    return BG

def
chimera_block_quotient
(
G
,
blocks
)
:
from
networkx
import
Graph
from
itertools
import
product
BG
=
Graph
(
)
blockid
=
{
}
for
i
,
b
in
enumerate
(
blocks
)
:
BG
.
add_node
(
i
)
if
not
b
or
not
all
(
G
.
has_node
(
x
)
for
x
in
b
)
:
continue
for
q
in
b
:
if
q
in
blockid
:
raise
(
RuntimeError
,
"two blocks overlap"
)
blockid
[
q
]
=
i
for
q
,
u
in
blockid
.
items
(
)
:
ublock
=
blocks
[
u
]
for
p
in
G
[
q
]
:
if
p
not
in
blockid
:
continue
v
=
blockid
[
p
]
if
BG
.
has_edge
(
u
,
v
)
or
u
==
v
:
continue
vblock
=
blocks
[
v
]
if
ublock
[
0
]
[
2
]
==
vblock
[
0
]
[
2
]
:
block_edges
=
zip
(
ublock
,
vblock
)
else
:
block_edges
=
product
(
ublock
,
vblock
)
if
all
(
G
.
has_edge
(
x
,
y
)
for
x
,
y
in
block_edges
)
:
BG
.
add_edge
(
u
,
v
)
return
BG

Extract the blocks from a graph and returns a block - quotient graph according to the acceptability functions block_good and eblock_good


def enumerate_resonance_smiles(smiles):
    """Return a set of resonance forms as SMILES strings, given a SMILES string.

    :param smiles: A SMILES string.
    :returns: A set containing SMILES strings for every possible resonance form.
    :rtype: set of strings.
    """
    mol = Chem.MolFromSmiles(smiles)
    #Chem.SanitizeMol(mol)  # MolFromSmiles does Sanitize by default
    mesomers = ResonanceEnumerator().enumerate(mol)
    return {Chem.MolToSmiles(m, isomericSmiles=True) for m in mesomers}

Return a set of resonance forms as SMILES strings given a SMILES string .


def enumerate(self, mol):
        """Enumerate all possible resonance forms and return them as a list.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: A list of all possible resonance forms of the molecule.
        :rtype: list of rdkit.Chem.rdchem.Mol
        """
        flags = 0
        if self.kekule_all:
            flags = flags | Chem.KEKULE_ALL
        if self.allow_incomplete_octets:
            flags = flags | Chem.ALLOW_INCOMPLETE_OCTETS
        if self.allow_charge_separation:
            flags = flags | Chem.ALLOW_CHARGE_SEPARATION
        if self.unconstrained_anions:
            flags = flags | Chem.UNCONSTRAINED_ANIONS
        if self.unconstrained_cations:
            flags = flags | Chem.UNCONSTRAINED_CATIONS
        results = []
        for result in Chem.ResonanceMolSupplier(mol, flags=flags, maxStructs=self.max_structures):
            # This seems necessary? ResonanceMolSupplier only does a partial sanitization
            Chem.SanitizeMol(result)
            results.append(result)
        return results

Enumerate all possible resonance forms and return them as a list .


def normalize(self, mol):
        """Apply a series of Normalization transforms to correct functional groups and recombine charges.

        A series of transforms are applied to the molecule. For each Normalization, the transform is applied repeatedly
        until no further changes occur. If any changes occurred, we go back and start from the first Normalization
        again, in case the changes mean an earlier transform is now applicable. The molecule is returned once the entire
        series of Normalizations cause no further changes or if max_restarts (default 200) is reached.

        :param mol: The molecule to normalize.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The normalized fragment.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        log.debug('Running Normalizer')
        # Normalize each fragment separately to get around quirky RunReactants behaviour
        fragments = []
        for fragment in Chem.GetMolFrags(mol, asMols=True):
            fragments.append(self._normalize_fragment(fragment))
        # Join normalized fragments into a single molecule again
        outmol = fragments.pop()
        for fragment in fragments:
            outmol = Chem.CombineMols(outmol, fragment)
        Chem.SanitizeMol(outmol)
        return outmol

Apply a series of Normalization transforms to correct functional groups and recombine charges .


def _apply_transform(self, mol, rule):
        """Repeatedly apply normalization transform to molecule until no changes occur.

        It is possible for multiple products to be produced when a rule is applied. The rule is applied repeatedly to
        each of the products, until no further changes occur or after 20 attempts. If there are multiple unique products
        after the final application, the first product (sorted alphabetically by SMILES) is chosen.
        """
        mols = [mol]
        for n in six.moves.range(20):
            products = {}
            for mol in mols:
                for product in [x[0] for x in rule.RunReactants((mol,))]:
                    if Chem.SanitizeMol(product, catchErrors=True) == 0:
                        products[Chem.MolToSmiles(product, isomericSmiles=True)] = product
            if products:
                mols = [products[s] for s in sorted(products)]
            else:
                # If n == 0, the rule was not applicable and we return None
                return mols[0] if n > 0 else None

Repeatedly apply normalization transform to molecule until no changes occur .


def canonicalize(self, mol):
        """Return a canonical tautomer by enumerating and scoring all possible tautomers.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The canonical tautomer.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        # TODO: Overload the mol parameter to pass a list of pre-enumerated tautomers
        tautomers = self._enumerate_tautomers(mol)
        if len(tautomers) == 1:
            return tautomers[0]
        # Calculate score for each tautomer
        highest = None
        for t in tautomers:
            smiles = Chem.MolToSmiles(t, isomericSmiles=True)
            log.debug('Tautomer: %s', smiles)
            score = 0
            # Add aromatic ring scores
            ssr = Chem.GetSymmSSSR(t)
            for ring in ssr:
                btypes = {t.GetBondBetweenAtoms(*pair).GetBondType() for pair in pairwise(ring)}
                elements = {t.GetAtomWithIdx(idx).GetAtomicNum() for idx in ring}
                if btypes == {BondType.AROMATIC}:
                    log.debug('Score +100 (aromatic ring)')
                    score += 100
                    if elements == {6}:
                        log.debug('Score +150 (carbocyclic aromatic ring)')
                        score += 150
            # Add SMARTS scores
            for tscore in self.scores:
                for match in t.GetSubstructMatches(tscore.smarts):
                    log.debug('Score %+d (%s)', tscore.score, tscore.name)
                    score += tscore.score
            # Add (P,S,Se,Te)-H scores
            for atom in t.GetAtoms():
                if atom.GetAtomicNum() in {15, 16, 34, 52}:
                    hs = atom.GetTotalNumHs()
                    if hs:
                        log.debug('Score %+d (%s-H bonds)', -hs, atom.GetSymbol())
                        score -= hs
            # Set as highest if score higher or if score equal and smiles comes first alphabetically
            if not highest or highest['score'] < score or (highest['score'] == score and smiles < highest['smiles']):
                log.debug('New highest tautomer: %s (%s)', smiles, score)
                highest = {'smiles': smiles, 'tautomer': t, 'score': score}
        return highest['tautomer']

Return a canonical tautomer by enumerating and scoring all possible tautomers .


def validate_smiles(smiles):
    """Return log messages for a given SMILES string using the default validations.

    Note: This is a convenience function for quickly validating a single SMILES string. It is more efficient to use
    the :class:`~molvs.validate.Validator` class directly when working with many molecules or when custom options
    are needed.

    :param string smiles: The SMILES for the molecule.
    :returns: A list of log messages.
    :rtype: list of strings.
    """
    # Skip sanitize as standardize does this anyway
    mol = Chem.MolFromSmiles(smiles)
    logs = Validator().validate(mol)
    return logs

Return log messages for a given SMILES string using the default validations .


def disconnect(self, mol):
        """Break covalent bonds between metals and organic atoms under certain conditions.

        The algorithm works as follows:

        - Disconnect N, O, F from any metal.
        - Disconnect other non-metals from transition metals + Al (but not Hg, Ga, Ge, In, Sn, As, Tl, Pb, Bi, Po).
        - For every bond broken, adjust the charges of the begin and end atoms accordingly.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The molecule with metals disconnected.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        log.debug('Running MetalDisconnector')
        # Remove bonds that match SMARTS
        for smarts in [self._metal_nof, self._metal_non]:
            pairs = mol.GetSubstructMatches(smarts)
            rwmol = Chem.RWMol(mol)
            orders = []
            for i, j in pairs:
                # TODO: Could get the valence contributions of the bond instead of GetBondTypeAsDouble?
                orders.append(int(mol.GetBondBetweenAtoms(i, j).GetBondTypeAsDouble()))
                rwmol.RemoveBond(i, j)
            # Adjust neighbouring charges accordingly
            mol = rwmol.GetMol()
            for n, (i, j) in enumerate(pairs):
                chg = orders[n]
                atom1 = mol.GetAtomWithIdx(i)
                atom1.SetFormalCharge(atom1.GetFormalCharge() + chg)
                atom2 = mol.GetAtomWithIdx(j)
                atom2.SetFormalCharge(atom2.GetFormalCharge() - chg)
                log.info('Removed covalent bond between %s and %s', atom1.GetSymbol(), atom2.GetSymbol())
        Chem.SanitizeMol(mol)
        return mol

Break covalent bonds between metals and organic atoms under certain conditions .


def standardize_smiles(smiles):
    """Return a standardized canonical SMILES string given a SMILES string.

    Note: This is a convenience function for quickly standardizing a single SMILES string. It is more efficient to use
    the :class:`~molvs.standardize.Standardizer` class directly when working with many molecules or when custom options
    are needed.

    :param string smiles: The SMILES for the molecule.
    :returns: The SMILES for the standardized molecule.
    :rtype: string.
    """
    # Skip sanitize as standardize does this anyway
    mol = Chem.MolFromSmiles(smiles, sanitize=False)
    mol = Standardizer().standardize(mol)
    return Chem.MolToSmiles(mol, isomericSmiles=True)

Return a standardized canonical SMILES string given a SMILES string .


def enumerate_tautomers_smiles(smiles):
    """Return a set of tautomers as SMILES strings, given a SMILES string.

    :param smiles: A SMILES string.
    :returns: A set containing SMILES strings for every possible tautomer.
    :rtype: set of strings.
    """
    # Skip sanitize as standardize does this anyway
    mol = Chem.MolFromSmiles(smiles, sanitize=False)
    mol = Standardizer().standardize(mol)
    tautomers = TautomerEnumerator().enumerate(mol)
    return {Chem.MolToSmiles(m, isomericSmiles=True) for m in tautomers}

Return a set of tautomers as SMILES strings given a SMILES string .


def canonicalize_tautomer_smiles(smiles):
    """Return a standardized canonical tautomer SMILES string given a SMILES string.

    Note: This is a convenience function for quickly standardizing and finding the canonical tautomer for a single
    SMILES string. It is more efficient to use the :class:`~molvs.standardize.Standardizer` class directly when working
    with many molecules or when custom options are needed.

    :param string smiles: The SMILES for the molecule.
    :returns: The SMILES for the standardize canonical tautomer.
    :rtype: string.
    """
    # Skip sanitize as standardize does this anyway
    mol = Chem.MolFromSmiles(smiles, sanitize=False)
    mol = Standardizer().standardize(mol)
    tautomer = TautomerCanonicalizer().canonicalize(mol)
    return Chem.MolToSmiles(tautomer, isomericSmiles=True)

Return a standardized canonical tautomer SMILES string given a SMILES string .


def standardize(self, mol):
        """Return a standardized version the given molecule.

        The standardization process consists of the following stages: RDKit
        :py:func:`~rdkit.Chem.rdmolops.RemoveHs`, RDKit :py:func:`~rdkit.Chem.rdmolops.SanitizeMol`,
        :class:`~molvs.metal.MetalDisconnector`, :class:`~molvs.normalize.Normalizer`,
        :class:`~molvs.charge.Reionizer`, RDKit :py:func:`~rdkit.Chem.rdmolops.AssignStereochemistry`.

        :param mol: The molecule to standardize.
        :type mol: rdkit.Chem.rdchem.Mol
        :returns: The standardized molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        mol = copy.deepcopy(mol)
        Chem.SanitizeMol(mol)
        mol = Chem.RemoveHs(mol)
        mol = self.disconnect_metals(mol)
        mol = self.normalize(mol)
        mol = self.reionize(mol)
        Chem.AssignStereochemistry(mol, force=True, cleanIt=True)
        # TODO: Check this removes symmetric stereocenters
        return mol

Return a standardized version the given molecule .


def tautomer_parent(self, mol, skip_standardize=False):
        """Return the tautomer parent of a given molecule.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The tautomer parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        tautomer = self.canonicalize_tautomer(mol)
        tautomer = self.standardize(tautomer)
        return tautomer

def tautomer_parent ( self , mol , skip_standardize = False ) : if not skip_standardize : mol = self . standardize ( mol ) tautomer = self . canonicalize_tautomer ( mol ) tautomer = self . standardize ( tautomer ) return tautomer

Return the tautomer parent of a given molecule .


def fragment_parent(self, mol, skip_standardize=False):
        """Return the fragment parent of a given molecule.

        The fragment parent is the largest organic covalent unit in the molecule.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The fragment parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        # TODO: Consider applying FragmentRemover first to remove salts, solvents?
        fragment = self.largest_fragment(mol)
        return fragment

Return the fragment parent of a given molecule .


def stereo_parent(self, mol, skip_standardize=False):
        """Return the stereo parent of a given molecule.

        The stereo parent has all stereochemistry information removed from tetrahedral centers and double bonds.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The stereo parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        else:
            mol = copy.deepcopy(mol)
        Chem.RemoveStereochemistry(mol)
        return mol

def stereo_parent ( self , mol , skip_standardize = False ) : if not skip_standardize : mol = self . standardize ( mol ) else : mol = copy . deepcopy ( mol ) Chem . RemoveStereochemistry ( mol ) return mol

Return the stereo parent of a given molecule .


def isotope_parent(self, mol, skip_standardize=False):
        """Return the isotope parent of a given molecule.

        The isotope parent has all atoms replaced with the most abundant isotope for that element.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The isotope parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        else:
            mol = copy.deepcopy(mol)
        # Replace isotopes with common weight
        for atom in mol.GetAtoms():
            atom.SetIsotope(0)
        return mol

def
isotope_parent
(
self
,
mol
,
skip_standardize
=
False
)
:
if
not
skip_standardize
:
mol
=
self
.
standardize
(
mol
)
else
:
mol
=
copy
.
deepcopy
(
mol
)
# Replace isotopes with common weight
for
atom
in
mol
.
GetAtoms
(
)
:
atom
.
SetIsotope
(
0
)
return
mol

Return the isotope parent of a given molecule .


def charge_parent(self, mol, skip_standardize=False):
        """Return the charge parent of a given molecule.

        The charge parent is the uncharged version of the fragment parent.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The charge parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        # TODO: All ionized acids and bases should be neutralised.
        if not skip_standardize:
            mol = self.standardize(mol)
        fragment = self.fragment_parent(mol, skip_standardize=True)
        if fragment:
            uncharged = self.uncharge(fragment)
            # During final standardization, the Reionizer ensures any remaining charges are in the right places
            uncharged = self.standardize(uncharged)
            return uncharged

Return the charge parent of a given molecule .


def super_parent(self, mol, skip_standardize=False):
        """Return the super parent of a given molecule.

        THe super parent is fragment, charge, isotope, stereochemistry and tautomer insensitive. From the input
        molecule, the largest fragment is taken. This is uncharged and then isotope and stereochemistry information is
        discarded. Finally, the canonical tautomer is determined and returned.

        :param mol: The input molecule.
        :type mol: rdkit.Chem.rdchem.Mol
        :param bool skip_standardize: Set to True if mol has already been standardized.
        :returns: The super parent molecule.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        if not skip_standardize:
            mol = self.standardize(mol)
        # We don't need to get fragment parent, because the charge parent is the largest fragment
        mol = self.charge_parent(mol, skip_standardize=True)
        mol = self.isotope_parent(mol, skip_standardize=True)
        mol = self.stereo_parent(mol, skip_standardize=True)
        mol = self.tautomer_parent(mol, skip_standardize=True)
        mol = self.standardize(mol)
        return mol

Return the super parent of a given molecule .


def main():
    """Main function for molvs command line interface."""

    # Root options
    parser = MolvsParser(epilog='use "molvs <command> -h" to show help for a specific command')
    subparsers = parser.add_subparsers(title='Available commands')

    # Options common to all commands

    common_parser = MolvsParser(add_help=False)
    common_parser.add_argument('infile', nargs='?', help='input filename', type=argparse.FileType('r'), default=sys.stdin)
    common_parser.add_argument('-i', '--intype', help='input filetype', choices=FILETYPES)
    common_parser.add_argument('-:', '--smiles', help='input SMILES instead of file', metavar='<smiles>')
    common_parser.add_argument('-O', '--outfile', help='output filename', type=argparse.FileType('w'), default=sys.stdout, metavar='<outfile>')

    # Standardize options
    standardize_parser = subparsers.add_parser('standardize', help='standardize a molecule', parents=[common_parser])
    standardize_parser.add_argument('-o', '--outtype', help='output filetype', choices=FILETYPES)
    standardize_parser.set_defaults(func=standardize_main)

    # Validate options
    validate_parser = subparsers.add_parser('validate', help='validate a molecule', parents=[common_parser])
    validate_parser.set_defaults(func=validate_main)

    args = parser.parse_args()
    try:
        args.func(args)
    except Exception as e:
        sys.stderr.write('Error: %s\n\n'.encode() % e.message)
        parser.print_help()
        sys.exit(2)

Main function for molvs command line interface .


def remove(self, mol):
        """Return the molecule with specified fragments removed.

        :param mol: The molecule to remove fragments from.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The molecule with fragments removed.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        log.debug('Running FragmentRemover')
        # Iterate FragmentPatterns and remove matching fragments
        for frag in self.fragments:
            # If nothing is left or leave_last and only one fragment, end here
            if mol.GetNumAtoms() == 0 or (self.leave_last and len(Chem.GetMolFrags(mol)) <= 1):
                break
            # Apply removal for this FragmentPattern
            removed = Chem.DeleteSubstructs(mol, frag.smarts, onlyFrags=True)
            if not mol.GetNumAtoms() == removed.GetNumAtoms():
                log.info('Removed fragment: %s', frag.name)
            if self.leave_last and removed.GetNumAtoms() == 0:
                # All the remaining fragments match this pattern - leave them all
                break
            mol = removed
        return mol

Return the molecule with specified fragments removed .


def choose(self, mol):
        """Return the largest covalent unit.

        The largest fragment is determined by number of atoms (including hydrogens). Ties are broken by taking the
        fragment with the higher molecular weight, and then by taking the first alphabetically by SMILES if needed.

        :param mol: The molecule to choose the largest fragment from.
        :type mol: rdkit.Chem.rdchem.Mol
        :return: The largest fragment.
        :rtype: rdkit.Chem.rdchem.Mol
        """
        log.debug('Running LargestFragmentChooser')
        # TODO: Alternatively allow a list of fragments to be passed as the mol parameter
        fragments = Chem.GetMolFrags(mol, asMols=True)
        largest = None
        for f in fragments:
            smiles = Chem.MolToSmiles(f, isomericSmiles=True)
            log.debug('Fragment: %s', smiles)
            organic = is_organic(f)
            if self.prefer_organic:
                # Skip this fragment if not organic and we already have an organic fragment as the largest so far
                if largest and largest['organic'] and not organic:
                    continue
                # Reset largest if it wasn't organic and this fragment is organic
                if largest and organic and not largest['organic']:
                    largest = None
            # Count atoms
            atoms = 0
            for a in f.GetAtoms():
                atoms += 1 + a.GetTotalNumHs()
            # Skip this fragment if fewer atoms than the largest
            if largest and atoms < largest['atoms']:
                continue
            # Skip this fragment if equal number of atoms but weight is lower
            weight = rdMolDescriptors.CalcExactMolWt(f)
            if largest and atoms == largest['atoms'] and weight < largest['weight']:
                continue
            # Skip this fragment if equal atoms and equal weight but smiles comes last alphabetically
            if largest and atoms == largest['atoms'] and weight == largest['weight'] and smiles > largest['smiles']:
                continue
            # Otherwise this is the largest so far
            log.debug('New largest fragment: %s (%s)', smiles, atoms)
            largest = {'smiles': smiles, 'fragment': f, 'atoms': atoms, 'weight': weight, 'organic': organic}
        return largest['fragment']


def integrate_ivp(u0=1.0, v0=0.0, mu=1.0, tend=10.0, dt0=1e-8, nt=0,
                  nsteps=600, t0=0.0, atol=1e-8, rtol=1e-8, plot=False,
                  savefig='None', method='bdf', dpi=100, verbose=False):
    """
    Example program integrating an IVP problem of van der Pol oscillator
    """
    f, j = get_f_and_j(mu)
    if nt > 1:
        tout = np.linspace(t0, tend, nt)
        yout, nfo = integrate_predefined(
            f, j, [u0, v0], tout, dt0, atol, rtol, nsteps=nsteps,
            check_indexing=False, method=method)
    else:
        tout, yout, nfo = integrate_adaptive(
            f, j, [u0, v0], t0, tend, dt0, atol, rtol, nsteps=nsteps,
            check_indexing=False, method=method)  # dfdt[:] also for len == 1
    if verbose:
        print(nfo)
    if plot:
        import matplotlib.pyplot as plt
        plt.plot(tout, yout[:, 1], 'g--')
        plt.plot(tout, yout[:, 0], 'k-', linewidth=2)
        if savefig == 'None':
            plt.show()
        else:
            plt.savefig(savefig, dpi=dpi)

def
integrate_ivp
(
u0
=
1.0
,
v0
=
0.0
,
mu
=
1.0
,
tend
=
10.0
,
dt0
=
1e-8
,
nt
=
0
,
nsteps
=
600
,
t0
=
0.0
,
atol
=
1e-8
,
rtol
=
1e-8
,
plot
=
False
,
savefig
=
'None'
,
method
=
'bdf'
,
dpi
=
100
,
verbose
=
False
)
:
f
,
j
=
get_f_and_j
(
mu
)
if
nt
>
1
:
tout
=
np
.
linspace
(
t0
,
tend
,
nt
)
yout
,
nfo
=
integrate_predefined
(
f
,
j
,
[
u0
,
v0
]
,
tout
,
dt0
,
atol
,
rtol
,
nsteps
=
nsteps
,
check_indexing
=
False
,
method
=
method
)
else
:
tout
,
yout
,
nfo
=
integrate_adaptive
(
f
,
j
,
[
u0
,
v0
]
,
t0
,
tend
,
dt0
,
atol
,
rtol
,
nsteps
=
nsteps
,
check_indexing
=
False
,
method
=
method
)
# dfdt[:] also for len == 1
if
verbose
:
print
(
nfo
)
if
plot
:
import
matplotlib
.
pyplot
as
plt
plt
.
plot
(
tout
,
yout
[
:
,
1
]
,
'g--'
)
plt
.
plot
(
tout
,
yout
[
:
,
0
]
,
'k-'
,
linewidth
=
2
)
if
savefig
==
'None'
:
plt
.
show
(
)
else
:
plt
.
savefig
(
savefig
,
dpi
=
dpi
)