#!/usr/bin/env python

'''

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

import sregistry
import argparse
import sys
import os

def get_parser():
    parser = argparse.ArgumentParser(description="Singularity Registry tools")

    # Customize parser depending on client
    from sregistry.main import Client as cli

    # Global Variables
    parser.add_argument('--debug', dest="debug",
                        help="use verbose logging to debug.",
                        default=False, action='store_true')

    parser.add_argument('--quiet', dest="quiet",
                        help="suppress additional output.",
                        default=False, action='store_true')

    description = 'actions for Singularity Registry Global Client'
    subparsers = parser.add_subparsers(help='sregistry actions',
                                       title='actions',
                                       description=description,
                                       dest="command")

    # print version and exit
    version = subparsers.add_parser("version",
                                    help="show software version")

    # Backend controller
    backend = subparsers.add_parser("backend",
                                    help="list, remove, or activate a backend.")

    backend.add_argument("commands", nargs='*',
                         help='activate, deactivate, ls, status, delete, add/rm settings from a client',
                         type=str)

    backend.add_argument('--force','-f', dest="force",
                         help="force add if variable exists.",
                         default=False, action='store_true')

    # Local shell with client loaded
    shell = subparsers.add_parser("shell",
                                  help="shell into a session a client.")

    shell.add_argument("endpoint", nargs='?',
                       help="the endpoint to use (eg. docker)",
                       type=str, default=None)

    # List local containers and collections
    if hasattr(cli, 'images'):
        images = subparsers.add_parser("images",
                                       help="list local images, optionally with query")

        images.add_argument("query", nargs='*',
                            help="container search query",
                            type=str, default="*")

    # List local containers and collections
    inspect = subparsers.add_parser("inspect",
                                    help="inspect a container in your database")

    inspect.add_argument("query", nargs=1,
                          help="container search query to inspect",
                          type=str, default=None)

    # Get path to an image
    get = subparsers.add_parser("get",
                                    help="get a container path from your storage")

    get.add_argument("query", nargs='*',
                     help="container search query to inspect",
                     type=str, default="*")

    # Add/copy local containers to storage, if client has it
    if hasattr(cli,'storage'):

        # Add an image file
        add = subparsers.add_parser("add",
                                    help="add a container to local storage")

        add.add_argument("image", nargs=1,
                         help="full path to image file",
                         type=str)

        add.add_argument("--name", dest='name',
                         help='name of image, in format "library/image"',
                         type=str)

        add.add_argument('--copy', dest="copy",
                         help="copy the container instead of moving it.",
                         default=False, action='store_true')

        mv = subparsers.add_parser("mv",
                                   help="move a container and update database")

        mv.add_argument("name", nargs=1,
                         help="container name or uri to move from database",
                         type=str)

        mv.add_argument("path", nargs=1,
                         help="directory or image file to move image.",
                         type=str)

        rename = subparsers.add_parser("rename",
                                        help="rename a container in storage")

        rename.add_argument("name", nargs=1,
                            help="container name or uri to rename in database",
                            type=str)

        rename.add_argument("path", nargs=1,
                             help="path to rename image (will use basename)",
                             type=str)

        rm = subparsers.add_parser("rm",
                                   help="remove a container from the database")

        rm.add_argument("image", nargs=1,
                        help='name of image, in format "library/image"',
                        type=str)

        rmi = subparsers.add_parser("rmi",
                                    help="remove a container from the database AND storage")

        rmi.add_argument("image", nargs=1,
                         help='name of image, in format "library/image"',
                         type=str)

    # List or search containers and collections
    if hasattr(cli,'search'):

        search = subparsers.add_parser("search",
                                   help="search remote containers")

        search.add_argument("query", nargs='*',
                            help="container search query, don't specify for all",
                            type=str, default="*")

        search.add_argument("--endpoint", default=None, dest='endpoint',
                            help='remote endpoint and path. id:/path"',
                            type=str)

    # Build an image
    if hasattr(cli,'build'):

        build = subparsers.add_parser("build",
                                     help="build a container using a remote.")

        build.add_argument('--preview','-p', dest="preview",
                           help="preview the parsed configuration file only.",
                           default=False, action='store_true')

        build.add_argument("commands", nargs='*',
                           help='''Google Build + Storage
                                   --------------------------------------------------------
                                   build [recipe] [context] -------------------------------
                                   build [recipe] . ---------------------------------------
                                   build [recipe] file1 file2 -----------------------------
                                   ''',
                           type=str)

        build.add_argument("--name", dest='name',
                           help='name of image, in format "library/image"',
                           type=str, default=None)

        build.add_argument("--outfile", dest='outfile',
                           help='name of output file to write contents to',
                           type=str, default=None)

    # Push an image
    if hasattr(cli,'push'):

        push = subparsers.add_parser("push",
                                     help="push one or more images to a registry")

        push.add_argument("image", nargs=1,
                           help="full path to image file",
                           type=str)

        push.add_argument("--tag", dest='tag',
                           help="tag for image. If not provided, defaults to latest",
                           type=str, default=None)

        push.add_argument("--name", dest='name',
                           help='name of image, in format "library/image"',
                           type=str, required=True)

    # Share an image
    if hasattr(cli,'share'):

        share = subparsers.add_parser("share",
                                       help="share a remote image")

        share.add_argument("image", nargs=1,
                           help="full uri of image",
                           type=str)

        share.add_argument("--email", dest='share_to',
                            help='email (or share point) to share with',
                            type=str, default=None)

    # Pull an image
    if hasattr(cli,'pull'):

        pull = subparsers.add_parser("pull",
                                     help="pull an image from a registry")

        pull.add_argument("image", nargs=1,
                           help="full uri of image",
                           type=str)

        pull.add_argument("--name", dest='name',
                           help='custom name for image',
                           type=str, default=None)

        pull.add_argument('--force','-f', dest="force",
                            help="force overwrite of existing image",
                            default=False, action='store_true')

        pull.add_argument('--no-cache', dest="nocache",
                           help="if storage active, don't add the image to it",
                           default=False, action='store_true')

    # List or search labels
    if hasattr(cli,'label_search'):

        labels = subparsers.add_parser("labels",
                                    help="query for labels")

        labels.add_argument("--key", "-k", dest='key',
                            help="A label key to search for",
                            type=str, default=None)

        labels.add_argument("--value", "-v", dest='value',
                            help="A value to search for",
                            type=str, default=None)

    # List or search labels
    if hasattr(cli,'remove'):

        # Remove
        delete = subparsers.add_parser("delete",
                                        help="delete an image from the registry.")

        delete.add_argument('--force','-f', dest="force",
                            help="don't prompt before deletion",
                            default=False, action='store_true')

        delete.add_argument("image", nargs=1,
                            help="full path to image file",
                            type=str)

    return parser

def get_subparsers(parser):
    '''get_subparser will get a dictionary of subparsers, to help with printing help
    '''

    actions = [action for action in parser._actions
               if isinstance(action, argparse._SubParsersAction)]

    subparsers = dict()
    for action in actions:
        # get all subparsers and print help
        for choice, subparser in action.choices.items():
            subparsers[choice] = subparser

    return subparsers

def main():
    '''main is the entrypoint to the sregistry client. The flow works to first
    to determine the subparser in use based on the command. The command then
    imports the correct main (files imported in this folder) associated with
    the action of choice. When the client is imported, it is actually importing
    a return of the function get_client() under sregistry/main, which plays
    the job of "sniffing" the environment to determine what flavor of client
    the user wants to activate. Installed within a singularity image, this
    start up style maps well to Standard Container Integration Format (SCIF)
    apps, where each client is a different entrypoint activated based on the
    environment variables.
    '''

    from sregistry.main import Client as cli
    parser = get_parser()
    subparsers = get_subparsers(parser)

    def help(return_code=0):
        '''print help, including the software version and active client
           and exit with return code.
        '''

        version = sregistry.__version__
        name = cli.client_name

        print("\nSingularity Registry Global Client v%s [%s]" %(version, name))
        parser.print_help()
        sys.exit(return_code)

    # If the user didn't provide any arguments, show the full help
    if len(sys.argv) == 1:
        help()
    try:
        args = parser.parse_args()
    except:
        sys.exit(0)

    if args.debug is False:
        os.environ['MESSAGELEVEL'] = "DEBUG"

    # Show the version and exit
    if args.command == "version":
        print(sregistry.__version__)
        sys.exit(0)

    from sregistry.logger import bot

    # Does the user want a shell?
    if args.command == "add": from .add import main
    elif args.command == "backend": from .backend import main
    elif args.command == "build": from .build import main
    elif args.command == "get": from .get import main
    elif args.command == "delete": from .delete import main
    elif args.command == "inspect": from .inspect import main
    elif args.command == "images": from .images import main
    elif args.command == "labels": from .labels import main
    elif args.command == "mv": from .mv import main
    elif args.command == "push": from .push import main
    elif args.command == "pull": from .pull import main
    elif args.command == "rename": from .rename import main
    elif args.command == "rm": from .rm import main
    elif args.command == "rmi": from .rmi import main
    elif args.command == "search": from .search import main
    elif args.command == "share": from .share import main
    elif args.command == "shell": from .shell import main

    # Pass on to the correct parser
    return_code = 0
    try:
        main(args=args,
             parser=parser,
             subparser=subparsers[args.command])
        sys.exit(return_code)
    except UnboundLocalError:
        return_code = 1

    help(return_code)

if __name__ == '__main__':
    main()


#!/usr/bin/env python

'''

Copyright (C) 2018-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from random import choice

class RobotNamer:

    _descriptors = [
        'chunky', 'buttery', 'delicious', 'scruptious', 'dinosaur', 'boopy',
        'lovely', 'carniverous', 'hanky', 'loopy', 'doopy', 'astute', 'gloopy',
        'outstanding', 'stinky', 'conspicuous', 'fugly', 'frigid', 'angry',
        'adorable', 'sticky', 'moolicious', 'cowy', 'spicy', 'grated', 'crusty',
        'stanky', 'blank', 'bumfuzzled', 'fuzzy', 'hairy', 'peachy', 'tart',
        'creamy', 'arid', 'strawberry', 'butterscotch', 'wobbly', 'persnickety',
        'nerdy', 'dirty', 'placid', 'bloated', 'swampy', 'pusheena', 'hello',
        'goodbye', 'milky', 'purple', 'rainbow', 'bricky', 'muffled', 'anxious',
        'misunderstood', 'eccentric', 'quirky', 'lovable', 'reclusive', 'faux',
        'evasive', 'confused', 'crunchy', 'expensive', 'ornery', 'fat', 'phat',
        'joyous', 'expressive', 'psycho', 'chocolate', 'salted', 'gassy', 'red',
        'blue'
    ]

    _nouns = [
        'squidward', 'hippo', 'butter', 'animal', 'peas', 'lettuce', 'carrot',
        'onion', 'peanut', 'cupcake', 'muffin', 'buttface', 'leopard', 'parrot',
        'parsnip', 'poodle', 'itch', 'punk', 'kerfuffle', 'soup', 'noodle',
        'avocado', 'peanut-butter', 'latke', 'milkshake', 'banana', 'lizard',
        'lemur', 'lentil', 'bits', 'house', 'leader', 'toaster', 'signal',
        'pancake', 'kitty', 'cat', 'cattywampus', 'poo', 'malarkey',
        'general', 'rabbit', 'chair', 'staircase', 'underoos', 'snack', 'lamp',
        'eagle', 'hobbit', 'diablo', 'earthworm', 'pot', 'plant', 'leg', 'arm',
        'bike', 'citrus', 'dog', 'puppy', 'blackbean', 'ricecake', 'gato',
        'nalgas', 'lemon', 'caramel', 'fudge', 'cherry', 'sundae', 'truffle',
        'cinnamonbun', 'pastry', 'egg', 'omelette', 'fork', 'knife', 'spoon',
        'salad', 'train', 'car', 'motorcycle', 'bicycle', 'platanos', 'mango',
        'taco', 'pedo', 'nunchucks', 'destiny', 'hope', 'despacito', 'frito',
        'chip'
    ]

    def generate(self, delim='-', length=4, chars='0123456789'):
        '''
        Generate a robot name. Inspiration from Haikunator, but much more
                 poorly implemented ;)

        Parameters
        ==========
        delim: Delimiter
        length: TokenLength
        chars: TokenChars
        '''

        descriptor = self._select(self._descriptors)
        noun = self._select(self._nouns)
        numbers = ''.join((self._select(chars) for _ in range(length)))
        return delim.join([descriptor, noun, numbers])

    def _select(self, select_from):
        ''' select an element from a list using random.choice

            Parameters
            ==========
            should be a list of things to select from
        '''
        if len(select_from) <= 0:
            return ''

        return choice(select_from)

def main():
    bot = RobotNamer()
    print(bot.generate())

if __name__ == '__main__':
    main()


'''

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

import datetime
import hashlib
import errno
import os
import pwd
import re
import shutil
import tempfile
import tarfile
import requests

import json
from sregistry.utils import get_installdir
from sregistry.logger import bot
import io
import sys

################################################################################
## FOLDER OPERATIONS ###########################################################
################################################################################

def mkdir_p(path):
    '''mkdir_p attempts to get the same functionality as mkdir -p
    :param path: the path to create.
    '''
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            bot.error("Error creating path %s, exiting." % path)
            sys.exit(1)

def get_tmpfile(requested_tmpdir=None, prefix=""):
    '''get a temporary file with an optional prefix. By default will be
       created in /tmp unless SREGISTRY_TMPDIR is set. By default, the file
       is closed (and just a name returned).

       Parameters
       ==========
       requested_tmpdir: an optional requested temporary directory, first
       priority as is coming from calling function.
       prefix: Given a need for a sandbox (or similar), prefix the file
       with this string.
    '''

    # First priority for the base goes to the user requested.
    tmpdir = get_tmpdir(requested_tmpdir)

    # If tmpdir is set, add to prefix
    if tmpdir is not None:
        prefix = os.path.join(tmpdir, os.path.basename(prefix))

    fd, tmp_file = tempfile.mkstemp(prefix=prefix)
    os.close(fd)

    return tmp_file

def get_tmpdir(requested_tmpdir=None, prefix="", create=True):
    '''get a temporary directory for an operation. If SREGISTRY_TMPDIR
       is set, return that. Otherwise, return the output of tempfile.mkdtemp

       Parameters
       ==========
       requested_tmpdir: an optional requested temporary directory, first
       priority as is coming from calling function.
       prefix: Given a need for a sandbox (or similar), we will need to
       create a subfolder *within* the SREGISTRY_TMPDIR.
       create: boolean to determine if we should create folder (True)
    '''
    from sregistry.defaults import SREGISTRY_TMPDIR

    # First priority for the base goes to the user requested.
    tmpdir = requested_tmpdir or SREGISTRY_TMPDIR

    prefix = prefix or "sregistry-tmp"
    prefix = "%s.%s" %(prefix, next(tempfile._get_candidate_names()))
    tmpdir = os.path.join(tmpdir, prefix)

    if not os.path.exists(tmpdir) and create is True:
        os.mkdir(tmpdir)

    return tmpdir

def get_userhome():
    '''get the user home based on the effective uid
    '''
    return pwd.getpwuid(os.getuid())[5]

################################################################################
## COMPRESSION #################################################################
################################################################################

def extract_tar(archive, output_folder, handle_whiteout=False):
    '''extract a tar archive to a specified output folder

        Parameters
        ==========
        archive: the archive file to extract
        output_folder: the output folder to extract to
        handle_whiteout: use docker2oci variation to handle whiteout files

    '''
    from .terminal import run_command

    # Do we want to remove whiteout files?
    if handle_whiteout is True:
        return _extract_tar(archive, output_folder)

    # If extension is .tar.gz, use -xzf
    args = '-xf'
    if archive.endswith(".tar.gz"):
        args = '-xzf'

    # Just use command line, more succinct.
    command = ["tar", args, archive, "-C", output_folder, "--exclude=dev/*"]
    if not bot.is_quiet():
        print("Extracting %s" % archive)

    return run_command(command)

def _extract_tar(archive, output_folder):
    '''use blob2oci to handle whiteout files for extraction. Credit for this
       script goes to docker2oci by Olivier Freyermouth, and see script
       folder for license.

       Parameters
       ==========
       archive: the archive to extract
       output_folder the output folder (sandbox) to extract to

    '''
    from .terminal import ( run_command, which )

    result = which('blob2oci')
    if result['return_code'] != 0:
        bot.error('Cannot find blob2oci script on path, exiting.')
        sys.exit(1)

    script = result['message']
    command = ['exec' ,script, '--layer', archive, '--extract', output_folder]

    if not bot.is_quiet():
        print("Extracting %s" % archive)

    return run_command(command)

def create_tar(files, output_folder=None):
    '''create_memory_tar will take a list of files (each a dictionary
        with name, permission, and content) and write the tarfile
        (a sha256 sum name is used) to the output_folder.
        If there is no output folde specified, the
        tar is written to a temporary folder.
    '''
    if output_folder is None:
        output_folder = tempfile.mkdtemp()

    finished_tar = None
    additions = []
    contents = []

    for entity in files:
        info = tarfile.TarInfo(name=entity['name'])
        info.mode = entity['mode']
        info.mtime = int(datetime.datetime.now().strftime('%s'))
        info.uid = entity["uid"]
        info.gid = entity["gid"]
        info.uname = entity["uname"]
        info.gname = entity["gname"]

        # Get size from stringIO write
        filey = io.StringIO()
        content = None
        try:  # python3
            info.size = filey.write(entity['content'])
            content = io.BytesIO(entity['content'].encode('utf8'))
        except Exception:  # python2
            info.size = int(filey.write(entity['content'].decode('utf-8')))
            content = io.BytesIO(entity['content'].encode('utf8'))
        pass

        if content is not None:
            addition = {'content': content,
                        'info': info}
            additions.append(addition)
            contents.append(content)

    # Now generate the sha256 name based on content
    if len(additions) > 0:
        hashy = get_content_hash(contents)
        finished_tar = "%s/sha256:%s.tar.gz" % (output_folder, hashy)

        # Warn the user if it already exists
        if os.path.exists(finished_tar):
            msg = "metadata file %s already exists " % finished_tar
            msg += "will over-write."
            bot.debug(msg)

        # Add all content objects to file
        tar = tarfile.open(finished_tar, "w:gz")
        for a in additions:
            tar.addfile(a["info"], a["content"])
        tar.close()

    else:
        msg = "No contents, environment or labels"
        msg += " for tarfile, will not generate."
        bot.debug(msg)

    return finished_tar

def get_content_hash(contents):
    '''get_content_hash will return a hash for a list of content (bytes/other)
    '''
    hasher = hashlib.sha256()
    for content in contents:
        if isinstance(content, io.BytesIO):
            content = content.getvalue()
        if not isinstance(content, bytes):
            content = bytes(content)
        hasher.update(content)
    return hasher.hexdigest()

def get_file_hash(filename):
    '''find the SHA256 hash string of a file
    '''
    hasher = hashlib.sha256()
    with open(filename, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hasher.update(chunk)
    return hasher.hexdigest()

################################################################################
## FILE OPERATIONS #############################################################
################################################################################

def copyfile(source, destination, force=True):
    '''copy a file from a source to its destination.
    '''
    # Case 1: It's already there, we aren't replacing it :)
    if source == destination and force is False:
        return destination

    # Case 2: It's already there, we ARE replacing it :)
    if os.path.exists(destination) and force is True:
        os.remove(destination)

    shutil.copyfile(source, destination)
    return destination

def write_file(filename, content, mode="w"):
    '''write_file will open a file, "filename" and write content, "content"
       and properly close the file
    '''
    with open(filename, mode) as filey:
        filey.writelines(content)
    return filename

def write_json(json_obj, filename, mode="w", print_pretty=True):
    '''write_json will (optionally,pretty print) a json object to file

       Parameters
       ==========
       json_obj: the dict to print to json
       filename: the output file to write to
       pretty_print: if True, will use nicer formatting
    '''
    with open(filename, mode) as filey:
        if print_pretty:
            filey.writelines(print_json(json_obj))
        else:
            filey.writelines(json.dumps(json_obj))
    return filename

def print_json(json_obj):
    ''' just dump the json in a "pretty print" format
    '''
    return json.dumps(
                    json_obj,
                    indent=4,
                    separators=(
                        ',',
                        ': '))

def read_file(filename, mode="r", readlines=True):
    '''write_file will open a file, "filename" and write content, "content"
       and properly close the file
    '''
    with open(filename, mode) as filey:
        if readlines is True:
            content = filey.readlines()
        else:
            content = filey.read()
    return content

def read_json(filename, mode='r'):
    '''read_json reads in a json file and returns
       the data structure as dict.
    '''
    with open(filename, mode) as filey:
        data = json.load(filey)
    return data

def clean_up(files):
    '''clean up will delete a list of files, only if they exist
    '''
    if not isinstance(files, list):
        files = [files]

    for f in files:
        if os.path.exists(f):
            bot.verbose3("Cleaning up %s" % f)
            os.remove(f)


'''

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
from spython.main import Client as Singularity
from sregistry.utils import (
    get_tmpdir,
    parse_image_name,
    remove_uri,
    extract_tar
)
import shutil
import os
import re
import sys

def pull(self, images, file_name=None, save=True, force=False, **kwargs):
    '''pull an image from a docker hub. This is a (less than ideal) workaround
       that actually does the following:

       - creates a sandbox folder
       - adds docker layers, metadata folder, and custom metadata to it
       - converts to a squashfs image with build

    the docker manifests are stored with registry metadata.

    Parameters
    ==========
    images: refers to the uri given by the user to pull in the format
    <collection>/<namespace>. You should have an API that is able to
    retrieve a container based on parsing this uri.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add()

    Returns
    =======
    finished: a single container path, or list of paths
    '''

    if not isinstance(images,list):
        images = [images]

    bot.debug('Execution of PULL for %s images' %len(images))

    # If used internally we want to return a list to the user.

    finished = []
    for image in images:

        q = parse_image_name( remove_uri(image),
                              default_collection='aws' )

        image_file = self._pull(file_name=file_name,
                                save=save,
                                force=force,
                                names=q,
                                kwargs=kwargs)

        finished.append(image_file)

    if len(finished) == 1:
        finished = finished[0]
    return finished

def _pull(self,
          file_name,
          names,
          save=True,
          force=False,
          **kwargs):

    '''pull an image from aws. This is a (less than ideal) workaround
       that actually does the following:

       - creates a sandbox folder
       - adds docker layers from S3
       - converts to a squashfs image with build

    Parameters
    ==========
    images: refers to the uri given by the user to pull in the format
    <collection>/<namespace>. You should have an API that is able to
    retrieve a container based on parsing this uri.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add()

    Returns
    =======
    finished: a single container path, or list of paths
    '''

    # Use Singularity to build the image, based on user preference
    if file_name is None:
        file_name = self._get_storage_name(names)

    # Determine if the user already has the image
    if os.path.exists(file_name) and force is False:
        bot.error('Image exists! Remove first, or use --force to overwrite')
        sys.exit(1)

    digest = names['version'] or names['tag']

    # Build from sandbox
    sandbox = get_tmpdir(prefix="sregistry-sandbox")

    # First effort, get image via Sregistry
    layers, url = self._download_layers(names['url'], digest)

    # Add environment to the layers
    envtar = self._get_environment_tar()
    layers = [envtar] + layers

    # Create singularity image from an empty folder
    for layer in layers:
        bot.info('Exploding %s' %layer)
        result = extract_tar(layer, sandbox, handle_whiteout=True)
        if result['return_code'] != 0:
            bot.error(result['message'])
            sys.exit(1)

    sudo = kwargs.get('sudo', False)

    # Build from a sandbox (recipe) into the image_file (squashfs)
    image_file = Singularity.build(image=file_name,
                                   recipe=sandbox,
                                   sudo=sudo)

    # Fall back to using Singularity
    if image_file is None:
        bot.info('Downloading with native Singularity, please wait...')
        image = image.replace('aws://', 'docker://')
        image_file = Singularity.pull(image, pull_folder=sandbox)

    # Save to local storage
    if save is True:

        # Did we get the manifests?
        manifests = {}
        if hasattr(self, 'manifest'):
            manifest = self.manifest

        container = self.add(image_path = image_file,
                             image_uri = names['uri'],
                             metadata = manifest,
                             url = url)

        # When the container is created, this is the path to the image
        image_file = container.image

    if os.path.exists(image_file):
        bot.debug('Retrieved image file %s' %image_file)
        bot.custom(prefix="Success!", message=image_file)

    # Clean up sandbox
    shutil.rmtree(sandbox)

    return image_file


'''

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
from sregistry.utils import ( parse_image_name, remove_uri )
import os

def pull(self, images, file_name=None, save=True, **kwargs):
    '''pull an image from a singularity registry

    Parameters
    ==========
    images: refers to the uri given by the user to pull in the format
    <collection>/<namespace>. You should have an API that is able to
    retrieve a container based on parsing this uri.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add()

    Returns
    =======
    finished: a single container path, or list of paths
    '''

    # Here we take an entire list or a single image by ensuring we have a list
    # This makes the client flexible to command line or internal function use,
    # for one or more images.
    if not isinstance(images,list):
        images = [images]

    bot.debug('Execution of PULL for %s images' %len(images))

    # If used internally we want to return a list to the user.
    finished = []
    for image in images:

        q = parse_image_name(remove_uri(image))

        # Verify image existence, and obtain id
        url = "..." # write your custom endpoint URL here
        bot.debug('Retrieving manifest at %s' %url)

        # You can use the client get function to retrieve a url manifest
        manifest = self._get(url)

        # it's good practice to add the url as a `selfLink`
        manifest['selfLink'] = url

        # Make sure to parse the response (manifest) in case it's not what
        # you expect!

        # If the user didn't provide a file, make one based on the names
        if file_name is None:
            file_name = q['storage'].replace('/','-')

        # You can then use the client download function to get the url
        # for some image in your manifest. In this example, it's in the `image`
        # field and we want to show the progress bar.
        image_file = self.download(url=manifest['image'],
                                   file_name=file_name,
                                   show_progress=True)

        # If the user is saving to local storage, you need to assumble the uri
        # here in the expected format <collection>/<namespace>:<tag>@<version>
        if save is True:
            image_uri = "%s/%s:%s" %(manifest['collection'],
                                     manifest['name'],
                                     manifest['tag'])

            # Importantly, the client add function will take the image file, the
            # uri, the download link, and any relevant metadata (dictionary)
            # for the database
            container = self.add(image_path = image_file, # the file path
                                 image_uri = image_uri,   # the full uri
                                 image_name = file_name, # a custom name?
                                 metadata = manifest,
                                 url = manifest['image'])

            # When the container is created, this is the path to the image
            image_file = container.image

        if os.path.exists(image_file):
            bot.debug('Retrieved image file %s' %image_file)
            bot.custom(prefix="Success!", message=image_file)
            finished.append(image_file)

    if len(finished) == 1:
        finished = finished[0]
    return finished


'''

Copyright (C) 2018-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
from sregistry.utils import (
    parse_image_name,
    remove_uri
)

import sys
import os

def push(self, path, name, tag=None):
    '''push an image to an S3 endpoint'''

    path = os.path.abspath(path)
    image = os.path.basename(path)
    bot.debug("PUSH %s" % path)

    if not os.path.exists(path):
        bot.error('%s does not exist.' %path)
        sys.exit(1)

    # Extract the metadata
    names = parse_image_name(remove_uri(name), tag=tag)
    image_size = os.path.getsize(path) >> 20

    # Create extra metadata, this is how we identify the image later
    # *important* bug in boto3 will return these capitalized
    # see https://github.com/boto/boto3/issues/1709
    metadata = {'sizemb': "%s" % image_size,
                'client': 'sregistry' }

    self.bucket.upload_file(path, names['storage_uri'], {"Metadata": metadata })


'''

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
from sregistry.utils import (
    check_install,
    copyfile,
    get_image_hash,
    parse_image_name,
    remove_uri
)
from sqlalchemy import or_
from glob import glob
import os
import json
import shutil
import sys

# COLLECTIONS ##################################################################

def get_or_create_collection(self, name):
    '''get a collection if it exists. If it doesn't exist, create it first.

    Parameters
    ==========
    name: the collection name, usually parsed from get_image_names()['name']

    '''
    from sregistry.database.models import Collection
    collection = self.get_collection(name)

    # If it doesn't exist, create it
    if collection is None:
        collection = Collection(name=name)
        self.session.add(collection)
        self.session.commit()

    return collection

def get_collection(self, name):
    '''get a collection, if it exists, otherwise return None.
    '''
    from sregistry.database.models import Collection
    return Collection.query.filter(Collection.name == name).first()

def get_container(self, name, collection_id, tag="latest", version=None):
    '''get a container, otherwise return None.
    '''
    from sregistry.database.models import Container
    if version is None:
        container = Container.query.filter_by(collection_id = collection_id,
                                              name = name,
                                              tag = tag).first()
    else:
        container = Container.query.filter_by(collection_id = collection_id,
                                              name = name,
                                              tag = tag,
                                              version = version).first()
    return container

# ACTIONS ######################################################################

def get(self, name, quiet=False):
    '''Do a get for a container, and then a collection, and then return None
       if no result is found.
    Parameters
    ==========
    name: should coincide with either the collection name, or the container
          name with the collection. A query is done first for the collection,
          and then the container, and the path to the image file returned.
    '''
    from sregistry.database.models import Collection, Container
    names = parse_image_name( remove_uri (name) )

    # First look for a collection (required)
    collection = self.get_collection(name=names['collection'])
    container = None

    if collection is not None:
        container = self.get_container(collection_id=collection.id,
                                       name=names['image'],
                                       tag=names['tag'],
                                       version=names['version'])

        if container is not None and quiet is False:

            # The container image file exists [local]
            if container.image is not None:
                print(container.image)

            # The container has a url (but not local file)
            elif container.url is not None:
                print(container.url)
            else:
                bot.info('No storage file found for %s' %name)

    return container

def images(self, query=None):
    '''List local images in the database, optionally with a query.

       Paramters
       =========
       query: a string to search for in the container or collection name|tag|uri

    '''
    from sregistry.database.models import Collection, Container

    rows = []
    if query is not None:
        like = "%" + query + "%"
        containers = Container.query.filter(or_(Container.name == query,
                                                Container.tag.like(like),
                                                Container.uri.like(like),
                                                Container.name.like(like))).all()
    else:
        containers = Container.query.all()

    if len(containers) > 0:
        message = "  [date]   [client]\t[uri]"
        bot.custom(prefix='Containers:', message=message, color="RED")
        for c in containers:
            uri = c.get_uri()
            created_at = c.created_at.strftime('%B %d, %Y')
            rows.append([created_at, "   [%s]" %c.client, uri])
        bot.table(rows)
    return containers

def inspect(self, name):
    '''Inspect a local image in the database, which typically includes the
       basic fields in the model.

    '''
    print(name)
    container = self.get(name)
    if container is not None:
        collection = container.collection.name
        fields = container.__dict__.copy()
        fields['collection'] = collection
        fields['metrics'] = json.loads(fields['metrics'])
        del fields['_sa_instance_state']
        fields['created_at'] = str(fields['created_at'])
        print(json.dumps(fields, indent=4, sort_keys=True))
        return fields

def rename(self, image_name, path):
    '''rename performs a move, but ensures the path is maintained in storage

       Parameters
       ==========
       image_name: the image name (uri) to rename to.
       path: the name to rename (basename is taken)

    '''
    container = self.get(image_name, quiet=True)

    if container is not None:
        if container.image is not None:

            # The original directory for the container stays the same
            dirname = os.path.dirname(container.image)

            # But we derive a new filename and uri

            names = parse_image_name( remove_uri (path) )
            storage = os.path.join( self.storage,
                                    os.path.dirname(names['storage']) )

            # This is the collection folder

            if not os.path.exists(storage):
                os.mkdir(storage)

            # Here we get the new full path, rename the container file

            fullpath = os.path.abspath(os.path.join(dirname, names['storage']))
            container = self.cp(move_to=fullpath,
                                container=container,
                                command="rename")

            # On successful rename of file, update the uri

            if container is not None:
                container.uri = names['uri']
                self.session.commit()
                return container

    bot.warning('%s not found' %(image_name))

def mv(self, image_name, path):
    '''Move an image from it's current location to a new path.
       Removing the image from organized storage is not the recommended approach
       however is still a function wanted by some.

       Parameters
       ==========
       image_name: the parsed image name.
       path: the location to move the image to

    '''

    container = self.get(image_name, quiet=True)

    if container is not None:

        name = container.uri or container.get_uri()
        image = container.image or ''

        # Only continue if image file exists
        if os.path.exists(image):

            # Default assume directory, use image name and path fully
            filename = os.path.basename(image)
            filedir = os.path.abspath(path)

            # If it's a file, use filename provided
            if not os.path.isdir(path):
                filename = os.path.basename(path)
                filedir = os.path.dirname(path)

            # If directory is empty, assume $PWD
            if filedir == '':
                filedir = os.getcwd()

            # Copy to the fullpath from the storage
            fullpath = os.path.abspath(os.path.join(filedir,filename))
            return self.cp(move_to=fullpath,
                           container=container,
                           command="move")

    bot.warning('%s not found' %(image_name))

def cp(self, move_to, image_name=None, container=None, command="copy"):
    '''_cp is the shared function between mv (move) and rename, and performs
       the move, and returns the updated container

       Parameters
       ==========
       image_name: an image_uri to look up a container in the database
       container: the container object to move (must have a container.image
       move_to: the full path to move it to

    '''
    if container is None and image_name is None:
        bot.error('A container or image_name must be provided to %s' %command)
        sys.exit(1)

    # If a container isn't provided, look for it from image_uri
    if container is None:
        container = self.get(image_name, quiet=True)

    image = container.image or ''

    if os.path.exists(image):

        filedir = os.path.dirname(move_to)

        # If the two are the same, doesn't make sense
        if move_to == image:
            bot.warning('%s is already the name.' %image)
            sys.exit(1)

        # Ensure directory exists
        if not os.path.exists(filedir):
            bot.error('%s does not exist. Ensure exists first.' %filedir)
            sys.exit(1)

        # Ensure writable for user
        if not os.access(filedir, os.W_OK):
            bot.error('%s is not writable' %filedir)
            sys.exit(1)

        original = os.path.basename(image)

        try:
            shutil.move(image, move_to)
            container.image = move_to
            self.session.commit()
            bot.info('[%s] %s => %s' %(command, original, move_to))
            return container
        except:
            bot.error('Cannot %s %s to %s' %(command, original, move_to))
            sys.exit(1)

    bot.warning('''Not found! Please pull %s and then %s to the appropriate
                   location.''' %(container.uri, command))

def rmi(self, image_name):
    '''Remove an image from the database and filesystem.
    '''
    container = self.rm(image_name, delete=True)
    if container is not None:
        bot.info("[rmi] %s" % container)

def rm(self, image_name, delete=False):
    '''Remove an image from the database, akin to untagging the image. This
    does not delete the file from the cache, unless delete is set to True
    (as called by rmi).
    '''
    container = self.get(image_name)
    if container is not None:
        name = container.uri or container.get_uri()
        image = container.image
        self.session.delete(container)
        self.session.commit()
        if image is not None:
            if os.path.exists(image) and delete is True:
                os.remove(container.image)
            return image
        bot.info("[rm] %s" % name)

def add(self, image_path=None,
              image_uri=None,
              image_name=None,
              url=None,
              metadata=None,
              save=True,
              copy=False):

    '''get or create a container, including the collection to add it to.
    This function can be used from a file on the local system, or via a URL
    that has been downloaded. Either way, if one of url, version, or image_file
    is not provided, the model is created without it. If a version is not
    provided but a file path is, then the file hash is used.

    Parameters
    ==========
    image_path: full path to image file
    image_name: if defined, the user wants a custom name (and not based on uri)
    metadata: any extra metadata to keep for the image (dict)
    save: if True, move the image to the cache if it's not there
    copy: If True, copy the image instead of moving it.

    image_name: a uri that gets parsed into a names object that looks like:

    {'collection': 'vsoch',
     'image': 'hello-world',
     'storage': 'vsoch/hello-world-latest.img',
     'tag': 'latest',
     'version': '12345'
     'uri': 'vsoch/hello-world:latest@12345'}

    After running add, the user will take some image in a working
    directory, add it to the database, and have it available for search
    and use under SREGISTRY_STORAGE/<collection>/<container>

    If the container was retrieved from a webby place, it should have version
    If no version is found, the file hash is used.
    '''

    from sregistry.database.models import (
        Container,
        Collection
    )

    # We can only save if the image is provided
    if image_path is not None:
        if not os.path.exists(image_path) and save is True:
            bot.error('Cannot find %s' %image_path)
            sys.exit(1)

    # An image uri is required for version, tag, etc.
    if image_uri is None:
        bot.error('You must provide an image uri <collection>/<namespace>')
        sys.exit(1)

    names = parse_image_name( remove_uri(image_uri) )
    bot.debug('Adding %s to registry' % names['uri'])

    # If Singularity is installed, inspect image for metadata
    metadata = self.get_metadata(image_path, names=names)
    collection = self.get_or_create_collection(names['collection'])

    # Get a hash of the file for the version, or use provided
    version = names.get('version')
    if version == None:
        if image_path != None:
            version = get_image_hash(image_path)
        else:
            version = ''  # we can't determine a version, not in API/no file
        names = parse_image_name( remove_uri(image_uri), version=version )

    # If save, move to registry storage first
    if save is True and image_path is not None:

        # If the user hasn't defined a custom name
        if image_name is None:
            image_name = self._get_storage_name(names)

        if copy is True:
            copyfile(image_path, image_name)
        else:
            shutil.move(image_path, image_name)

        image_path = image_name

    # Just in case the client didn't provide it, see if we have in metadata
    if url is None and "url" in metadata:
        url = metadata['url']

    # First check that we don't have one already!
    container = self.get_container(name=names['image'],
                                   collection_id=collection.id,
                                   tag=names['tag'],
                                   version=version)

    # The container did not exist, create it
    if container is None:
        action = "new"
        container = Container(metrics=json.dumps(metadata),
                              name=names['image'],
                              image=image_path,
                              client=self.client_name,
                              tag=names['tag'],
                              version=version,
                              url=url,
                              uri=names['uri'],
                              collection_id=collection.id)

        self.session.add(container)
        collection.containers.append(container)

    # The container existed, update it.
    else:
        action="update"
        metrics=json.loads(container.metrics)
        metrics.update(metadata)
        container.url= url
        container.client=self.client_name
        if image_path is not None:
            container.image=image_path
        container.metrics=json.dumps(metrics)

    self.session.commit()
    bot.info("[container][%s] %s" % (action,names['uri']))
    return container


'''

push.py: push functions for sregistry client

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot, ProgressBar
from sregistry.utils import (
    parse_image_name,
    parse_header,
    remove_uri
)
from requests_toolbelt.streaming_iterator import StreamingIterator
from requests_toolbelt import (
    MultipartEncoder,
    MultipartEncoderMonitor
)

import requests
import hashlib
import sys
import os

def push(self, path, name, tag=None):
    '''push an image to Singularity Registry'''

    path = os.path.abspath(path)
    image = os.path.basename(path)
    bot.debug("PUSH %s" % path)

    if not os.path.exists(path):
        bot.error('%s does not exist.' %path)
        sys.exit(1)

    # Interaction with a registry requires secrets
    self.require_secrets()

    # Extract the metadata
    names = parse_image_name(remove_uri(name), tag=tag)
    image_size = os.path.getsize(path) >> 20

# COLLECTION ###################################################################

    # If the registry is provided in the uri, use it
    if names['registry'] == None:
        names['registry'] = self.base

    # If the base doesn't start with http or https, add it
    names = self._add_https(names)

    # Prepare push request, this will return a collection ID if permission
    url = '%s/push/' % names['registry']
    auth_url = '%s/upload/chunked_upload' % names['registry']
    SREGISTRY_EVENT = self.authorize(request_type="push",
                                     names=names)

    # Data fields for collection
    fields = { 'collection': names['collection'],
               'name':names['image'],
               'tag': names['tag']}

    headers = { 'Authorization': SREGISTRY_EVENT }

    r = requests.post(auth_url, json=fields, headers=headers)

    # Always tell the user what's going on!
    message = self._read_response(r)
    print('\n[1. Collection return status {0} {1}]'.format(r.status_code, message))

    # Get the collection id, if created, and continue with upload
    if r.status_code != 200:
        sys.exit(1)

# UPLOAD #######################################################################

    url = '%s/upload' % names['registry'].replace('/api','')
    bot.debug('Seting upload URL to {0}'.format(url))

    cid = r.json()['cid']
    upload_to = os.path.basename(names['storage'])

    SREGISTRY_EVENT = self.authorize(request_type="upload",
                                     names=names)

    encoder = MultipartEncoder(fields={'SREGISTRY_EVENT': SREGISTRY_EVENT,
                                       'name': names['image'],
                                       'collection': str(cid),
                                       'tag': names['tag'],
                                       'file1': (upload_to, open(path, 'rb'), 'text/plain')})

    progress_callback = create_callback(encoder, self.quiet)
    monitor = MultipartEncoderMonitor(encoder, progress_callback)
    headers = {'Content-Type': monitor.content_type,
               'Authorization': SREGISTRY_EVENT }

    try:
        r = requests.post(url, data=monitor, headers=headers)
        r.raise_for_status()
        message = r.json()['message']
        print('\n[Return status {0} {1}]'.format(r.status_code, message))
    except requests.HTTPError as e:
        print('\nUpload failed: {0}.'.format(e))
    except KeyboardInterrupt:
        print('\nUpload cancelled.')
    except Exception as e:
        print(e)

def create_callback(encoder, quiet=False):
    encoder_len = encoder.len / (1024*1024.0)

    bar = ProgressBar(expected_size=encoder_len,
                      filled_char='=',
                      hide=quiet)

    def callback(monitor):
        bar.show(monitor.bytes_read / (1024*1024.0))
    return callback


'''

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

import os
import fnmatch
import re

def get_recipe_tag(path):
    '''get a recipe tag (the extension of a Singularity file). The extension
       determines the tag. If no extension is found, latest is used.

       Parameters
       ==========
       path: the path to the recipe file (e.g. /opt/Singularity.tag)

    '''
    tag = re.sub('.+Singularity[.]?','', path)
    if tag in ['', None]:
        tag = 'latest'
    return tag

def parse_header(recipe, header="from", remove_header=True):
    '''take a recipe, and return the complete header, line. If
       remove_header is True, only return the value.

       Parameters
       ==========
       recipe: the recipe file
       headers: the header key to find and parse
       remove_header: if true, remove the key

    '''
    parsed_header = None
    fromline = [x for x in recipe.split('\n') if "%s:" %header in x.lower()]

    # Case 1: We did not find the fromline
    if len(fromline) == 0:
        return ""

    # Case 2: We found it!
    if len(fromline) > 0:
        fromline = fromline[0]
        parsed_header = fromline.strip()

    # Does the user want to clean it up?
    if remove_header is True:
        parsed_header = fromline.split(':', 1)[-1].strip()
    return parsed_header

def find_recipes(folders, pattern=None, base=None):
    '''find recipes will use a list of base folders, files,
       or patterns over a subset of content to find recipe files
       (indicated by Starting with Singularity

       Parameters
       ==========
        base: if defined, consider folders recursively below this level.

    '''
    # If the user doesn't provide a list of folders, use $PWD
    if folders is None:
        folders = os.getcwd()

    if not isinstance(folders,list):
        folders = [folders]

    manifest = dict()
    for base_folder in folders:

        # If we find a file, return the one file
        custom_pattern = None
        if os.path.isfile(base_folder):  # updates manifest
            manifest = find_single_recipe(filename=base_folder,
                                          pattern=pattern,
                                          manifest=manifest)
            continue

        # The user likely provided a custom pattern
        elif not os.path.isdir(base_folder):
            custom_pattern = base_folder.split('/')[-1:][0]
            base_folder = "/".join(base_folder.split('/')[0:-1])

        # If we don't trigger loop, we have directory
        manifest = find_folder_recipes(base_folder=base_folder,
                                       pattern=custom_pattern or pattern,
                                       manifest=manifest,
                                       base=base)

    return manifest

def find_folder_recipes(base_folder,
                        pattern="Singularity",
                        manifest=None,
                        base=None):

    '''find folder recipes will find recipes based on a particular pattern.

       Parameters
       ==========
       base_folder: the base folder to recursively walk
       pattern: a default pattern to search for
       manifest: an already started manifest
       base: if defined, consider folders under this level recursively.

    '''

    # The user is not appending to an existing manifest
    if manifest is None:
        manifest = dict()

    for root, dirnames, filenames in os.walk(base_folder):

        for filename in fnmatch.filter(filenames, pattern):

            container_path = os.path.join(root, filename)
            if base is not None:
                container_base = container_path.replace(base,'').strip('/')
                collection = container_base.split('/')[0]
                recipe = os.path.basename(container_base)
                container_uri = "%s/%s" %(collection,recipe)
            else:
                container_uri = '/'.join(container_path.strip('/').split('/')[-2:])

            add_container = True

            # Add the most recently updated container
            if container_uri in manifest:
                if manifest[container_uri]['modified'] > os.path.getmtime(container_path):
                    add_container = False

            if add_container:
                manifest[container_uri] = {'path': os.path.abspath(container_path),
                                           'modified':os.path.getmtime(container_path)}

    return manifest

def find_single_recipe(filename, pattern="Singularity", manifest=None):
    '''find_single_recipe will parse a single file, and if valid,
       return an updated manifest

       Parameters
       ==========
       filename: the filename to assess for a recipe
       pattern: a default pattern to search for
       manifest: an already started manifest

    '''

    if pattern is None:
        pattern = "Singularity*"

    recipe = None
    file_basename = os.path.basename(filename)
    if fnmatch.fnmatch(file_basename, pattern):
        recipe = {'path': os.path.abspath(filename),
                  'modified':os.path.getmtime(filename)}

    # If we already have the recipe, only add if more recent
    if manifest is not None and recipe is not None:
        container_uri = '/'.join(filename.split('/')[-2:])
        if container_uri in manifest:
            if manifest[container_uri]['modified'] < os.path.getmtime(filename):
                manifest[container_uri] = recipe
        else:
            manifest[container_uri] = recipe
        return manifest

    return recipe


'''

Copyright (C) 2018-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
from sregistry.utils import (
    get_file_hash,
    get_tmpdir,
    parse_image_name,
    remove_uri
)

from sregistry.main.google_storage.utils import prepare_metadata
from sregistry.main.google_build.utils import get_build_template

from retrying import retry
from glob import glob
import time
import requests
import tarfile
import shutil
import json
import sys
import os

def build(self, name,
                recipe="Singularity",
                context=None,
                preview=False):

    '''trigger a build on Google Cloud (builder then storage) given a name
       recipe, and Github URI where the recipe can be found.

       Parameters
       ==========
       recipe: the local recipe to build.
       name: should be the complete uri that the user has requested to push.
       context: the dependency files needed for the build. If not defined, only
                the recipe is uploaded.
       preview: if True, preview but don't run the build

       Environment
       ===========
       SREGISTRY_GOOGLE_BUILD_SINGULARITY_VERSION: the version of Singularity
           to use, defaults to 3.0.2-slim
       SREGISTRY_GOOGLE_BUILD_CLEANUP: after build, delete intermediate
           dependencies in cloudbuild bucket.

    '''
    bot.debug("BUILD %s" % recipe)

    # This returns a data structure with collection, container, based on uri
    names = parse_image_name(remove_uri(name))

    # Load the build configuration
    config = self._load_build_config(name=names['uri'], recipe=recipe)

    build_package = [recipe]
    if context not in [None, '', []]:

        # If the user gives a ., include recursive $PWD
        if '.' in context:
            context = glob(os.getcwd() + '/**/*', recursive=True)
        build_package = build_package + context

    package = create_build_package(build_package)

    # Does the package already exist? If the user cached, it might
    destination='source/%s' % os.path.basename(package)
    blob = self._build_bucket.blob(destination)

    # if it doesn't exist, upload it
    if not blob.exists() and preview is False:
        bot.log('Uploading build package!')
        manifest = self._upload(source=package,
                                bucket=self._build_bucket,
                                destination=destination)
    else:
        bot.log('Build package found in %s.' % self._build_bucket.name)

    # The source should point to the bucket with the .tar.gz, latest generation
    config["source"]["storageSource"]['bucket'] = self._build_bucket.name
    config["source"]["storageSource"]['object'] = destination

    # If not a preview, run the build and return the response
    if preview is False:
        config = self._run_build(config, self._bucket, names)

    # If the user wants to cache cloudbuild files, this will be set
    if not self._get_and_update_setting('SREGISTRY_GOOGLE_BUILD_CACHE'):
        blob.delete()

    # Clean up either way, return config or response
    shutil.rmtree(os.path.dirname(package))
    return config

def create_build_package(package_files):
    '''given a list of files, copy them to a temporary folder,
       compress into a .tar.gz, and rename based on the file hash.
       Return the full path to the .tar.gz in the temporary folder.

       Parameters
       ==========
       package_files: a list of files to include in the tar.gz

    '''
    # Ensure package files all exist
    for package_file in package_files:
        if not os.path.exists(package_file):
            bot.exit('Cannot find %s.' % package_file)

    bot.log('Generating build package for %s files...' % len(package_files))
    build_dir = get_tmpdir(prefix="sregistry-build")
    build_tar = '%s/build.tar.gz' % build_dir
    tar = tarfile.open(build_tar, "w:gz")

    # Create the tar.gz
    for package_file in package_files:
        tar.add(package_file)
    tar.close()

    # Get hash (sha256), and rename file
    sha256 = get_file_hash(build_tar)
    hash_tar =  "%s/%s.tar.gz" %(build_dir, sha256)
    shutil.move(build_tar, hash_tar)
    return hash_tar

def load_build_config(self, name, recipe):
    '''load a google compute config, meaning that we start with a template,
       and mimic the following example cloudbuild.yaml:

        steps:
        - name: "singularityware/singularity:${_SINGULARITY_VERSION}"
          args: ['build', 'julia-centos-another.sif', 'julia.def']
        artifacts:
          objects:
            location: 'gs://sregistry-gcloud-build-vanessa'
            paths: ['julia-centos-another.sif']

        Parameters
        ==========
        recipe: the local recipe file for the builder.
        name: the name of the container, based on the uri

    '''
    version_envar = 'SREGISTRY_GOOGLE_BUILD_SINGULARITY_VERSION'
    version = self._get_and_update_setting(version_envar, '3.0.2-slim')
    config = get_build_template()

    # Name is in format 'dinosaur/container-latest'

    # The command to give the builder, with image name
    container_name = '%s.sif' % name.replace('/','-', 1)
    config['steps'][0]['name'] = 'singularityware/singularity:%s' % version
    config['steps'][0]['args'] = ['build', container_name, recipe]

    config["artifacts"]["objects"]["location"] = "gs://%s" % self._bucket_name
    config["artifacts"]["objects"]["paths"] = [container_name]

    return config

def run_build(self, config, bucket, names):
    '''run a build, meaning creating a build. Retry if there is failure
    '''

    project = self._get_project()

    #          prefix,    message, color
    bot.custom('PROJECT', project, "CYAN")
    bot.custom('BUILD  ', config['steps'][0]['name'], "CYAN")

    response = self._build_service.projects().builds().create(body=config,
                                              projectId=project).execute()

    build_id = response['metadata']['build']['id']
    status = response['metadata']['build']['status']
    bot.log("build %s: %s" % (build_id, status))

    start = time.time()
    while status not in ['COMPLETE', 'FAILURE', 'SUCCESS']:
        time.sleep(15)
        response = self._build_service.projects().builds().get(id=build_id,
                                                  projectId=project).execute()

        build_id = response['id']
        status = response['status']
        bot.log("build %s: %s" % (build_id, status))

    end = time.time()
    bot.log('Total build time: %s seconds' % (round(end - start, 2)))

    # If successful, update blob metadata and visibility
    if status == 'SUCCESS':

        # Does the user want to keep the container private?
        env = 'SREGISTRY_GOOGLE_STORAGE_PRIVATE'
        blob = bucket.blob(response['artifacts']['objects']['paths'][0])

        # Make Public, if desired
        if self._get_and_update_setting(env) == None:
            blob.make_public()
            response['public_url'] = blob.public_url

        # Add the metadata directly to the object
        update_blob_metadata(blob, response, config, bucket, names)
        response['media_link'] = blob.media_link
        response['size'] = blob.size
        response['file_hash'] = blob.md5_hash

    return response

def update_blob_metadata(blob, response, config, bucket, names):
    '''a specific function to take a blob, along with a SUCCESS response
       from Google build, the original config, and update the blob
       metadata with the artifact file name, dependencies, and image hash.
    '''

    manifest = os.path.basename(response['results']['artifactManifest'])
    manifest = json.loads(bucket.blob(manifest).download_as_string())

    metadata = {'file_hash': manifest['file_hash'][0]['file_hash'][0]['value'],
                'artifactManifest': response['results']['artifactManifest'],
                'location': manifest['location'],
                'storageSourceBucket': config['source']['storageSource']['bucket'],
                'storageSourceObject': config['source']['storageSource']['object'],
                'buildCommand': ' '.join(config['steps'][0]['args']),
                'builder': config['steps'][0]['name'],
                'media_link': blob.media_link,
                'self_link': blob.self_link,
                'size': blob.size,
                'name': names['tag_uri'],
                'type': "container"} # identifier that the blob is a container

    blob.metadata = metadata
    blob._properties['metadata'] = metadata
    blob.patch()


'''

search and query functions for client

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
from sregistry.utils import ( parse_image_name, remove_uri )
import sys

def search(self, query=None, args=None):
    '''query a Singularity registry for a list of images.
     If query is None, collections are listed.

    EXAMPLE QUERIES:
    '''

    # You can optionally better parse the image uri (query), but not
    # necessary
    # names = parse_image_name(remove_uri(query))

    if query is not None:

        # Here you might do a function that is a general list
        # Note that this means adding the function Client in __init__
        return self._container_query(query)

    # or default to listing (searching) all things.
    return self._search_all()

# These functions are added in __init__.py like so:
# from .query import search_all, container_query
# Client._search_all = search_all
# Client._container_query = container_query

def search_all(self):
    '''a "show all" search that doesn't require a query'''

    # This should be your apis url for a search
    url = '...'

    # paginte get is what it sounds like, and what you want for multiple
    # pages of results
    results = self._paginate_get(url)

    if len(results) == 0:
        bot.info("No container collections found.")
        sys.exit(1)

    bot.info("Collections")

    # Here is how to create a simple table. You of course must parse your
    # custom result and form the fields in the table to be what you think
    # are important!
    rows = []
    for result in results:
        if "containers" in result:
            for c in result['containers']:
                rows.append([ c['uri'],
                              c['detail'] ])

    bot.table(rows)
    return rows

def container_query(self, query):
    '''search for a specific container.
    This function would likely be similar to the above, but have different
    filter criteria from the user (based on the query)
    '''
    # Write your functions here (likely a derivation of search_all) to do
    # a more specific search. The entrypoint to both these last functions
    # is via the main search function.
    rows = ['','','']
    bot.table(rows)
    return rows


'''

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
import hashlib
import os
import re

def get_image_hash(image_path):
    '''return an md5 hash of the file based on a criteria level. This
       is intended to give the file a reasonable version.

       Parameters
       ==========
       image_path: full path to the singularity image

    '''
    hasher = hashlib.md5()
    with open(image_path, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hasher.update(chunk)
    return hasher.hexdigest()

# Regular expressions to parse registry, collection, repo, tag and version
_docker_uri = re.compile(
    "(?:(?P<registry>[^/@]+[.:][^/@]*)/)?"
    "(?P<collection>(?:[^:@/]+/)+)?"
    "(?P<repo>[^:@/]+)"
    "(?::(?P<tag>[^:@]+))?"
    "(?:@(?P<version>.+))?"
    "$")

# Reduced to match registry:port/repo or registry.com/repo
_reduced_uri = re.compile(
    "(?:(?P<registry>[^/@]+[.:][^/@]*)/)?"
    "(?P<repo>[^:@/]+)"
    "(?::(?P<tag>[^:@]+))?"
    "(?:@(?P<version>.+))?"
    "$"
    "(?P<collection>.)?")

# Default
_default_uri = re.compile(
    "(?:(?P<registry>[^/@]+)/)?"
    "(?P<collection>(?:[^:@/]+/)+)"
    "(?P<repo>[^:@/]+)"
    "(?::(?P<tag>[^:@]+))?"
    "(?:@(?P<version>.+))?"
    "$")

def set_default(item, default, use_default):
    '''if an item provided is None and boolean use_default is set to True,
       return the default. Otherwise, return the item.
    '''
    if item == None and use_default:
        return default
    return item

def parse_image_name(image_name,
                     tag=None,
                     version=None,
                     defaults=True,
                     ext="sif",
                     default_collection="library",
                     default_tag="latest",
                     base=None,
                     lowercase=True):

    '''return a collection and repo name and tag
    for an image file.

    Parameters
    =========
    image_name: a user provided string indicating a collection,
                image, and optionally a tag.
    tag: optionally specify tag as its own argument
         over-rides parsed image tag
    defaults: use defaults "latest" for tag and "library"
              for collection.
    base: if defined, remove from image_name, appropriate if the
          user gave a registry url base that isn't part of namespace.
    lowercase: turn entire URI to lowercase (default is True)
    '''

    # Save the original string
    original = image_name

    if base is not None:
        image_name = image_name.replace(base,'').strip('/')

    # If a file is provided, remove extension
    image_name = re.sub('[.](img|simg|sif)','', image_name)

    # Parse the provided name
    uri_regexes = [ _reduced_uri,
                    _default_uri,
                    _docker_uri ]

    for r in uri_regexes:
        match = r.match(image_name)
        if match:
            break

    if not match:
        bot.exit('Could not parse image "%s"! Exiting.' % image)

    # Get matches
    registry = match.group('registry')
    collection = match.group('collection')
    repo_name = match.group('repo')
    repo_tag = match.group('tag')
    version = match.group('version')

    # A repo_name is required
    assert(repo_name)

    # If a collection isn't provided
    collection = set_default(collection, default_collection, defaults)
    repo_tag = set_default(repo_tag, default_tag, defaults)

    # The collection, name must be all lowercase
    if lowercase:
        collection = collection.lower().rstrip('/')
        repo_name = repo_name.lower()
        repo_tag = repo_tag.lower()
    else:
        collection = collection.rstrip('/')

    if version != None:
        version = version.lower()

    # Piece together the uri base
    if registry == None:
        uri = "%s/%s" % (collection, repo_name)
    else:
        uri = "%s/%s/%s" % (registry, collection, repo_name)

    url = uri

    # Tag is defined
    if repo_tag != None:
        uri = "%s-%s" % (uri, repo_tag)
        tag_uri = "%s:%s" % (url, repo_tag)

    # Version is defined
    storage_version = None
    if version is not None:
        uri = "%s@%s" % (uri, version)
        tag_uri = "%s@%s" % (tag_uri, version)
        storage_version = "%s@%s.%s" % (tag_uri, version, ext)

    # A second storage URI honors the tag (:) separator

    storage = "%s.%s" %(uri, ext)
    storage_uri = "%s.%s" %(tag_uri, ext)
    result = {"collection": collection,
              "original": original,
              "registry": registry,
              "image": repo_name,
              "url": url,
              "tag": repo_tag,
              "version": version,
              "storage": storage,
              "storage_uri": storage_uri,
              "storage_version": storage_version or storage_uri,
              "tag_uri": tag_uri,
              "uri": uri}

    return result

def format_container_name(name, special_characters=None):
    '''format_container_name will take a name supplied by the user,
    remove all special characters (except for those defined by "special-characters"
    and return the new image name.
    '''
    if special_characters is None:
        special_characters = []
    return ''.join(e.lower()
                   for e in name if e.isalnum() or e in special_characters)

def get_uri(image):
    '''get the uri for an image, if within acceptable

       Parameters
       ==========
       image: the image uri, in the format <uri>://<registry>/<namespace>:<tag>

    '''
    # Ensure we have a string
    image = image or ''

    # Find uri prefix, including ://
    regexp = re.compile('^.+://')
    uri = regexp.match(image)

    if uri is not None:
        uri = (uri.group().lower()
                          .replace('_','-')
                          .replace('://',''))

        accepted_uris = ['aws',
                         'docker',
                         'http', 'https', # Must be allowed for pull
                         'dropbox',
                         'gitlab',
                         'globus',
                         'google-build',
                         'google-storage',
                         'google-drive',
                         'hub',
                         'nvidia',
                         'registry',
                         's3',
                         'swift']

        # Allow for Singularity compatability
        if "shub" in uri: uri = "hub"

        if uri not in accepted_uris:
            bot.warning('%s is not a recognized uri.' % uri)
            uri = None

    return uri

def remove_uri(image):
    '''remove_uri will remove the uri from the image path, if provided.

       Parameters
       ==========
       image: the image uri, in the format <uri>://<registry>/<namespace>:<tag>

    '''
    return re.sub('^.+://','', image)


'''

Copyright (C) 2018-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
from swiftclient.exceptions import ClientException
from sregistry.utils import ( parse_image_name, remove_uri )
import os
import sys

def pull(self, images, file_name=None, save=True, **kwargs):
    '''pull an image from storage using Swift. The image is found based on the
       storage uri

       Parameters
       ==========
       images: refers to the uri given by the user to pull in the format
       <collection>/<namespace>. You should have an API that is able to
       retrieve a container based on parsing this uri.
       file_name: the user's requested name for the file. It can
                  optionally be None if the user wants a default.
       save: if True, you should save the container to the database
             using self.add()

       Returns
       =======
       finished: a single container path, or list of paths

    '''
    force = False
    if "force" in kwargs:
        force = kwargs['force']

    if not isinstance(images, list):
        images = [images]

    bot.debug('Execution of PULL for %s images' % len(images))

    # If used internally we want to return a list to the user.
    finished = []
    for image in images:

        names = parse_image_name(remove_uri(image))

        # First try to get the collection
        collection = self._get_collection(names['collection'])
        if collection is None:
            bot.error('Collection %s does not exist.' % names['collection'])

            # Show the user collections he/she does have access to
            collections = self.get_collections()
            if collections:
                bot.info('Collections available to you: \n%s' %'\n'.join(collections))
            sys.exit(1)

        # Determine if the container exists in storage
        image_name = os.path.basename(names['storage'])

        try:
            obj_tuple = self.conn.get_object(names['collection'], image_name)
        except ClientException:
            bot.exit('%s does not exist.' % names['storage'])

        # Give etag as version if version not defined
        if names['version'] == None:
            names['version'] = obj_tuple[0]['etag']

        # If the user didn't provide a file, make one based on the names
        if file_name is None:
            file_name = self._get_storage_name(names)

        # If the file already exists and force is False
        if os.path.exists(file_name) and force is False:
            bot.error('Image exists! Remove first, or use --force to overwrite')
            sys.exit(1)

        # Write to file
        with open(file_name, 'wb') as filey:
            filey.write(obj_tuple[1])

        # If we save to storage, the uri is the dropbox_path
        if save is True:

            names.update(obj_tuple[0])
            container = self.add(image_path = file_name,
                                 image_uri = names['uri'],
                                 metadata = names)

            # When the container is created, this is the path to the image
            image_file = container.image

            if os.path.exists(image_file):
                bot.debug('Retrieved image file %s' %image_file)
                bot.custom(prefix="Success!", message=image_file)
                finished.append(image_file)

        else:
            bot.error('%s does not exist. Try sregistry search to see images.' % path)

    if len(finished) == 1:
        finished = finished[0]
    return finished


'''

Copyright (C) 2017-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
from sregistry.utils import (
    get_image_hash,
    get_thumbnail,
    parse_image_name,
    remove_uri
)

from googleapiclient.http import MediaFileUpload
from googleapiclient.errors import HttpError
import base64
import json
import sys
import os

def push(self, path, name, tag=None):
    '''push an image to Google Cloud Drive, meaning uploading it

    path: should correspond to an absolte image path (or derive it)
    name: should be the complete uri that the user has requested to push.
    tag: should correspond with an image tag. This is provided to mirror Docker
    '''
    # The root of the drive for containers (the parent folder)
    parent = self._get_or_create_folder(self._base)

    image = None
    path = os.path.abspath(path)
    bot.debug("PUSH %s" % path)

    if not os.path.exists(path):
        bot.error('%s does not exist.' %path)
        sys.exit(1)

    names = parse_image_name(remove_uri(name),tag=tag)
    if names['version'] is None:
        version = get_image_hash(path)
        names = parse_image_name(remove_uri(name), tag=tag, version=version)

    # Update metadata with names, flatten to only include labels
    metadata = self.get_metadata(path, names=names)
    metadata = metadata['data']
    metadata.update(names)
    metadata.update(metadata['attributes']['labels'])
    del metadata['attributes']

    file_metadata = {
        'name': names['storage'],
        'mimeType' : 'application/octet-stream',
        'parents': [parent['id']],
        'properties': metadata
    }

    media = MediaFileUpload(path,resumable=True)
    try:
        bot.spinner.start()
        image = self._service.files().create(body=file_metadata,
                                             media_body=media,
                                             fields='id').execute()

        # Add a thumbnail!
        thumbnail = get_thumbnail()

        with open(thumbnail, "rb") as f:
            body = { "contentHints": {
                     "thumbnail": { "image": base64.urlsafe_b64encode(f.read()).decode('utf8'),
                                     "mimeType": "image/png" }
                                  }}
            image = self._service.files().update(fileId=image['id'],
                                                 body = body).execute()

        bot.spinner.stop()
        print(image['name'])

    except HttpError:
        bot.error('Error uploading %s' %path)
        pass

    return image


'''

Copyright (C) 2018-2019 Vanessa Sochat.

This Source Code Form is subject to the terms of the
Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''

from sregistry.logger import bot
from sregistry.auth import basic_auth_header
from sregistry.main import ApiConnection
import json
import sys
import os

# The core of nvidia is actually docker
from sregistry.main.docker.api import (
    get_config,
    get_environment_tar,
    get_download_cache,
    get_size
)
from sregistry.main.aws.api import ( download_layers, update_token, get_digests,
                                     get_manifest )
from .pull import ( pull, _pull )

class Client(ApiConnection):

    def __init__(self, secrets=None, **kwargs):
        '''to work with nvidia container registry. If you need help with
           their API, the best reference is:

           https://docs.aws.amazon.com/AmazonECR/latest/APIReference/ecr-api.pdf

        '''
        self.reverseLayers = False
        self._reset_headers()
        self._update_secrets()
        self._set_base()
        self._update_headers()
        super(ApiConnection, self).__init__(**kwargs)

    def _speak(self):
        '''if you want to add an extra print (of a parameter, for example)
           for the user when the client initalizes, write it here, eg:
           bot.info('[setting] value')
        '''
        pass

    def _reset_headers(self):
        '''reset headers is called from update_headers, and will update the
           headers based on what is found with the client secrets
        '''

        # specify wanting version 2 schema
        # meaning the correct order of digests
        # returned (base to child)
        accept = 'application/vnd.docker.distribution.manifest.v2+json,'
        accept += 'application/vnd.docker.distribution.manifest.list.v2+json,'
        accept += 'application/json'

        self.headers = {"Accept": accept,
                        "Accept-Encoding": "identity",
                        'Content-Type': 'application/x-amz-json-1.1; application/json; charset=utf-8'}

    def _set_base(self, zone=None):
        '''set the API base or default to use Docker Hub. The user is able
           to set the base, api version, and protocol via a settings file
           of environment variables:
        '''
        if hasattr(self.aws._client_config, 'region_name'):
            zone = self.aws._client_config.region_name

        aws_id = self._required_get_and_update('SREGISTRY_AWS_ID')
        aws_zone = self._required_get_and_update('SREGISTRY_AWS_ZONE', zone)
        version = self._get_setting('SREGISTRY_AWS_VERSION', 'v2')
        base = self._get_setting('SREGISTRY_AWS_BASE')

        if base is None:
            base = "%s.dkr.ecr.%s.amazonaws.com" % (aws_id, aws_zone)

        nohttps = self._get_setting('SREGISTRY_AWS_NOHTTPS')
        if nohttps is None:
            nohttps = "https://"
        else:
            nohttps = "http://"

        # <protocol>://<base>/<version>
        self.base = "%s%s/%s" %(nohttps, base.strip('/'), version)

    def _update_secrets(self):
        '''update secrets will take a secrets credential file
           either located at .sregistry or the environment variable
           SREGISTRY_CLIENT_SECRETS and update the current client
           secrets as well as the associated API base. For the case of
           using Docker Hub, if we find a .docker secrets file, we update
           from there.
        '''
        bot.debug('Creating aws client...')
        try:
            from awscli.clidriver import create_clidriver
        except:
            bot.exit('Please install pip install sregistry[aws]')

        driver = create_clidriver()
        self.aws = driver.session.create_client('ecr')

    def __str__(self):
        return type(self)

# Functions exposed to the client
Client.pull = pull
Client._pull = _pull

# Api functions for image layers and manifests (hidden)
Client._download_layers = download_layers
Client._get_digests = get_digests
Client._get_download_cache = get_download_cache
Client._get_manifest = get_manifest
Client._get_size = get_size
Client._update_token = update_token
Client._get_environment_tar = get_environment_tar