KEY_CREDENTIALS = 'credentials' class HyperparameterParser(object):
    '''Class for parsing hyperparameters'''

    def __init__(self, runner_args, logger):
        self.runner_args = runner_args
        self.logger = logger

    def convert_to_tuples(self, hyperparameters):
        if self.runner_args.optimizer == "grid":
            all_hyperparam_values = []
            for h in hyperparameters:
                assert h.values is not None
                hyperparam_values = [(h.name, value) for value in h.values]
                all_hyperparam_values.append(hyperparam_values)

            hyperparam_tuples = []
            for item in itertools.product(*all_hyperparam_values):
                hyperparam_tuple = {}
                for name, param in item:
                    hyperparam_tuple[name] = param
                hyperparam_tuples.append(hyperparam_tuple)
        else:
            hyperparam_tuples = []
            for hyperparam_list in hyperparameters:
                hyperparam_dict = {}
                for h in hyperparam_list:
                    hyperparam_dict[h.name] = h.values
                hyperparam_tuples.append(hyperparam_dict)

        # print hyperparam_tuples
        return hyperparam_tuples

    def parse(self):
        self.index = 0
        hyperparameters = []
        for hyperparam in self.runner_args.hyperparam:
            param_name = hyperparam.split('=')[0]
            param_values_str = hyperparam.split('=')[1]
            if self.runner_args.optimizer == "grid":
                hyperparameters.append(self._parse_grid(param_name,
                                                        param_values_str))
            else:
                hyperparameters.append(self._parse_opt(param_name,
                                                       param_values_str))
        if self.runner_args.verbose:
            self.logger.info("Parsed the following hyperparameters:")
            for h in hyperparameters:
                self.logger.info(str(h))
        return hyperparameters

    def _parse_opt(self, param_name, range_str):
        unbounded = is_log = rand_init = False
        min_range = max_range = array_length = None
        raw_fields = range_str.split(":")

        correct_format = True
        flags = ""
        if len(raw_fields) > 2:
            flags = raw_fields[-1]
            allowed_flags = 'ualr'
            for letter in flags:
                if letter not in allowed_flags or flags.count(letter) != 1:
                    correct_format = False
            proper_field_length = 4 if "a" in flags else 3
            if len(raw_fields) != proper_field_length:
                correct_format = False
        elif len(raw_fields) < 2:
            correct_format = False

        if not correct_format:
            raise ValueError("Hyperparameter flags (%s) are incorrect for %s" %
                             (range_str, self.runner_args.optimizer))

        try:
            min_range = float(raw_fields[0])
            max_range = float(raw_fields[1])
            array_length = int(raw_fields[2]) if "a" in flags else None
            if array_length is not None and array_length <= 0:
                raise ValueError
        except ValueError:
            raise ValueError(
                "Hyperparameter values (%s) are incorrect for %s" %
                (range_str, self.runner_args.optimizer))

        if min_range > max_range:
            raise ValueError("Min range (%s) is larger than max range (%s) " %
                             (min_range, max_range))

        unbounded = True if "u" in flags else False
        rand_init = True if "r" in flags else False
        if "l" in flags:
            is_log = True
            min_range = math.log(min_range)
            max_range = math.log(max_range)
        else:
            is_log = False

        if not hasattr(self, 'index'):
            self.index = 0
        h = Hyperparameter(
            param_name,
            index=self.index,
            min_range=min_range,
            max_range=max_range,
            array_length=array_length,
            unbounded=unbounded,
            is_log=is_log,
            rand_init=rand_init)
        self.index += array_length if array_length is not None else 1
        return h

    def _parse_grid(self, param_name, range_str):
        return_val = None
        if ',' in range_str:
            # return numpy array for consistency with other cases
            return_val = np.array([float(s) for s in range_str.split(',')])
        elif ':' in range_str:
            range_limits = range_str.split(':')
            assert len(range_limits) > 1
            if len(range_limits) == 2:
                try:
                    limit1 = float(range_limits[0])
                except ValueError:
                    limit1 = 0.0
                limit2 = float(range_limits[1])
                return_val = np.arange(limit1, limit2 + 1)
            else:
                try:
                    limit1 = float(range_limits[0])
                except ValueError:
                    limit1 = 0.0

                limit3 = float(range_limits[2])

                try:
                    limit2 = float(range_limits[1])
                    if int(limit2) == limit2 and limit2 > abs(limit3 - limit1):
                        return_val = np.linspace(limit1, limit3, int(limit2))
                    else:
                        return_val = np.arange(
                            limit1, limit3 + 0.5 * limit2, limit2)

                except ValueError:
                    if 'l' in range_limits[1]:
                        limit2 = int(range_limits[1].replace('l', ''))
                        return_val = np.exp(
                            np.linspace(
                                np.log(limit1),
                                np.log(limit3),
                                limit2))
                    else:
                        raise ValueError(
                            'unknown limit specification ' +
                            range_limits[1])

        else:
            return_val = [float(range_str)]
        if not isinstance(return_val, list):
            return_val = return_val.tolist()

        if not hasattr(self, 'index'):
            self.index = 0
        h = Hyperparameter(param_name, index=self.index, values=return_val)
        self.index += 1
        return h class Progbar(object):
    """Displays a progress bar.

    # Arguments
        target: Total number of steps expected, None if unknown.
        interval: Minimum visual progress update interval (in seconds).
    """

    def __init__(self, target, width=30, verbose=1, interval=0.05):
        self.width = width
        if target is None:
            target = -1
        self.target = target
        self.sum_values = {}
        self.unique_values = []
        self.start = time.time()
        self.last_update = 0
        self.interval = interval
        self.total_width = 0
        self.seen_so_far = 0
        self.verbose = verbose

    def update(self, current, values=None, force=False):
        """Updates the progress bar.

        # Arguments
            current: Index of current step.
            values: List of tuples (name, value_for_last_step).
                The progress bar will display averages for these values.
            force: Whether to force visual progress update.
        """
        values = values or []
        for k, v in values:
            if k not in self.sum_values:
                self.sum_values[k] = [v * (current - self.seen_so_far),
                                      current - self.seen_so_far]
                self.unique_values.append(k)
            else:
                self.sum_values[k][0] += v * (current - self.seen_so_far)
                self.sum_values[k][1] += (current - self.seen_so_far)
        self.seen_so_far = current

        now = time.time()
        if self.verbose == 1:
            if not force and (now - self.last_update) < self.interval:
                return

            prev_total_width = self.total_width
            sys.stdout.write('\b' * prev_total_width)
            sys.stdout.write('\r')

            if self.target != -1:
                numdigits = int(np.floor(np.log10(self.target))) + 1
                barstr = '%%%dd/%%%dd [' % (numdigits, numdigits)
                bar = barstr % (current, self.target)
                prog = float(current) / self.target
                prog_width = int(self.width * prog)
                if prog_width > 0:
                    bar += ('=' * (prog_width - 1))
                    if current < self.target:
                        bar += '>'
                    else:
                        bar += '='
                bar += ('.' * (self.width - prog_width))
                bar += ']'
                sys.stdout.write(bar)
                self.total_width = len(bar)

            if current:
                time_per_unit = (now - self.start) / current
            else:
                time_per_unit = 0
            eta = time_per_unit * (self.target - current)
            info = ''
            if current < self.target and self.target != -1:
                info += ' - ETA: %ds' % eta
            else:
                info += ' - %ds' % (now - self.start)
            for k in self.unique_values:
                info += ' - %s:' % k
                if isinstance(self.sum_values[k], list):
                    avg = np.mean(
                        self.sum_values[k][0] / max(1, self.sum_values[k][1]))
                    if abs(avg) > 1e-3:
                        info += ' %.4f' % avg
                    else:
                        info += ' %.4e' % avg
                else:
                    info += ' %s' % self.sum_values[k]

            self.total_width += len(info)
            if prev_total_width > self.total_width:
                info += ((prev_total_width - self.total_width) * ' ')

            sys.stdout.write(info)
            sys.stdout.flush()

            if current >= self.target:
                sys.stdout.write('\n')

        if self.verbose == 2:
            if current >= self.target:
                info = '%ds' % (now - self.start)
                for k in self.unique_values:
                    info += ' - %s:' % k
                    avg = np.mean(
                        self.sum_values[k][0] / max(1, self.sum_values[k][1]))
                    if avg > 1e-3:
                        info += ' %.4f' % avg
                    else:
                        info += ' %.4e' % avg
                sys.stdout.write(info + "\n")

        self.last_update = now

    def add(self, n, values=None):
        self.update(self.seen_so_far + n, values) def rsync_cp(source, dest, ignore_arg='', logger=None):
    try:
        if os.path.exists(dest):
            shutil.rmtree(dest) if os.path.isdir(dest) else os.remove(dest)
        os.makedirs(dest)
    except OSError:
        pass

    if ignore_arg != '':
        source += "/"
        tool = 'rsync'
        args = [tool, ignore_arg, '-aHAXE', source, dest]
    else:
        try:
            os.rmdir(dest)
        except OSError:
            pass

        tool = 'cp'
        args = [
            tool,
            '-pR',
            source,
            dest
        ]

    pcp = subprocess.Popen(args, stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
    cpout, _ = pcp.communicate()
    if pcp.returncode != 0 and logger is not None:
        logger.info('%s returned non-zero exit code. Output:' % tool)
        logger.info(cpout) def create_experiment(
        filename,
        args,
        experiment_name=None,
        project=None,
        artifacts=None,
        resources_needed=None,
        metric=None,
        max_duration=None,
        dependency_policy: DependencyPolicy = None,
        from_compl_service: bool = False):

    if artifacts is None:
        artifacts = {}

    key = experiment_name if experiment_name else \
        str(int(time.time())) + "_" + str(uuid.uuid4())

    if dependency_policy is None:
        raise ValueError("dependency_policy argument must be set.")

    packages = dependency_policy.generate(resources_needed)

    return Experiment(
        key=key,
        filename=filename,
        args=args,
        pythonenv=packages,
        project=project,
        artifacts=artifacts,
        resources_needed=resources_needed,
        metric=metric,
        max_duration=max_duration,
        from_compl_service=from_compl_service) def submit_experiments(
        experiments,
        config=None,
        logger=None,
        queue=None,
        python_pkg=[],
        external_payload_builder: PayloadBuilder=None):

    num_experiments = len(experiments)

    payload_builder = external_payload_builder
    if payload_builder is None:
        # Setup our own payload builder
        payload_builder = UnencryptedPayloadBuilder("simple-payload")
        # Are we using experiment payload encryption?
        public_key_path = config.get('public_key_path', None)
        if public_key_path is not None:
            logger.info("Using RSA public key path: {0}".format(public_key_path))
            signing_key_path = config.get('signing_key_path', None)
            if signing_key_path is not None:
                logger.info("Using RSA signing key path: {0}".format(signing_key_path))
            payload_builder = \
                EncryptedPayloadBuilder(
                    "cs-rsa-encryptor [{0}]".format(public_key_path),
                    public_key_path, signing_key_path)

    start_time = time.time()

    # Reset our storage setup, which will guarantee
    # that we rebuild our database and storage provider objects
    # that's important in the case that previous experiment batch
    # cleaned up after itself.
    storage_setup.reset_storage()

    for experiment in experiments:
        # Update Python environment info for our experiments:
        experiment.pythonenv = util.add_packages(experiment.pythonenv, python_pkg)

        # Add experiment to database:
        try:
            with db_provider_setup.get_db_provider(config) as db:
                _add_git_info(experiment, logger)
                db.add_experiment(experiment)
        except BaseException:
            traceback.print_exc()
            raise

        payload = payload_builder.construct(experiment, config, python_pkg)

        logger.info("Submitting experiment: {0}"
                     .format(json.dumps(payload, indent=4)))

        queue.enqueue(json.dumps(payload))
        logger.info("studio run: submitted experiment " + experiment.key)

    logger.info("Added {0} experiment(s) in {1} seconds to queue {2}"
                .format(num_experiments, int(time.time() - start_time), queue.get_name()))
    return queue.get_name() def reset_storage_providers():
def get_config(config_file=None):
def get_db_provider(config=None, blocking_auth=True):
def get_queue(
        queue_name=None,
        cloud=None,
        config=None,
        logger=None,
        close_after=None,
        verbose=10):
def shutdown_queue(queue, logger=None, delete_queue=True):
def get_worker_manager(config, cloud=None, verbose=10): def get_git_info(path='.', abort_dirty=True):
def is_git(path='.'):
def is_clean(path='.'):
def get_repo_url(path='.', remove_user=True):
def get_branch(path='.'):
def get_commit(path='.'):
def get_my_repo_url():
def get_my_branch():
def get_my_checkout_target(): class LocalExecutor(object):
    def __init__(self, queue, args):
    def run(self, experiment):
                def kill_subprocess():
                def get_duration(tag: str):
                def checkpoint():
                def kill_if_stopped():
def allocate_resources(experiment, config=None, verbose=10):
def allocate_gpus(gpus_needed, resources_needed={}, config=None):
def main(args=sys.argv):
def which(program):
    def is_exe(fpath):
def worker_loop(queue, parsed_args,
                single_experiment=False,
                timeout=0,
                verbose=None):
            def try_get_experiment():
                def hold_job():
def pip_install_packages(packages, python='python', logger=None):
def wait_for_messages(queue, timeout, logger=None):
def save_metrics(path):
def pip_needed_packages(packages, python='python'): STUDIOML_EXPERIMENT = 'STUDIOML_EXPERIMENT'
STUDIOML_HOME = 'STUDIOML_HOME'
STUDIOML_ARTIFACT_MAPPING = 'STUDIOML_ARTIFACT_MAPPING'
def get_experiment_key():
def get_studio_home():
def setup_experiment(env, experiment, clean=True):
def get_artifact(tag):
def get_artifacts():
def get_model_directory(experiment_name=None):
def get_artifact_cache(tag, experiment_name=None):
def get_blob_cache(blobkey):
def get_model_directory(experiment_name=None):
def _get_artifact_mapping_path(experiment_name=None):
def _get_experiment_key(experiment):
def _setup_model_directory(experiment_name, clean=False):
def get_queue_directory():
def get_tensorboard_dir(experiment_name=None): class StudioDependencyPolicy(DependencyPolicy):
    """
    StudioML policy for adjusting experiment dependencies
    to specific execution environment and required resources.
    """

    def generate(self, resources_needed):
        if freeze is None:
            raise ValueError(
                "freeze operation is not available for StudioDependencyPolicy")

        needs_gpu = self._needs_gpu(resources_needed)
        packages = freeze.freeze()
        result = []
        for pkg in packages:
            if self._is_special_reference(pkg):
                # git repo or directly installed package
                result.append(pkg)
            elif '==' in pkg:
                # pypi package
                pkey = re.search(r'^.*?(?=\=\=)', pkg).group(0)
                pversion = re.search(r'(?<=\=\=).*\Z', pkg).group(0)

                if needs_gpu and \
                        (pkey == 'tensorflow' or pkey == 'tf-nightly'):
                    pkey = pkey + '-gpu'

                # TODO add installation logic for torch
                result.append(pkey + '==' + pversion)
        return result

    def _is_special_reference(self, pkg: str):
        if pkg.startswith('-e git+'):
            return True
        if 'git+https://' in pkg or 'file://' in pkg:
            return True
        return False

    def _needs_gpu(self, resources_needed):
        return resources_needed is not None and \
            int(resources_needed.get('gpus')) > 0 DEBUG = logging.DEBUG
INFO = logging.INFO
ERROR = logging.ERROR
def get_logger(name):
def debug(line):
def error(line):
def info(line): DAY = 86400
HOUR = 3600
MINUTE = 60
def remove_backspaces(line):
def report_fatal(msg: str, logger):
def sha256_checksum(filename, block_size=65536):
def filehash(filename, block_size=65536, hashobj=None):
def rand_string(length):
def _looks_like_url(name):
def parse_s3_path(qualified: str):
def retry(f,
          no_retries=5, sleep_time=1,
          exception_class=BaseException, logger=None):
def compression_to_extension(compression):
def compression_to_taropt(compression):
def _compression_to_extension_taropt(compression):
def timeit(method):
    def timed(*args, **kw):
def sixdecode(sval):
def shquote(sval):
def parse_duration(duration_str):
def parse_verbosity(verbosity=None):
def str2duration(sval):
def get_temp_filename() -> str:
def delete_local_path(local_path: str, root: str, shallow: bool):
def delete_local_folders(local_folder_path: str, root: str):
def rm_rf(path):
def check_for_kb_interrupt():
def add_packages(list1, list2):
import copy
import sys
import argparse
import re
import os
import uuid
import importlib
import time
import six
import numpy as np
from studio.credentials.credentials import KEY_CREDENTIALS
from studio.hyperparameter import HyperparameterParser
from studio.extra_util import Progbar, rsync_cp
from studio.experiments.experiment import create_experiment
from studio.experiment_submitter import submit_experiments
from studio import model
from studio import git_util
from studio import local_worker
from studio import fs_tracker
from studio.dependencies_policies.studio_dependencies_policy import StudioDependencyPolicy
from studio.util import logs, util
        tag = re.sub('.*:(?=[^:]*\Z)', '', entry)

        if url_schema.match(entry):
            assert not mutable, \
                'artifacts specfied by url can only be immutable'
            retval[tag] = {
                'url': path,
                'mutable': False,
                'unpack': False
            }
        elif s3_schema.match(entry) or \
                gcs_schema.match(entry) or \
                dhub_schema.match(entry) or \
                shub_schema.match(entry):

            assert not mutable, \
                'artifacts specfied by url can only be immutable'
            retval[tag] = {
                'qualified': path,
                'mutable': False,
                'unpack': False
            }
        else:
            retval[tag] = {
                'local': os.path.expanduser(path),
                'mutable': mutable,
                'unpack': True
            }
            if not mutable:
                assert os.path.exists(retval[tag]['local'])
    return retval


def _parse_external_artifacts(art_list, db):
    retval = {}
    for entry in art_list:
        external = re.sub(':.*', '', entry)
        tag = re.sub('.*:', '', entry)

        experiment_key = re.sub('/.*', '', external)
        external_tag = re.sub('.*/', '', external)
        experiment = db.get_experiment(experiment_key, getinfo=False)

        retval[tag] = {
            'key': experiment.artifacts[external_tag].key,
            'mutable': False
        }
    return retval


def _setup_artifacts_creds(artifacts, storage_creds):
    for tag, art_dict in artifacts.items():
        if 'local' in art_dict.keys():
            art_dict[KEY_CREDENTIALS] = copy.deepcopy(storage_creds)


def _parse_hardware(runner_args, config={}):
    resources_needed = {}
    parse_list = ['gpus', 'cpus', 'ram', 'hdd', 'gpuMem', 'gpuCount']
    for key in parse_list:
        from_args = runner_args.__dict__.get(key)
        from_config = config.get(key, None)
        if from_args is not None:
            resources_needed[key] = from_args
        elif from_config is not None:
            resources_needed[key] = from_config

    return resources_needed


def _add_hyperparam_experiments(
        exec_filename,
        other_args,
        runner_args,
        artifacts,
        resources_needed,
        logger,
        optimizer=None,
        hyperparam_tuples=None):

    experiment_name_base = runner_args.experiment if runner_args.experiment \
        else str(uuid.uuid4())

    project = runner_args.project if runner_args.project else \
        ('hyperparam_' + experiment_name_base)

    workspace_orig = artifacts['workspace']['local'] \
        if 'workspace' in artifacts.keys() else '.'

    ignore_arg = ''
    ignore_filepath = os.path.join(workspace_orig, ".studioml_ignore")
    if os.path.exists(ignore_filepath) and \
            not os.path.isdir(ignore_filepath):
        ignore_arg = "--exclude-from=%s" % ignore_filepath

    def create_experiments(hyperparam_tuples):
        experiments = []
        # experiment_names = {}
        for hyperparam_tuple in hyperparam_tuples:
            experiment_name = experiment_name_base
            experiment_name += "__opt__%s__%s" % (util.rand_string(32),
                                                  int(time.time()))
            experiment_name = experiment_name.replace('.', '_')

            workspace_new = fs_tracker.get_artifact_cache(
                'workspace', experiment_name)

            current_artifacts = artifacts.copy()
            current_artifacts.update({
                'workspace': {
                    'local': workspace_new,
                    'mutable': True
                }
            })

            rsync_cp(workspace_orig, workspace_new, ignore_arg, logger)
            # shutil.copytree(workspace_orig, workspace_new)

            for param_name, param_value in six.iteritems(hyperparam_tuple):
                if isinstance(param_value, np.ndarray):

                    array_filepath = '/tmp/%s.npy' % util.rand_string(32)