class Api(object):
    """One Codex Base API object class.

    Instantiates a Potion-Client object under the hood for making requests.
    """

    def __init__(
        self,
        api_key=None,
        bearer_token=None,
        cache_schema=True,
        base_url=None,
        telemetry=None,
        schema_path="/api/v1/schema",
        load_extensions=True,
        **kwargs
    ):

        if base_url is None:
            base_url = os.environ.get("ONE_CODEX_API_BASE", "https://app.onecodex.com")
            if base_url != "https://app.onecodex.com":
                warnings.warn("Using base API URL: {}".format(base_url))

        if kwargs.get("experimental", False):
            warnings.warn(
                "Experimental API mode enabled. Features of the experimental API are subject to "
                "change without notice and should not be relied upon in a production environment."
            )
            schema_path = "/api/v1_experimental/schema"
            cache_schema = False

        self._req_args = {}
        self._base_url = base_url
        self._schema_path = schema_path

        # Attempt to automatically fetch API key from
        # ~/.onecodex file, API key, or bearer token environment vars
        # *if and only if* no auth is explicitly passed to Api
        #
        # TODO: Consider only doing this if an add'l env var like
        #       'ONE_CODEX_AUTO_LOGIN' or similar is set.
        if api_key is None and bearer_token is None:
            creds_file = os.path.expanduser("~/.onecodex")

            if not os.path.exists(creds_file):
                pass
            elif not os.access(creds_file, os.R_OK):
                warnings.warn("Check permissions on {}".format(collapse_user(creds_file)))
            else:
                try:
                    with filelock.FileLock("{}.lock".format(creds_file)):
                        api_key = json.load(open(creds_file, "r"))["api_key"]
                except KeyError:
                    # lacking an api_key doesn't mean the file is corrupt--it can just be that the
                    # schema was cached after logging in anonymously
                    pass
                except ValueError:
                    warnings.warn(
                        "Credentials file ({}) is corrupt".format(collapse_user(creds_file))
                    )

            if api_key is None:
                api_key = os.environ.get("ONE_CODEX_API_KEY")
            if bearer_token is None:
                bearer_token = os.environ.get("ONE_CODEX_BEARER_TOKEN")

        if bearer_token:  # prefer bearer token where available
            self._req_args["auth"] = BearerTokenAuth(bearer_token)
        elif api_key:
            self._req_args["auth"] = HTTPBasicAuth(api_key, "")

        self._req_args["headers"] = {"X-OneCodex-Client-User-Agent": __version__}

        # Create client instance
        self._client = ExtendedPotionClient(
            self._base_url, schema_path=self._schema_path, fetch_schema=False, **self._req_args
        )
        self._client._fetch_schema(cache_schema=cache_schema)
        self._session = self._client.session
        self._copy_resources()

        # Optionally configure custom One Codex altair theme
        if load_extensions:
            from onecodex.viz import configure_onecodex_theme

            configure_onecodex_theme()

        # Optionally configure Sentry
        if telemetry is True or (
            telemetry is None and os.environ.get("ONE_CODEX_AUTO_TELEMETRY", False)
        ):
            init_sentry(user_context={"email": self._fetch_account_email()})
            self._telemetry = True
        else:
            self._telemetry = False

    def _fetch_account_email(self):
        creds_file = os.path.expanduser("~/.onecodex")

        if not os.path.exists(creds_file):
            pass
        elif not os.access(creds_file, os.R_OK):
            warnings.warn("Check permissions on {}".format(collapse_user(creds_file)))
        else:
            try:
                return json.load(open(creds_file, "r"))["email"]
            except KeyError:
                pass
            except ValueError:
                warnings.warn("Credentials file ({}) is corrupt".format(collapse_user(creds_file)))

        return os.environ.get("ONE_CODEX_USER_EMAIL", os.environ.get("ONE_CODEX_USER_UUID"))

    def _copy_resources(self):
        """Copy subclassed potion resources into this instance of Api().

        Notes
        -----
        Will only make available any objects that were listed in the REST API schema, either cached
        or retrieved from the server. If a non-standard schema_path was given, make sure we match
        e.g., /api/v1_experimental/samples up with /api/v1/samples.

        Returns
        -------
        `None`
        """
        from onecodex.models import _model_lookup

        if self._schema_path != "/api/v1/schema":
            new_schema_base = self._schema_path.rstrip("schema")

            aliased_resources = {}

            for resource, oc_cls in _model_lookup.items():
                aliased_resources[resource.replace("/api/v1/", new_schema_base)] = oc_cls

            _model_lookup.update(aliased_resources)

        for resource in self._client._resources:
            # set the name param, the keys now have / in them
            potion_resource = self._client._resources[resource]

            try:
                oc_cls = _model_lookup[resource]
                oc_cls._api = self
                oc_cls._resource = potion_resource
                setattr(self, oc_cls.__name__, oc_cls)
            except KeyError:  # Ignore resources we don't explicitly model
                pass def _login(server, creds_file=None, api_key=None, silent=False):
    """Login main function."""
    # fetch_api_key and check_version expect server to end in /
    if server[-1] != "/":
        server = server + "/"

    # creds file path setup
    if creds_file is None:
        creds_file = os.path.expanduser("~/.onecodex")

    # check if the creds file exists and is readable
    if not os.path.exists(creds_file):
        if silent:
            return None

        creds = {}
    elif not os.access(creds_file, os.R_OK):
        click.echo("Please check the permissions on {}".format(collapse_user(creds_file)), err=True)
        sys.exit(1)
    else:
        with filelock.FileLock("{}.lock".format(creds_file)):
            # it is, so let's read it!
            with open(creds_file, "r") as fp:
                try:
                    creds = json.load(fp)
                except ValueError:
                    click.echo(
                        "Your ~/.onecodex credentials file appears to be corrupted. "  # noqa
                        "Please delete it and re-authorize.",
                        err=True,
                    )
                    sys.exit(1)

            # check for updates if logged in more than one day ago
            last_update = creds.get("updated_at") or creds.get("saved_at")
            last_update = (
                last_update if last_update else datetime.datetime.now().strftime(DATE_FORMAT)
            )
            diff = datetime.datetime.now() - datetime.datetime.strptime(last_update, DATE_FORMAT)

            if diff.days >= 1:
                # if creds_file is old, check for updates
                upgrade_required, msg = check_version(__version__, server)
                creds["updated_at"] = datetime.datetime.now().strftime(DATE_FORMAT)

                try:
                    json.dump(creds, open(creds_file, "w"))
                except Exception as e:
                    if e.errno == errno.EACCES:
                        click.echo(
                            "Please check the permissions on {}".format(collapse_user(creds_file)),
                            err=True,
                        )
                        sys.exit(1)
                    else:
                        raise

                if upgrade_required:
                    click.echo("\nWARNING: {}\n".format(msg), err=True)

            # finally, give the user back what they want (whether silent or not)
            if silent:
                return creds.get("api_key", None)

            click.echo(
                "Credentials file already exists ({}). Logout first.".format(
                    collapse_user(creds_file)
                ),
                err=True,
            )
            return creds.get("email", None)

    # creds_file was not found and we're not silent, so prompt user to login
    email, api_key = login_uname_pwd(server, api_key=api_key)

    if api_key is None:
        click.echo(
            "We could not verify your credentials. Please check your email and password again. "
            "If you have single sign-on (SSO) enabled, you must login with `onecodex --api-key $API_KEY login`. "
            "Please contact help@onecodex.com if you continue to experience problems."
        )
        sys.exit(1)

    creds.update(
        {
            "api_key": api_key,
            "saved_at": datetime.datetime.now().strftime(DATE_FORMAT),
            "updated_at": None,
            "email": email,
        }
    )

    with filelock.FileLock("{}.lock".format(creds_file)):
        try:
            json.dump(creds, open(creds_file, "w"))
        except Exception as e:
            if e.errno == errno.EACCES:
                click.echo("Please check the permissions on {}".format(creds_file), err=True)
                sys.exit(1)
            else:
                raise

    click.echo("Your ~/.onecodex credentials file was successfully created.", err=True)

    return email def _logout(creds_file=None):
    """Logout main function, just rm ~/.onecodex more or less."""
    if _remove_creds(creds_file=creds_file):
        click.echo("Successfully removed One Codex credentials.", err=True)
        sys.exit(0)
    else:
        click.echo("No One Codex API keys found.", err=True)
        sys.exit(1) def _remove_creds(creds_file=None):
    """Remove ~/.onecodex file, returning True if successul or False if the file didn't exist."""
    if creds_file is None:
        creds_file = os.path.expanduser("~/.onecodex")

    try:
        os.remove(creds_file)
    except Exception as e:
        if e.errno == errno.ENOENT:
            return False
        elif e.errno == errno.EACCES:
            click.echo(
                "Please check the permissions on {}".format(collapse_user(creds_file)), err=True
            )
            sys.exit(1)
        else:
            raise

    try:
        os.remove("{}.lock".format(creds_file))
    except (IOError, OSError):
        pass

    return True def login_required(fn):
    """Require login before proceeding, but does not prompt the user to login.

    Decorator should be used only on Click CLI commands.

    Notes
    -----
    Different means of authentication will be attempted in this order:
        1. An API key present in the Click context object from a previous successful authentication.
        2. A bearer token (ONE_CODEX_BEARER_TOKEN) in the environment.
        3. An API key (ONE_CODEX_API_KEY) in the environment.
        4. An API key in the credentials file (~/.onecodex).
    """

    @wraps(fn)
    def login_wrapper(ctx, *args, **kwargs):
        base_url = os.environ.get("ONE_CODEX_API_BASE", "https://app.onecodex.com")

        api_kwargs = {"telemetry": ctx.obj["TELEMETRY"], "load_extensions": False}

        api_key_prior_login = ctx.obj.get("API_KEY")
        bearer_token_env = os.environ.get("ONE_CODEX_BEARER_TOKEN")
        api_key_env = os.environ.get("ONE_CODEX_API_KEY")
        api_key_creds_file = _login(base_url, silent=True)

        if api_key_prior_login is not None:
            api_kwargs["api_key"] = api_key_prior_login
        elif bearer_token_env is not None:
            api_kwargs["bearer_token"] = bearer_token_env
        elif api_key_env is not None:
            api_kwargs["api_key"] = api_key_env
        elif api_key_creds_file is not None:
            api_kwargs["api_key"] = api_key_creds_file
        else:
            click.echo(
                "The command you specified requires authentication. Please login first.\n", err=True
            )
            ctx.exit(1)

        ctx.obj["API"] = Api(**api_kwargs)

        return fn(ctx, *args, **kwargs)

    return login_wrapper DEFAULT_THREADS = 4 def validate_appendables(appendables, api):
    appendables["valid_tags"] = []
    appendables["valid_metadata"] = {"custom": {}}
    validate_tags(appendables, api)
    validate_metadata(appendables, api)
    return appendables def fastfastq(file_path):
def validating_parser(file_path, **io_kwargs):
def get_filtered_filename(file_path):
def make_taxonomy_dict(classification, parent=False):
def recurse_taxonomy_map(tax_id_map, tax_id, parent=False):
        def _child_recurse(tax_id, visited):
def too_many_fastx_records():
def cli(
    ctx,
    classification_id,
    fastx,
    reverse,
    tax_ids,
    with_children,
    subset_pairs_independently,
    exclude_reads,
    include_lowconf,
    out,
    validate,
): def click_path_autocomplete_helper(ctx, args, incomplete, filename=True, directory=True):
    """Suggest paths to complete a partially typed filename or directory.

    Parameters
    ----------
    ctx : `click.Context`
    args : `list`
    incomplete : `str`
        A partially typed path to a file or directory.
    filename : `bool`
        If True, include filenames in the list of suggestions.
    directory : `bool`
        If True, include directories in the list of suggestions.

    Returns
    -------
    A list of suggestions for completing the path.
    """
    dir_name = os.path.dirname(incomplete)
    file_name = os.path.basename(incomplete)

    if dir_name.startswith("~"):
        abs_dir_name = os.path.expanduser(dir_name)
    else:
        abs_dir_name = os.path.abspath(dir_name)

    if not os.path.exists(abs_dir_name):
        return []

    suggestions = []

    for item in os.listdir(abs_dir_name):
        if item.startswith(file_name):
            abs_item_path = os.path.join(abs_dir_name, item)
            item_path = os.path.join(dir_name, item)

            if directory and os.path.isdir(abs_item_path):
                suggestions.append(item_path)
            elif filename and os.path.isfile(abs_item_path):
                suggestions.append(item_path)

    return suggestions def cli_resource_fetcher(ctx, resource, uris, print_results=True):
    try:
        # analyses is passed, want Analyses
        resource_name = resource[0].upper() + resource[1:]
        if len(uris) == 0:

            # if non given fetch all
            log.debug("No %s IDs given, fetching all...", resource_name)
            instances = getattr(ctx.obj["API"], resource_name).all()
            log.debug("Fetched %i %ss", len(instances), resource)
            objs_to_return = [x._resource._properties for x in instances]
        else:
            uris = list(set(uris))
            log.debug("Fetching %s: %s", resource_name, ",".join(uris))

            instances = []
            for uri in uris:
                try:
                    instance = getattr(ctx.obj["API"], resource_name).get(uri)
                    if instance is not None:
                        instances.append(instance._resource._properties)
                    else:
                        log.error(
                            "Could not find {} {} (404 status code)".format(resource_name, uri)
                        )
                except requests.exceptions.HTTPError as e:
                    log.error(
                        "Could not find {} {} ({} status code)".format(
                            resource_name, uri, e.response.status_code
                        )
                    )
            objs_to_return = instances

        if print_results:
            pprint(objs_to_return, ctx.obj["NOPPRINT"])
        else:
            return objs_to_return
    except requests.exceptions.HTTPError:
        click.echo(
            "Failed to authenticate. Please check your API key "
            "or trying logging out and back in with `onecodex logout` "
            "and `onecodex login`."
        ) class CliLogFormatter(logging.Formatter):
    formats = {logging.DEBUG: "DEBUG: %(module)s: %(lineno)d: %(msg)s", logging.INFO: "\n%(msg)s"}

    def __init__(self):
        # Note: only style="%" is supported in Python 2 and so we cannot pass style= here
        super(CliLogFormatter, self).__init__(fmt="\n%(levelname)s: %(msg)s", datefmt=None)
        self.original_format = self._style._fmt if hasattr(self, "_style") else self._fmt

    def format(self, record):
        if hasattr(self, "_style"):
            # Python 3
            self._style._fmt = self.formats.get(record.levelno, self.original_format)
        else:
            # Python 3
            self._fmt = self.formats.get(record.levelno, self.original_format)
        result = logging.Formatter.format(self, record)
        return result def download_file_helper(url, input_path):
    """Manage the chunked downloading of a file given an url."""
    r = requests.get(url, stream=True)
    if r.status_code != 200:
        log.error("Failed to download file: %s" % r.json()["message"])
    local_full_path = get_download_dest(input_path, r.url)
    original_filename = os.path.split(local_full_path)[-1]
    with open(local_full_path, "wb") as f:
        click.echo("Downloading {}".format(original_filename), err=True)
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()
    pprint("Successfully downloaded %s to %s" % (original_filename, local_full_path), True)  # noqa def valid_api_key(ctx, param, value):
    """Ensure an API has valid length (this is a click callback)."""
    if value is not None and len(value) != 32:
        raise click.BadParameter(
            "API Key must be 32 characters long, not {}".format(str(len(value)))
        )
    else:
        return value OPTION_HELP = {
    "api_key": "Manually provide a One Codex API key",
    "no_pprint": "Do not pretty-print JSON responses",
    "threads": "Do not use multiple background threads to upload files",  # noqa
    "max_threads": "Specify a different max # of upload threads (defaults to 4)",  # noqa
    "verbose": "Log extra information to STDERR",
    "results": "Get a JSON array of the metagenomic classification results table",
    "readlevel": "Get the read-level data as a .tsv file",
    "readlevel_path": (
        "Output path or directory for a .tsv file with the raw "
        "read-level results. Defaults to original filename in "
        "the current working directory"
    ),
    "clean": (
        "Automatically clean up FASTX records during upload. This removes tabs from "
        "headers and converts U to T. If this isn't passed, these will cause errors."
    ),
    "interleave": (
        "Do not automatically interleave paired end files. Note this normally happens "
        "during the upload, does not require additional disk space, and does not change "
        "the files themselves."
    ),
    "prompt": (
        "Manually prompt about automatic paired file interleaving. Setting --no-prompt "
        "will allow running without any user intervention, e.g. in a script."
    ),
    "validate": (
        "Do not validate the FASTA/Q file before uploading. Incompatible with automatic "
        "paired end interleaving (NOT RECOMMENDED)."
    ),
    "telemetry": "Send errors to One Codex?",
    "forward": "Specify a forward reads file",
    "reverse": "Specify a reverse reads file",
    "tag": (
        "NOTE: We recommend invoking the upload command separately for each "
        "sample to add sample-specific tags.\n\nAdd one or more tags to "
        'all uploaded samples e.g., `onecodex upload --tag "Cohort A" $FILE`.'
    ),
    "metadata": (
        "NOTE: We recommend invoking the upload command separately for each "
        "sample to add sample-specific metadata.\n\nAdd one or more "
        "metadata attributes to all uploaded samples, "
        "e.g. `onecodex upload --metadata starred=true --metadata "
        'platform="Illumina MiSeq" $FILE`. '
    ),
    "project": "Provide the name, short name, or alphanumeric UUID of a "
    "project to automatically add the samples to that project on "
    "upload. NOTE: does not currently support adding a sample to "
    "a public project. Projects are searched by UUID, then name, "
    "then short name in that order.",
    "sample_id": "Provide an ID for a sample that was previously 'pre-uploaded' along with metadata.",
    "external_sample_id": "Provide an external sample ID for a sample that was previously 'pre-uploaded' along with metadata.",
} def progressbar(*args, **kwargs):
    bar = click.progressbar(*args, **kwargs)
    bar._update = bar.update

    def update(self, value):
        if getattr(self, "canceled", False) is True:
            self.label = "Canceling..."
            if self.eta_known:
                self.eta_known = 0
                self.pos = 0
                self._update(1)
        elif self.pct >= 0.98 and self.label.startswith("Uploading"):
            self.label = "Finalizing..."
            self._update(value)
        else:
            self._update(value)

    bar.update = partial(update, bar)
    return bar def pprint(j, no_pretty):
    """Print as formatted JSON."""
    if not no_pretty:
        click.echo(
            json.dumps(j, cls=PotionJSONEncoder, sort_keys=True, indent=4, separators=(",", ": "))
        )
    else:
        click.echo(j) def pretty_errors(fn):
    """Decorate CLI functions, catching errors and then calling sys.exit(1).

    This is intended for use with the CLI and scripts where we only use OneCodexException and its
    subclasses, e.g. ValidationError or UploadException.
    """

    @wraps(fn)
    def pretty_errors_wrapper(*args, **kwargs):
        try:
            fn(*args, **kwargs)
        except (OneCodexException, UploadException) as e:
            sys.stderr.write("\nERROR: {}".format(e))
            sys.stderr.write("\nPlease feel free to contact us for help at help@onecodex.com\n\n")
            sys.exit(1)

    return pretty_errors_wrapper def run_via_threadpool(fn, iterable, fn_kwargs, max_threads=1, graceful_exit=False):
    if max_threads == 1:
        for item in iterable:
            fn(item, **fn_kwargs)
        return

    with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor:
        futures = {executor.submit(fn, x, **fn_kwargs) for x in iterable}

        try:
            for future in concurrent.futures.as_completed(futures):
                try:
                    future.result()
                except Exception as e:
                    raise e
                else:
                    pass
        except KeyboardInterrupt as k:
            if not graceful_exit:
                executor._threads.clear()
                concurrent.futures.thread._threads_queues.clear()
            raise k def warn_if_insecure_platform():
    """Produce a nice message if SSLContext object is not available.

    Returns
    -------
    `True` if platform is insecure, `False` if platform is secure.
    """
    m = (
        "\n"
        "######################################################################################\n"  # noqa
        "#                                                                                    #\n"  # noqa
        "#  Your version of Python appears to be out of date and lack important security      #\n"  # noqa
        "#  features. Please update to Python >= 2.7.9 or `pip install requests[security]`.   #\n"  # noqa
        "#                                                                                    #\n"  # noqa
        "#  InsecurePlatformWarning: A true SSLContext object is not available. This          #\n"  # noqa
        "#  prevents urllib3 from configuring SSL appropriately and may cause certain         #\n"  # noqa
        "#  SSL connections to fail. For more information, see                                #\n"  # noqa
        "#  https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.  #\n"  # noqa
        "#                                                                                    #\n"  # noqa
        "######################################################################################\n"
    )  # noqa
    if is_insecure_platform():
        click.echo(m, err=True)
        return True
    else:
        log.debug("Python SSLContext passed")
        return False def telemetry(fn):
    """Decorate CLI and other functions that need special Sentry client handling.

    This decorator is only required on functions that:
        1) Specifically catch and re-raise exceptions.
        2) Call sys.exit() directly.
    """

    @wraps(fn)
    def telemetry_wrapper(*args, **kwargs):
        try:
            return fn(*args, **kwargs)
        except Exception as e:
            sentry_sdk.capture_exception(e)
            with sentry_sdk.configure_scope() as scope:
                scope.clear()
            raise e

    return telemetry_wrapper def auto_detect_pairs(files, prompt):
    """Group paired-end files into tuples in the files list.

    Returns the files list with paired-end files represented as tuples on that list.
    If `prompt` is set to True, the user is asked whether this should happen first.
    """

    # "intelligently" find paired files and tuple them
    paired_files = []
    single_files = set(files)

    for filename in files:
        # convert "read 1" filenames into "read 2" and check that they exist; if they do
        # upload the files as a pair, autointerleaving them
        paired_filename = re.sub(r"([._][Rr])1([._][\w.]+)$", r"\g<1>2\g<2>", filename)
        paired_filename = re.sub(r"([._])1([._][\D._]+)$", r"\g<1>2\g<2>", paired_filename)

        # we don't necessary need the R2 to have been passed in; we infer it anyways
        if paired_filename != filename and os.path.exists(paired_filename):
            if not prompt and paired_filename not in single_files:
                # if we're not prompting, don't automatically pull in files
                # not in the list the user passed in
                continue

            paired_files.append((filename, paired_filename))

            if paired_filename in single_files:
                single_files.remove(paired_filename)

            single_files.remove(filename)

    auto_pair = True

    if prompt and len(paired_files) > 0:
        pair_list = ""
        for p in paired_files:
            pair_list += "\n  {}  &  {}".format(os.path.basename(p[0]), os.path.basename(p[1]))

        answer = click.confirm(
            "It appears there are {n_paired_files} paired files (of {n_files} total):{pair_list}\nInterleave them after upload?".format(
                n_paired_files=len(paired_files) * 2,
                n_files=len(paired_files) * 2 + len(single_files),
                pair_list=pair_list,
            ),
            default="Y",
        )

        if not answer:
            auto_pair = False

    if auto_pair:
        return paired_files + list(single_files)
    else:
        return files def concatenate_multilane_files(files, prompt):
    """Concatenate multilane files before uploading.

    The files are grouped based on filename. If `prompt` is set to True, the user
    is asked whether this should happen first.

    The concatenated files replace the matched sequence files. They're put in a temporary
    directory and overwrite any existing files.

    Returns a new list with multilane groups replaced with a path to the single
    concatenated file.
    """

    def _concatenate_group(group, first_elem):
        """Concatenate all the files on the list and return the target file path."""
        target_file_name = re.sub(pattern_lane_num, r"\1", os.path.basename(first_elem))
        target_path = os.path.join(tempfile.gettempdir(), target_file_name)

        # Overwriting all files by default
        with open(target_path, "wb") as outf:
            for fname in group:
                with open(fname, "rb") as inf:
                    # TODO: check for newline at the end of file first?
                    shutil.copyfileobj(inf, outf)
        return target_path

    groups = _find_multilane_groups(files)

    if not groups:
        return files

    perform_concat = True
    if prompt:
        answer = click.confirm(
            "This data appears to have been split across multiple sequencing lanes.\nConcatenate lanes before upload?",
            default="Y",
        )
        if not answer:
            perform_concat = False

    if not perform_concat:
        return files

    files = files[:]
    pattern_lane_num = re.compile(r"[._]L\d+([._])")

    for group in groups:
        # The groups considered here will already have more than 1 element
        first_elem = group[0]
        if isinstance(first_elem, tuple):
            concat_fwd = _concatenate_group([fwd for fwd, _ in group], first_elem[0])
            concat_rev = _concatenate_group([rev for _, rev in group], first_elem[1])
            files.append((concat_fwd, concat_rev))
        elif isinstance(first_elem, str):
            concat = _concatenate_group(group, first_elem)
            files.append(concat)

        for elem in group:
            files.remove(elem)

    return files
import click
import copy
import dateutil
import logging
import os
import time
import warnings
from functools import partial
from onecodex.api import Api
from onecodex.auth import _login, _logout, _remove_creds, login_required
from onecodex.lib.upload import DEFAULT_THREADS
from onecodex.metadata_upload import validate_appendables
from onecodex.scripts import subset_reads
from onecodex.utils import (
    click_path_autocomplete_helper,
    cli_resource_fetcher,
    CliLogFormatter,
    download_file_helper,
    valid_api_key,
    OPTION_HELP,
    progressbar,
    pprint,
    pretty_errors,
    run_via_threadpool,
    warn_if_insecure_platform,
    telemetry,
)
from onecodex.input_helpers import auto_detect_pairs, concatenate_multilane_files
from onecodex.version import __version__
            fname = doc["filename"]
            owner = doc["uploader"].email
            table.append(
                [
                    doc["$uri"].split("/")[-1],
                    fname if len(fname) <= 32 else fname[:29] + "...",
                    owner if len(owner) <= 23 else owner[:20] + "...",
                    _size_formatter(doc["size"]) if doc["size"] else "N/A",
                    dateutil.parser.parse(doc["created_at"]).strftime("%Y-%m-%d"),
                ]
            )

        for row in table:
            formatted_row = []
            for formatter, content in zip(formatters, row):
                formatted_row.append(formatter % content)
            click.echo("".join(formatted_row))


@click.command("upload", help="Upload a file to the Document Portal")
@click.option(
    "--max-threads",
    default=DEFAULT_THREADS,
    help=OPTION_HELP["max_threads"],
    metavar="<int:threads>",
)
@click.argument(
    "files",
    nargs=-1,
    required=False,
    type=click.Path(exists=True),
    autocompletion=partial(click_path_autocomplete_helper, directory=False),
)
@click.pass_context
@pretty_errors
@telemetry
@login_required
def documents_upload(ctx, max_threads, files):
    """Upload a document file (of any type) to One Codex."""
    if len(files) == 0:
        click.echo(ctx.get_help())
        return

    files = list(files)

    bar = click.progressbar(length=sum([os.path.getsize(x) for x in files]), label="Uploading... ")
    run_via_threadpool(
        ctx.obj["API"].Documents.upload,
        files,
        {"progressbar": bar},
        max_threads=8 if max_threads > 8 else max_threads,
        graceful_exit=False,
    )


@click.command("download", help="Download a file that has been shared with you")
@click.argument("file_id", nargs=1, required=True)
@click.option(
    "--output-document",
    "-O",
    "file_path",
    help="Write document to PATH",
    required=False,
    type=click.Path(dir_okay=False, allow_dash=True),
    autocompletion=partial(click_path_autocomplete_helper, filename=False),
)
@click.pass_context
@pretty_errors
@telemetry
@login_required
def documents_download(ctx, file_id, file_path):
    doc_obj = ctx.obj["API"].Documents.get(file_id)

    if not doc_obj:
        log.error("Could not find document {} (404 status code)".format(file_id))
        ctx.exit(1)

    if file_path == "-" or file_path == "/dev/stdout":
        doc_obj.download(file_obj=open("/dev/stdout", "wb"), progressbar=False)
    else:
        path = doc_obj.download(path=file_path, progressbar=True)
        click.echo("{} saved to {}".format(file_id, path), err=True)


documents.add_command(documents_list, "list")
documents.add_command(documents_upload, "upload")
documents.add_command(documents_download, "download")


# resources
@onecodex.command("analyses")
@click.argument("analyses", nargs=-1, required=False)
@click.pass_context
@telemetry
@login_required
def analyses(ctx, analyses):
    """Retrieve performed analyses."""
    cli_resource_fetcher(ctx, "analyses", analyses)


@onecodex.command("classifications")
@click.option("--read-level", "readlevel", is_flag=True, help=OPTION_HELP["readlevel"])
@click.option(
    "--read-level-path",
    "readlevel_path",
    type=click.Path(),
    default="./",
    help=OPTION_HELP["readlevel_path"],
    autocompletion=partial(click_path_autocomplete_helper, filename=False),
)
@click.option("--results", "results", is_flag=True, help=OPTION_HELP["results"])
@click.pass_context
@click.argument("classifications", nargs=-1, required=False)
@telemetry
@login_required
def classifications(ctx, classifications, results, readlevel, readlevel_path):
    """Retrieve performed metagenomic classifications."""

    # basic operation -- just print
    if not readlevel and not results:

        cli_resource_fetcher(ctx, "classifications", classifications)