Returns dictionary of cluster IDs:seq IDs

    Overall function for reference-based clustering with usearch61

    seq_path:  fasta filepath to be clustered with usearch61
    refseqs_fp: reference fasta filepath, used to cluster sequences against.
    percent_id:  percentage id to cluster at
    rev: enable reverse strand matching for clustering
    save_intermediate_files: Saves intermediate files created during clustering
    minlen: minimum sequence length
    output_dir: directory to output log, OTU mapping, and intermediate files
    remove_usearch_logs: Saves usearch log files
    verbose: print current processing step to stdout
    wordlength: word length to use for clustering
    usearch_fast_cluster: Use usearch61 fast cluster option, not as memory
     efficient as the default cluster_smallmem option, requires sorting by
     length, and does not allow reverse strand matching.
    usearch61_sort_method:  Sort sequences by abundance or length by using
     functionality provided by usearch61, or do not sort by using None option.
    otu_prefix: label to place in front of OTU IDs, used to prevent duplicate
     IDs from appearing with reference based OTU picking.
    usearch61_maxrejects: Number of rejects allowed by usearch61
    usearch61_maxaccepts: Number of accepts allowed by usearch61
    sizeorder: used for clustering based upon abundance of seeds (only applies
     when doing open reference de novo clustering)
    suppress_new_clusters: If True, will allow de novo clustering on top of
     reference clusters.
    threads: Specify number of threads used per core per CPU
    HALT_EXEC: application controller option to halt execution.

    Description of analysis workflows
    ---------------------------------
    closed-reference approach:
      dereplicate sequences first, do reference based clustering,
      merge clusters/failures and dereplicated data,
      write OTU mapping and failures file.

    open-reference approach:
      dereplicate sequences first, do reference based clustering, parse failures,
      sort failures fasta according to chosen method, cluster failures, merge
      reference clustering results/de novo results/dereplicated data, write
      OTU mapping file.

    Dereplication should save processing time for large datasets.

def usearch61_ref_cluster(seq_path,
                          refseqs_fp,
                          percent_id=0.97,
                          rev=False,
                          save_intermediate_files=True,
                          minlen=64,
                          output_dir='.',
                          remove_usearch_logs=False,
                          verbose=False,
                          wordlength=8,
                          usearch_fast_cluster=False,
                          usearch61_sort_method='abundance',
                          otu_prefix="denovo",
                          usearch61_maxrejects=32,
                          usearch61_maxaccepts=1,
                          sizeorder=False,
                          suppress_new_clusters=False,
                          threads=1.0,
                          HALT_EXEC=False
                          ):
    """ Returns dictionary of cluster IDs:seq IDs

    Overall function for reference-based clustering with usearch61

    seq_path:  fasta filepath to be clustered with usearch61
    refseqs_fp: reference fasta filepath, used to cluster sequences against.
    percent_id:  percentage id to cluster at
    rev: enable reverse strand matching for clustering
    save_intermediate_files: Saves intermediate files created during clustering
    minlen: minimum sequence length
    output_dir: directory to output log, OTU mapping, and intermediate files
    remove_usearch_logs: Saves usearch log files
    verbose: print current processing step to stdout
    wordlength: word length to use for clustering
    usearch_fast_cluster: Use usearch61 fast cluster option, not as memory
     efficient as the default cluster_smallmem option, requires sorting by
     length, and does not allow reverse strand matching.
    usearch61_sort_method:  Sort sequences by abundance or length by using
     functionality provided by usearch61, or do not sort by using None option.
    otu_prefix: label to place in front of OTU IDs, used to prevent duplicate
     IDs from appearing with reference based OTU picking.
    usearch61_maxrejects: Number of rejects allowed by usearch61
    usearch61_maxaccepts: Number of accepts allowed by usearch61
    sizeorder: used for clustering based upon abundance of seeds (only applies
     when doing open reference de novo clustering)
    suppress_new_clusters: If True, will allow de novo clustering on top of
     reference clusters.
    threads: Specify number of threads used per core per CPU
    HALT_EXEC: application controller option to halt execution.

    Description of analysis workflows
    ---------------------------------
    closed-reference approach:
      dereplicate sequences first, do reference based clustering,
      merge clusters/failures and dereplicated data,
      write OTU mapping and failures file.

    open-reference approach:
      dereplicate sequences first, do reference based clustering, parse failures,
      sort failures fasta according to chosen method, cluster failures, merge
      reference clustering results/de novo results/dereplicated data, write
      OTU mapping file.

    Dereplication should save processing time for large datasets.

    """

    files_to_remove = []

    # Need absolute paths to avoid potential problems with app controller
    if output_dir:
        output_dir = join(abspath(output_dir), '')

    seq_path = abspath(seq_path)

    try:

        if verbose:
            print "Presorting sequences according to abundance..."
        intermediate_fasta, dereplicated_uc, app_result =\
            sort_by_abundance_usearch61(seq_path, output_dir, rev,
                                        minlen, remove_usearch_logs, HALT_EXEC,
                                        output_fna_filepath=join(
                                            output_dir,
                                            'abundance_sorted.fna'),
                                        output_uc_filepath=join(
                                            output_dir,
                                            'abundance_sorted.uc'),
                                        threads=threads)
        if not save_intermediate_files:
            files_to_remove.append(intermediate_fasta)
            files_to_remove.append(dereplicated_uc)

        if verbose:
            print "Performing reference based clustering..."
        clusters_fp, app_result = usearch61_cluster_ref(intermediate_fasta,
                                                        refseqs_fp, percent_id, rev, minlen, output_dir,
                                                        remove_usearch_logs, wordlength, usearch61_maxrejects,
                                                        usearch61_maxaccepts, HALT_EXEC,
                                                        output_uc_filepath=join(
                                                            output_dir,
                                                            'ref_clustered.uc'),
                                                        threads=threads)
        if not save_intermediate_files:
            files_to_remove.append(clusters_fp)

        clusters, failures =\
            parse_usearch61_clusters(open(clusters_fp, "U"), otu_prefix="",
                                     ref_clustered=True)
        dereplicated_clusters =\
            parse_dereplicated_uc(open(dereplicated_uc, "U"))
        clusters = merge_clusters_dereplicated_seqs(clusters,
                                                    dereplicated_clusters)
        failures = merge_failures_dereplicated_seqs(failures,
                                                    dereplicated_clusters)

        if not suppress_new_clusters and failures:
            if verbose:
                print "Parsing out sequences that failed to cluster..."
            failures_fasta = parse_usearch61_failures(seq_path, set(failures),
                                                      output_fasta_fp=join(output_dir, "failures_parsed.fna"))
            if not save_intermediate_files:
                files_to_remove.append(failures_fasta)
            denovo_clusters = usearch61_denovo_cluster(failures_fasta,
                                                       percent_id, rev, save_intermediate_files, minlen, output_dir,
                                                       remove_usearch_logs, verbose, wordlength, usearch_fast_cluster,
                                                       usearch61_sort_method, otu_prefix, usearch61_maxrejects,
                                                       usearch61_maxaccepts, sizeorder, threads, HALT_EXEC)
            failures = []

            # Merge ref and denovo clusters
            clusters.update(denovo_clusters)

    except ApplicationError:
        raise ApplicationError('Error running usearch61. Possible causes are '
                               'unsupported version (current supported version is usearch '
                               'v6.1.544) is installed or improperly formatted input file was '
                               'provided')

    except ApplicationNotFoundError:
        remove_files(files_to_remove)
        raise ApplicationNotFoundError('usearch61 not found, is it properly '
                                       'installed?')

    if not save_intermediate_files:
        remove_files(files_to_remove)

    return clusters, failures