<filename>anonymise/anonymise_files.py
import os
import argparse
import pymongo
import glob
import random
import string

from pymongo import MongoClient

from random_id import make_random_ids
from anon import anonymise_files
from get_files import get_files, FileTypeException, VCF_filename, BAM_filename, BAI_filename, FASTQ_filename, FASTQ_SUFFIX


def parse_args():
    parser = argparse.ArgumentParser(description='Copy files and randomize their names')
    parser.add_argument('--request-id',  metavar='data_request_id', type=str, required=True,
                        help='Data request id generated by Data Access and Release website')
    parser.add_argument('--flagship', metavar='flagship_code', type=str, required=True,
                        help='Flagship code')
    parser.add_argument('--data-dir', metavar='source_data_directory', type=str, required=True,
                        help='Full path to the directory')
    parser.add_argument('--release-dir', metavar='release_directory', type=str, required=True,
                        help='Full path to the directory')

    return parser.parse_args()


def init_db():
    conn = MongoClient('localhost', 27017)
    db = conn.data_access

    return db

def get_samples(request_id, flagship):
    """
    Find all the samples from this flagship to be released for this data request

    """
    db = init_db()
    samples = db.data_requests.find(filter={'flagship_id': flagship, 'data_request_id': request_id},
                                    projection=['flagship_id', 'lab_sample_id', 'batch_no'])


    return [sample for sample in samples]

def get_new_batch_ids(samples):
    """
    Given a list of FASTQ filenames, get a list of new anonymised batch IDs

    """
    new_batch_ids = {}
    batch_number = 0
    length = 5
    batch_prefix = 'AGRF_'

    for sample in samples:
        agrf_batch_no = batch_prefix + sample['batch_no']
        if len(new_batch_ids) == 0 or agrf_batch_no not in new_batch_ids:
            batch_number += 1
            new_batch_id = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(length))
            new_batch_ids[agrf_batch_no] = new_batch_id

    return new_batch_ids


def anonymise_fastq_files():
    """
    Anonymise FASTQ files data request per flagship

    """
    options = parse_args()

    data_dir = options.data_dir
    release_dir = options.release_dir
    flagship = options.flagship
    request_id = options.request_id

    sample_ids = []
    file_paths = []
    samples = get_samples(request_id, flagship)
    for sample in samples:
        batch_dir = os.path.join(data_dir, sample['batch_no'], sample['lab_sample_id'])
        sample_files = glob.glob(batch_dir + '*' + FASTQ_SUFFIX)
        if sample_files:
            sample_ids.append(sample['lab_sample_id'])
            file_paths += sample_files
        else:
            print('WARNING: No FASTQ file found for sample ' + sample['lab_sample_id'])

    randomised_ids = make_random_ids('used_random_sample_ids.db', sample_ids)
    randomised_files = anonymise_files(file_paths, randomised_ids, release_dir, FASTQ_filename)

    return randomised_files


if __name__ == '__main__':
    anonymise_fastq_files()

<filename>anonymise/anonymise_files.py
import os
import argparse
import pymongo
import glob
import random
import string

from pymongo import MongoClient

from random_id import make_random_ids
from anon import anonymise_files
from get_files import get_files, FileTypeException, VCF_filename, BAM_filename, BAI_filename, FASTQ_filename, FASTQ_SUFFIX


def parse_args():
    parser = argparse.ArgumentParser(description='Copy files and randomize their names')
    parser.add_argument('--request-id',  metavar='data_request_id', type=str, required=True,
                        help='Data request id generated by Data Access and Release website')
    parser.add_argument('--flagship', metavar='flagship_code', type=str, required=True,
                        help='Flagship code')
    parser.add_argument('--data-dir', metavar='source_data_directory', type=str, required=True,
                        help='Full path to the directory')
    parser.add_argument('--release-dir', metavar='release_directory', type=str, required=True,
                        help='Full path to the directory')

    return parser.parse_args()


def init_db():
    conn = MongoClient('localhost', 27017)
    db = conn.data_access

    return db

def get_samples(request_id, flagship):
    """
    Find all the samples from this flagship to be released for this data request

    """
    db = init_db()
    samples = db.data_requests.find(filter={'flagship_id': flagship, 'data_request_id': request_id},
                                    projection=['flagship_id', 'lab_sample_id', 'batch_no'])


    return [sample for sample in samples]

def get_new_batch_ids(samples):
    """
    Given a list of FASTQ filenames, get a list of new anonymised batch IDs

    """
    new_batch_ids = {}
    batch_number = 0
    length = 5
    batch_prefix = 'AGRF_'

    for sample in samples:
        agrf_batch_no = batch_prefix + sample['batch_no']
        if len(new_batch_ids) == 0 or agrf_batch_no not in new_batch_ids:
            batch_number += 1
            new_batch_id = ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(length))
            new_batch_ids[agrf_batch_no] = new_batch_id

    return new_batch_ids


def anonymise_fastq_files():
    """
    Anonymise FASTQ files data request per flagship

    """
    options = parse_args()

    data_dir = options.data_dir
    release_dir = options.release_dir
    flagship = options.flagship
    request_id = options.request_id

    sample_ids = []
    file_paths = []
    samples = get_samples(request_id, flagship)
    for sample in samples:
        batch_dir = os.path.join(data_dir, sample['batch_no'], sample['lab_sample_id'])
        sample_files = glob.glob(batch_dir + '*' + FASTQ_SUFFIX)
        if sample_files:
            sample_ids.append(sample['lab_sample_id'])
            file_paths += sample_files
        else:
            print('WARNING: No FASTQ file found for sample ' + sample['lab_sample_id'])

    randomised_ids = make_random_ids('used_random_sample_ids.db', sample_ids)
    randomised_files = anonymise_files(file_paths, randomised_ids, release_dir, FASTQ_filename)

    return randomised_files


if __name__ == '__main__':
    anonymise_fastq_files()

Find all the samples from this flagship to be released for this data request Given a list of FASTQ filenames, get a list of new anonymised batch IDs Anonymise FASTQ files data request per flagship