<reponame>tky823/pytorch_pneumonia-detection<gh_stars>0
#!/usr/bin/env python3

import argparse
import os

import pandas as pd

parser = argparse.ArgumentParser(description="Prepare NIH dataset.")

parser.add_argument('--ignore_labels', type=str, default="[]", help='Labels to ignore.')
parser.add_argument('--csv_path', type=str, default=None, help='Path to entry.csv')
parser.add_argument('--train_valid_list_path', type=str, default=None, help='Path to train_val_list.txt')
parser.add_argument('--test_list_path', type=str, default=None, help='Path to test_list.txt')
parser.add_argument('--train_txt_path', type=str, default=None, help='Path to train.txt')
parser.add_argument('--valid_txt_path', type=str, default=None, help='Path to valid.txt')
parser.add_argument('--test_txt_path', type=str, default=None, help='Path to test.txt')
parser.add_argument('--n_train', type=int, default=None, help='# of train data.')
parser.add_argument('--n_valid', type=int, default=None, help='# of validation data.')

def main(args):
    n_train, n_valid = args.n_train, args.n_valid
    train_valid_list_path = args.train_valid_list_path
    csv_path = args.csv_path

    ignore_labels = args.ignore_labels.replace('[', '').replace(']', '').split(',')

    df = pd.read_csv(csv_path)

    filenames = []
    with open(train_valid_list_path) as f:
        for line_idx, line in enumerate(f):
            filename = line.strip()
            filenames.append(filename)

    target_df = pd.DataFrame({
        'Image Index': filenames
    })

    entry = pd.merge(df, target_df, on='Image Index', how='inner')

    filenames = []
    for idx, data in entry.iterrows():
        filename = data['Image Index']
        _finding_labels = data['Finding Labels']
        _finding_labels = _finding_labels.split('|')
        if len(_finding_labels) > 1:
            continue
        _finding_labels = _finding_labels[0]
        if _finding_labels in ignore_labels:
            continue
        filenames.append(filename)
    
    train_filenames, valid_filenames = filenames[:n_train], filenames[n_train: n_train + n_valid]
    
    print("Training: {} samples".format(len(train_filenames)))
    print("Validation: {} samples".format(len(valid_filenames)))

    train_txt_path = args.train_txt_path
    valid_txt_path = args.valid_txt_path

    train_txt_root = os.path.dirname(train_txt_path)
    valid_txt_root = os.path.dirname(valid_txt_path)
    os.makedirs(train_txt_root, exist_ok=True)
    os.makedirs(valid_txt_root, exist_ok=True)
    
    with open(train_txt_path, 'w') as f:
        for line in train_filenames:
            f.write(line + '\n')

    with open(valid_txt_path, 'w') as f:
        for line in valid_filenames:
            f.write(line + '\n')
    
    # Test
    test_list_path = args.test_list_path

    filenames = []
    with open(test_list_path) as f:
        for line_idx, line in enumerate(f):
            filename = line.strip()
            filenames.append(filename)

    target_df = pd.DataFrame({
        'Image Index': filenames
    })

    entry = pd.merge(df, target_df, on='Image Index', how='inner')

    test_filenames = []
    for idx, data in entry.iterrows():
        filename = data['Image Index']
        _finding_labels = data['Finding Labels']
        _finding_labels = _finding_labels.split('|')
        if len(_finding_labels) > 1:
            continue
        _finding_labels = _finding_labels[0]
        if _finding_labels in ignore_labels:
            continue
        test_filenames.append(filename)
    
    print("Test: {} samples".format(len(test_filenames)))

    test_txt_path = args.test_txt_path

    test_txt_root = os.path.dirname(test_txt_path)
    os.makedirs(test_txt_root, exist_ok=True)

    with open(test_txt_path, 'w') as f:
        for line in test_filenames:
            f.write(line + '\n')

if __name__ == '__main__':
    args = parser.parse_args()
    print(args)
    main(args)
#!/usr/bin/env python3

import argparse
import os

import pandas as pd

parser = argparse.ArgumentParser(description="Prepare NIH dataset.")

parser.add_argument('--ignore_labels', type=str, default="[]", help='Labels to ignore.')
parser.add_argument('--csv_path', type=str, default=None, help='Path to entry.csv')
parser.add_argument('--train_valid_list_path', type=str, default=None, help='Path to train_val_list.txt')
parser.add_argument('--test_list_path', type=str, default=None, help='Path to test_list.txt')
parser.add_argument('--train_txt_path', type=str, default=None, help='Path to train.txt')
parser.add_argument('--valid_txt_path', type=str, default=None, help='Path to valid.txt')
parser.add_argument('--test_txt_path', type=str, default=None, help='Path to test.txt')
parser.add_argument('--n_train', type=int, default=None, help='# of train data.')
parser.add_argument('--n_valid', type=int, default=None, help='# of validation data.')

def main(args):
    n_train, n_valid = args.n_train, args.n_valid
    train_valid_list_path = args.train_valid_list_path
    csv_path = args.csv_path

    ignore_labels = args.ignore_labels.replace('[', '').replace(']', '').split(',')

    df = pd.read_csv(csv_path)

    filenames = []
    with open(train_valid_list_path) as f:
        for line_idx, line in enumerate(f):
            filename = line.strip()
            filenames.append(filename)

    target_df = pd.DataFrame({
        'Image Index': filenames
    })

    entry = pd.merge(df, target_df, on='Image Index', how='inner')

    filenames = []
    for idx, data in entry.iterrows():
        filename = data['Image Index']
        _finding_labels = data['Finding Labels']
        _finding_labels = _finding_labels.split('|')
        if len(_finding_labels) > 1:
            continue
        _finding_labels = _finding_labels[0]
        if _finding_labels in ignore_labels:
            continue
        filenames.append(filename)
    
    train_filenames, valid_filenames = filenames[:n_train], filenames[n_train: n_train + n_valid]
    
    print("Training: {} samples".format(len(train_filenames)))
    print("Validation: {} samples".format(len(valid_filenames)))

    train_txt_path = args.train_txt_path
    valid_txt_path = args.valid_txt_path

    train_txt_root = os.path.dirname(train_txt_path)
    valid_txt_root = os.path.dirname(valid_txt_path)
    os.makedirs(train_txt_root, exist_ok=True)
    os.makedirs(valid_txt_root, exist_ok=True)
    
    with open(train_txt_path, 'w') as f:
        for line in train_filenames:
            f.write(line + '\n')

    with open(valid_txt_path, 'w') as f:
        for line in valid_filenames:
            f.write(line + '\n')
    
    # Test
    test_list_path = args.test_list_path

    filenames = []
    with open(test_list_path) as f:
        for line_idx, line in enumerate(f):
            filename = line.strip()
            filenames.append(filename)

    target_df = pd.DataFrame({
        'Image Index': filenames
    })

    entry = pd.merge(df, target_df, on='Image Index', how='inner')

    test_filenames = []
    for idx, data in entry.iterrows():
        filename = data['Image Index']
        _finding_labels = data['Finding Labels']
        _finding_labels = _finding_labels.split('|')
        if len(_finding_labels) > 1:
            continue
        _finding_labels = _finding_labels[0]
        if _finding_labels in ignore_labels:
            continue
        test_filenames.append(filename)
    
    print("Test: {} samples".format(len(test_filenames)))

    test_txt_path = args.test_txt_path

    test_txt_root = os.path.dirname(test_txt_path)
    os.makedirs(test_txt_root, exist_ok=True)

    with open(test_txt_path, 'w') as f:
        for line in test_filenames:
            f.write(line + '\n')

if __name__ == '__main__':
    args = parser.parse_args()
    print(args)
    main(args)
#!/usr/bin/env python3 # Test