<reponame>sabuj7177/CovidProject<filename>dataset_preparation/chexpert_dataset.py
import os

import cv2
import numpy as np
import pandas as pd

import dataset_settings
from util import insert_into_df, write_info, resize_image


def prepare_chexpert_data(data_path, train_csv_path, valid_csv_path, source_url):
    chexpert_train_csv = pd.read_csv(train_csv_path, nrows=None)
    chexpert_valid_csv = pd.read_csv(valid_csv_path, nrows=None)
    final_csv = pd.concat([chexpert_train_csv, chexpert_valid_csv])
    final_csv = final_csv[final_csv['Frontal/Lateral'] != 'Lateral']
    normal_csv = final_csv[final_csv['No Finding'] == 1]
    normal_csv = normal_csv.sample(frac=1)
    pneumonia_csv = final_csv[final_csv['Pneumonia'] == 1]
    pneumonia_csv = pneumonia_csv.sample(frac=1)
    normal_need = 7200

    remove_list_pneumonia = []

    count = {'normal': 0, 'pneumonia': 0, 'covid-19': 0}
    image_count = 1
    total_data = pneumonia_csv.shape[0]
    for idx, row in pneumonia_csv.iterrows():
        file_path = os.path.join(data_path, row['Path'])
        patient_id = row['Path'].split('/')[2]
        if row['Sex'] == 'Male':
            sex = 'M'
        elif row['Sex'] == 'Female':
            sex = 'F'
        else:
            sex = np.nan
        final_label = 'pneumonia'


        if os.path.exists(file_path):
            file_name = 'chexpert_image_' + str(image_count) + '.png'
            image_count += 1
            if (image_count - 1) not in remove_list_pneumonia:
                count[final_label] += 1
                image = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE)
                img_resized = resize_image(image)
                cv2.imwrite(os.path.join(dataset_settings.output_path, file_name), img_resized)
                file_id = dataset_settings.image_id
                dataset_settings.image_id += 1
                dataset_settings.df = insert_into_df(df=dataset_settings.df, file_id=file_id,
                                                     patient_id='chexpert_'+patient_id, file_name=file_name,
                                                     org_file_name=row['Path'], sex=sex, age=row['Age'],
                                                     source_url=source_url, original_location='USA',
                                                     country='USA', view=row['AP/PA'], final_label=final_label)
        else:
            print(file_path + "does not exist")

        if image_count % 50 == 0:
            print("Completed pneumonia processing " + str(image_count) + " images of " + str(total_data))


    remove_list_normal = []
    total_data = normal_csv.shape[0]
    normal_taken = 0
    for idx, row in normal_csv.iterrows():
        file_path = os.path.join(data_path, row['Path'])
        patient_id = row['Path'].split('/')[2]
        if row['Sex'] == 'Male':
            sex = 'M'
        elif row['Sex'] == 'Female':
            sex = 'F'
        else:
            sex = np.nan
        final_label = 'normal'

        if os.path.exists(file_path):
            file_name = 'chexpert_image_' + str(image_count) + '.png'
            image_count += 1
            if (image_count - 1) not in remove_list_normal:
                count[final_label] += 1
                image = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE)
                img_resized = resize_image(image)
                cv2.imwrite(os.path.join(dataset_settings.output_path, file_name), img_resized)
                file_id = dataset_settings.image_id
                dataset_settings.image_id += 1
                dataset_settings.df = insert_into_df(df=dataset_settings.df, file_id=file_id,
                                                     patient_id='chexpert_' + patient_id, file_name=file_name,
                                                     org_file_name=row['Path'], sex=sex, age=row['Age'],
                                                     source_url=source_url, original_location='USA',
                                                     country='USA', view=row['AP/PA'], final_label=final_label)
                normal_taken += 1
                if normal_taken > normal_need:
                    break
        else:
            print(file_path + "does not exist")

        if image_count % 50 == 0:
            print("Completed normal processing " + str(image_count) + " images of " + str(total_data))

    dataset_settings.df.to_csv(dataset_settings.final_csv_file, index=False)
    print(count)

    write_info(source_url=source_url, count=count)

import os

import cv2
import numpy as np
import pandas as pd

import dataset_settings
from util import insert_into_df, write_info, resize_image


def prepare_chexpert_data(data_path, train_csv_path, valid_csv_path, source_url):
    chexpert_train_csv = pd.read_csv(train_csv_path, nrows=None)
    chexpert_valid_csv = pd.read_csv(valid_csv_path, nrows=None)
    final_csv = pd.concat([chexpert_train_csv, chexpert_valid_csv])
    final_csv = final_csv[final_csv['Frontal/Lateral'] != 'Lateral']
    normal_csv = final_csv[final_csv['No Finding'] == 1]
    normal_csv = normal_csv.sample(frac=1)
    pneumonia_csv = final_csv[final_csv['Pneumonia'] == 1]
    pneumonia_csv = pneumonia_csv.sample(frac=1)
    normal_need = 7200

    remove_list_pneumonia = []

    count = {'normal': 0, 'pneumonia': 0, 'covid-19': 0}
    image_count = 1
    total_data = pneumonia_csv.shape[0]
    for idx, row in pneumonia_csv.iterrows():
        file_path = os.path.join(data_path, row['Path'])
        patient_id = row['Path'].split('/')[2]
        if row['Sex'] == 'Male':
            sex = 'M'
        elif row['Sex'] == 'Female':
            sex = 'F'
        else:
            sex = np.nan
        final_label = 'pneumonia'


        if os.path.exists(file_path):
            file_name = 'chexpert_image_' + str(image_count) + '.png'
            image_count += 1
            if (image_count - 1) not in remove_list_pneumonia:
                count[final_label] += 1
                image = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE)
                img_resized = resize_image(image)
                cv2.imwrite(os.path.join(dataset_settings.output_path, file_name), img_resized)
                file_id = dataset_settings.image_id
                dataset_settings.image_id += 1
                dataset_settings.df = insert_into_df(df=dataset_settings.df, file_id=file_id,
                                                     patient_id='chexpert_'+patient_id, file_name=file_name,
                                                     org_file_name=row['Path'], sex=sex, age=row['Age'],
                                                     source_url=source_url, original_location='USA',
                                                     country='USA', view=row['AP/PA'], final_label=final_label)
        else:
            print(file_path + "does not exist")

        if image_count % 50 == 0:
            print("Completed pneumonia processing " + str(image_count) + " images of " + str(total_data))


    remove_list_normal = []
    total_data = normal_csv.shape[0]
    normal_taken = 0
    for idx, row in normal_csv.iterrows():
        file_path = os.path.join(data_path, row['Path'])
        patient_id = row['Path'].split('/')[2]
        if row['Sex'] == 'Male':
            sex = 'M'
        elif row['Sex'] == 'Female':
            sex = 'F'
        else:
            sex = np.nan
        final_label = 'normal'

        if os.path.exists(file_path):
            file_name = 'chexpert_image_' + str(image_count) + '.png'
            image_count += 1
            if (image_count - 1) not in remove_list_normal:
                count[final_label] += 1
                image = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE)
                img_resized = resize_image(image)
                cv2.imwrite(os.path.join(dataset_settings.output_path, file_name), img_resized)
                file_id = dataset_settings.image_id
                dataset_settings.image_id += 1
                dataset_settings.df = insert_into_df(df=dataset_settings.df, file_id=file_id,
                                                     patient_id='chexpert_' + patient_id, file_name=file_name,
                                                     org_file_name=row['Path'], sex=sex, age=row['Age'],
                                                     source_url=source_url, original_location='USA',
                                                     country='USA', view=row['AP/PA'], final_label=final_label)
                normal_taken += 1
                if normal_taken > normal_need:
                    break
        else:
            print(file_path + "does not exist")

        if image_count % 50 == 0:
            print("Completed normal processing " + str(image_count) + " images of " + str(total_data))

    dataset_settings.df.to_csv(dataset_settings.final_csv_file, index=False)
    print(count)

    write_info(source_url=source_url, count=count)