def test_autoclean_cv_real_data():
    adult_data = pd.read_csv('adult.csv.gz', sep='\t', compression='gzip')

    training_adult_data = adult_data[:int(len(adult_data) / 2.)].copy()
    testing_adult_data = adult_data[int(len(adult_data) / 2.):].copy()

    training_adult_data.loc[30:60, 'age'] = np.nan
    training_adult_data.loc[90:100, 'education'] = np.nan

    testing_adult_data.loc[90:110, 'age'] = np.nan
    testing_adult_data.loc[20:40, 'education'] = np.nan

    hand_cleaned_training_adult_data = training_adult_data.copy()
    hand_cleaned_testing_adult_data = testing_adult_data.copy()

    training_age_median = hand_cleaned_training_adult_data['age'].median()
    training_education_mode = hand_cleaned_training_adult_data['education'].mode()[0]

    hand_cleaned_training_adult_data['age'].fillna(training_age_median, inplace=True)
    hand_cleaned_training_adult_data['education'].fillna(training_education_mode, inplace=True)

    hand_cleaned_testing_adult_data['age'].fillna(training_age_median, inplace=True)
    hand_cleaned_testing_adult_data['education'].fillna(training_education_mode, inplace=True)

    for column in ['workclass', 'education', 'marital-status',
                   'occupation', 'relationship', 'race',
                   'sex', 'native-country', 'label']:
        encoder = LabelEncoder()
        hand_cleaned_training_adult_data[column] = encoder.fit_transform(hand_cleaned_training_adult_data[column].values)
        hand_cleaned_testing_adult_data[column] = encoder.transform(hand_cleaned_testing_adult_data[column].values)

    cleaned_adult_training_data, cleaned_adult_testing_data = autoclean_cv(training_adult_data, testing_adult_data)

    assert cleaned_adult_training_data.equals(hand_cleaned_training_adult_data)
    assert cleaned_adult_testing_data.equals(hand_cleaned_testing_adult_data)
Test autoclean_cv() with the adult data set
dataset/ETHPy150Open rhiever/datacleaner/tests.py/test_autoclean_cv_real_data