def normalize(df):
    """Flatten compound columns (DataFrames within columns) into top level columns"""
    # in ipython notebook, explore and describe the DataFrame columns
    print('Of the {} columns, {} are actually DataFrames'.format(len(df.columns), sum([not isinstance(df[col], pd.Series) for col in df.columns])))
    # remove dataframes with only 2 columns and one is the _str of the other:
    for col in df.columns:
        if isinstance(df[col], pd.DataFrame):
            print('Column {} is a {}-wide DataFrame'.format(col, len(df[col].columns)))
            if df[col].columns[1] == df[col].columns[0] + '_str':
                print('Column {} looks easy because it has sub-columns {}'.format(col, df[col].columns))
                df[col] = df[col][df[col].columns[1]]
            else:
                try:
                    assert(float(df[col].iloc[:, 0].max()) == float(df[col].iloc[:, 1].max()))
                    df[col] = df[col].fillna(-1, inplace=False)
                    series = pd.Series([int(Decimal(x)) for x in df[col].iloc[:, 1].values]).astype('int64').copy()
                    del df[col]
                    df[col] = series
                    print('Finished converting column {} to type {}({})'.format(col, type(df[col]), df[col].dtype))
                except:
                    print_exc()

    print('Of the {} columns, {} are still DataFrames after trying to convert both columns to long integers'.format(
        len(df.columns), sum([not isinstance(df[col], pd.Series) for col in df.columns])))
    return df
def normalize ( df ) : print ( 'Of the {} columns, {} are actually DataFrames' . format ( len ( df . columns ) , sum ( [ not isinstance ( df [ col ] , pd . Series ) for col in df . columns ] ) ) ) for col in df . columns : if isinstance ( df [ col ] , pd . DataFrame ) : print ( 'Column {} is a {}-wide DataFrame' . format ( col , len ( df [ col ] . columns ) ) ) if df [ col ] . columns [ 1 ] == df [ col ] . columns [ 0 ] + '_str' : print ( 'Column {} looks easy because it has sub-columns {}' . format ( col , df [ col ] . columns ) ) df [ col ] = df [ col ] [ df [ col ] . columns [ 1 ] ] else : try : assert ( float ( df [ col ] . iloc [ : , 0 ] . max ( ) ) == float ( df [ col ] . iloc [ : , 1 ] . max ( ) ) ) df [ col ] = df [ col ] . fillna ( - 1 , inplace = False ) series = pd . Series ( [ int ( Decimal ( x ) ) for x in df [ col ] . iloc [ : , 1 ] . values ] ) . astype ( 'int64' ) . copy ( ) del df [ col ] df [ col ] = series print ( 'Finished converting column {} to type {}({})' . format ( col , type ( df [ col ] ) , df [ col ] . dtype ) ) except : print_exc ( ) print ( 'Of the {} columns, {} are still DataFrames after trying to convert both columns to long integers' . format ( len ( df . columns ) , sum ( [ not isinstance ( df [ col ] , pd . Series ) for col in df . columns ] ) ) ) return df
Flatten compound columns (DataFrames within columns) into top level columns