class Factor(object):
    @preprocess(data=ensure_pd_df)
    def __init__(self, data, name, property_dict=defaultdict(str), **kwargs):
        self.data = copy.deepcopy(data)
        self.name = name
        self.property = update_factor_property(property_dict)
        self.production_data_format = kwargs.get('production_data_format', OutputDataFormat.MULTI_INDEX_DF)

        self._validate_data_format()
        self._validate_data_name()
        self._validate_date_index()

    def _validate_data_format(self):
        if self.property['data_format'] != self.production_data_format:
            self.data = convert_df_format(self.data, target_format=self.production_data_format)
        return

    def _validate_data_name(self):
        if self.production_data_format == OutputDataFormat.MULTI_INDEX_DF:
            self.data.columns = [self.name]
        return

    def _validate_date_index(self):
        self.data = ensure_pd_index_names(self.data,
                                          data_format=self.production_data_format,
                                          valid_index=INDEX_FACTOR)
        date_format = self.property.get('date_format', '%Y-%m-%d')
        date_index = self.data.index.get_level_values(INDEX_FACTOR.date_index)
        date_index = [ensure_datetime(date_, date_format) for date_ in date_index]
        self.data.reset_index(inplace=True)
        self.data[INDEX_FACTOR.date_index] = date_index
        if self.production_data_format == OutputDataFormat.MULTI_INDEX_DF:
            self.data.set_index(keys=INDEX_FACTOR.full_index, inplace=True)
        else:
            self.data.set_index(keys=INDEX_FACTOR.date_index, inplace=True)

    @property
    def type(self):
        return self.property['type']

    @property
    def freq(self):
        return self.property['freq']

    @property
    def production_format(self):
        return self.production_data_format

    @property
    def trade_date_list(self):
        if self.production_data_format == OutputDataFormat.MULTI_INDEX_DF:
            ret = set(self.data.index.get_level_values(INDEX_FACTOR.date_index))
            ret = list(sorted(set(ret)))
        else:
            ret = self.data.index.tolist()

        return ret class FactorContainer(object):
    def __init__(self, start_date=None, end_date=None, factors=None, freq=FreqType.EOM, **kwargs):
        self.start_date = start_date
        self.end_date = end_date
        self.freq = freq
        self._tiaocang_date = kwargs.get('tiaocang_date', None)
        self.calendar = kwargs.get('calendar', 'China.SSE')
        self.date_format = kwargs.get('date_format', '%Y-%m-%d')
        self.names = []
        self.property = defaultdict(str)
        self.data = pd.DataFrame()

        self._validate_tiaocang_date()
        self._update_tiaocang_date()
        self._merge_factors(factors)

    def _validate_tiaocang_date(self):
        if (self.start_date is None or self.end_date is None) and self._tiaocang_date is None:
            raise ValueError('tiaocang date must be set initially')

    def _update_tiaocang_date(self):
        if self._tiaocang_date is None:
            self._tiaocang_date = get_tiaocang_date(start_date=self.start_date,
                                                    end_date=self.end_date,
                                                    freq=self.freq,
                                                    calendar=self.calendar,
                                                    date_format=self.date_format)
        return

    @property
    def tiaocang_date(self):
        return self._tiaocang_date

    @expect_types(factor=(Factor))
    def _check_tiaocang_date(self, factor):
        """
        如果该因子的freq和container的freq一致，那么该因子的日期列应该包含所有的调仓日期，否则报错
        """
        py_assert(set(self._tiaocang_date).issubset(set(factor.trade_date_list)),
                    ValueError,
                    'factor {0} does not contain all tiaocang date in its trade date list'.format(factor.name))
        return

    @expect_types(factors=optional(Factor, list))
    def _merge_factors(self, factors):
        if factors is None:
            return

        factors_ = factors if isinstance(factors, list) else list(factors)
        for factor in factors_:
            py_assert(factor.production_format == OutputDataFormat.MULTI_INDEX_DF,
                        ValueError,
                        'factor {0} does not in multi-index dataframe format therefore can not be merged into container'
                        .format(factor.name))
            self._merge_factor(factor)
        self.data = self.data.loc[self._tiaocang_date]
        return

    def _merge_factor(self, factor):
        self.names.append(factor.name)
        self.property[factor.name] = factor.property
        self._concat_factor_data(factor)
        self.data.columns = self.names
        return

    def _concat_factor_data(self, factor):
        py_assert(self.freq == factor.freq,
                    ValueError,
                    'Failed to concatenate: factor {0} has different freq from containter'.format(factor.name))

        self._check_tiaocang_date(factor)
        self.data = pd.concat([self.data, factor.data], axis=1)

    @expect_types(factor=Factor)
    def add_factor(self, factor, overwrite=False):
        if overwrite is True:
            try:
                self.remove_factor(factor.name)
            except Exception:
                pass
        self._merge_factor(factor)
        self.data = self.data.loc[self._tiaocang_date]
        return

    @expect_types(factor=(Factor, str))
    def remove_factor(self, factor):
        factor_name = factor if isinstance(factor, str) else factor.name
        py_assert(factor_name in self.names, ValueError,
                    'unable to remove factor_name {0}, which does not exist in current container'.format(factor_name))
        self.names.remove(factor_name)
        self.data.drop(factor_name, axis=1, inplace=True)
        self.property = dissoc(self.property, factor_name)
        return

    @property
    def factor_names(self):
        return self.names

    @property
    def container_property(self):
        return self.property

    @property
    def industry_code(self):
        key = list(filter(lambda x: self.property[x]['type'] == FactorType.INDUSTY_CODE, self.names))
        py_warning(len(key) == 1, Warning, 'factor container should have only one industry code')
        return self.data[key[0]]

    @property
    def score(self):
        key = list(filter(lambda x: self.property[x]['type'] == FactorType.SCORE, self.names))
        py_warning(len(key) == 1, Warning, 'factor container should have only one score')
        return self.data[key[0]]

    @property
    def mkt_cap(self):
        key = list(filter(lambda x: self.property[x]['type'] == FactorType.ALPHA_FACTOR_MV, self.names))
        py_warning(len(key) == 1, Warning, 'factor container should have only one mkt cap')
        return self.data[key[0]]

    @property
    def alpha_factor_col(self):
        def _is_alpha_factor(x):
            return self.property[x]['type'] == FactorType.ALPHA_FACTOR or \
                   self.property[x]['type'] == FactorType.ALPHA_FACTOR_MV

        return list(filter(_is_alpha_factor, self.names))

    @property
    def fwd_return_col(self):
        key = list(filter(lambda x: self.property[x]['type'] == FactorType.FWD_RETURN, self.names))
        return key

    @preprocess(data_=ensure_np_array)
    def replace_data(self, data_):
        self.data = pd.DataFrame(data_, index=self.data.index, columns=self.data.columns)
        return class FactorType(StrEnum):
    INDUSTY_CODE = 'industry_code'
    ALPHA_FACTOR = 'alpha_factor'
    RETURN = 'return'
    PRICE = 'price'
    ALPHA_FACTOR_MV = 'alpha_factor_mv'
    SCORE = 'score'
    FWD_RETURN = 'fwd_return'
    WEIGHT = 'weight' class FactorNormType(IntEnum):
    Null = 0
    Industry_Neutral = 1
    Industry_Cap_Neutral = 2 class NAStrategy(StrEnum):
    MEAN = 'mean'
    MEDIAN = 'median'
    MOST_FREQ = 'most_frequent'
    CUSTOM = 'custom' @expect_types(x=(pd.DataFrame, pd.Series))
def fwd_return(x, date_index=INDEX_FACTOR.date_index, sec_index=INDEX_FACTOR.sec_index, period=1):
    """
    每个日期和股票代码对应的未来收益
    """
    dates = sorted(set(x.index.get_level_values(date_index)))
    ret = pd.DataFrame()
    for i in range(len(dates) - period):
        shift_date = dates[i + period]
        data_concat = x.loc[shift_date].reset_index()
        data_concat[date_index] = [dates[i]] * len(data_concat)
        ret = pd.concat([ret, data_concat], axis=0)
    ret.set_index([date_index, sec_index], inplace=True)
    ret.columns = ['fwd_return']
    return ret def load_factor_data_from_csv(csv_file, date_index=INDEX_FACTOR.date_index, sec_index=INDEX_FACTOR.sec_index):
    data = pd.read_csv(csv_file, encoding='gbk')
    data[date_index] = pd.to_datetime(data[date_index])
    data.set_index([date_index, sec_index], inplace=True)
    return data class FactorIC(FactorEstimator):
    def __init__(self):
        super(FactorIC, self).__init__()

    def predict(self, factor_container):
        data_df = factor_container.data
        tiaocang_date = factor_container.tiaocang_date
        result = pd.DataFrame()
        for fwd in factor_container.fwd_return_col:
            fwd_return = data_df[fwd]
            for alpha in factor_container.alpha_factor_col:
                alpha_factor = data_df[alpha]
                calc_data = [stats.spearmanr(fwd_return.loc[date_], alpha_factor.loc[date_])[0] for date_ in
                             tiaocang_date]
                result = pd.concat(
                    [result, pd.DataFrame(data=calc_data, index=tiaocang_date, columns=[alpha + '_' + fwd])], axis=1)

        return result class FactorImputer(FactorTransformer):
    def __init__(self,
                 missing_value='NaN',
                 numerical_strategy=NAStrategy.MEAN,
                 axis=0,
                 verbose=0,
                 copy=True,
                 custom_value=None,
                 categorical_strategy=NAStrategy.MOST_FREQ,
                 out_container=False):
        super(FactorImputer, self).__init__(copy=copy, out_container=out_container)
        self.missing_values = missing_value
        self.copy = copy
        self.numerical_strategy = numerical_strategy
        self.categorical_strategy = categorical_strategy
        self.axis = axis
        self.verbose = verbose
        self.custom_value = custom_value
        self.mapper = None
        self.df_mapper = None

    def _build_mapper(self, factor_container):
        """
        https://github.com/pandas-dev/sklearn-pandas/blob/master/sklearn_pandas/dataframe_mapper.py 
        """
        data = factor_container.data
        data_mapper = [([factor_name], self._get_mapper(factor_container.property[factor_name]['type']))
                       for factor_name in data.columns]
        return DataFrameMapper(data_mapper)

    def _get_mapper(self, factor_type):
        if factor_type == FactorType.INDUSTY_CODE:
            return ExtCategoricalImputer(strategy=self.categorical_strategy,
                                         custom_value=self.custom_value,
                                         copy=self.copy)
        else:
            return Imputer(missing_values=self.missing_values,
                           strategy=self.numerical_strategy,
                           axis=self.axis,
                           verbose=self.verbose,
                           copy=self.copy) class FactorWinsorizer(FactorTransformer):
    def __init__(self, quantile_range=(2.5, 97.5), copy=True, out_container=False):
        super(FactorWinsorizer, self).__init__(copy=copy, out_container=out_container)
        self.quantile_range = quantile_range
        self.q_max = None
        self.q_min = None

    def _build_mapper(self, factor_container):
        data = factor_container.data
        data_mapper = [([factor_name], self._get_mapper(factor_container.property[factor_name]['type']))
                       for factor_name in data.columns]
        return DataFrameMapper(data_mapper)

    def _get_mapper(self, factor_type):
        if factor_type == FactorType.INDUSTY_CODE:
            return None
        else:
            return Winsorizer(quantile_range=self.quantile_range, copy=self.copy) class FactorStandardizer(FactorTransformer):
    def __init__(self, copy=True, out_container=False, with_mean=True, with_std=True):
        super(FactorStandardizer, self).__init__(copy=copy, out_container=out_container)
        self.with_mean = with_mean
        self.with_std = with_std

    def _build_mapper(self, factor_container):
        data = factor_container.data
        data_mapper = [([factor_name], self._get_mapper(factor_container.property[factor_name]['type']))
                       for factor_name in data.columns]
        return DataFrameMapper(data_mapper)

    def _get_mapper(self, factor_type):
        if factor_type == FactorType.INDUSTY_CODE:
            return None
        else:
            return StandardScaler(copy=self.copy, with_mean=self.with_mean, with_std=self.with_std) class FactorNeutralizer(FactorTransformer):
    def __init__(self, copy=True, out_container=False):
        super(FactorNeutralizer, self).__init__(copy=copy, out_container=out_container)

    def _build_mapper(self, factor_container, **kwargs):
        data_mapper_by_date = pd.Series()
        for date in factor_container.tiaocang_date:
            industry_code = factor_container.industry_code.loc[date]
            mkt_cap = factor_container.mkt_cap.loc[date]
            data = factor_container.data.loc[date]

            data_mapper = [([factor_name],
                            self._get_mapper(factor_type=factor_container.property[factor_name]['type'],
                                             factor_norm_type=factor_container.property[factor_name]['norm_type'],
                                             industry_code=industry_code,
                                             mkt_cap=mkt_cap))
                           for factor_name in data.columns]
            data_mapper_by_date[date] = DataFrameMapper(data_mapper)
        return data_mapper_by_date

    @staticmethod
    def _get_mapper(factor_type, factor_norm_type, industry_code, mkt_cap=None):
        if factor_type == FactorType.INDUSTY_CODE or factor_norm_type == FactorNormType.Null:
            return None
        else:
            if factor_norm_type == FactorNormType.Industry_Cap_Neutral:
                return Neutralizer(industry_code, mkt_cap)
            else:
                return Neutralizer(industry_code)

    @preprocess(factor_container=ensure_factor_container)
    def transform(self, factor_container, **kwargs):
        if self.copy:
            factor_container = copy.deepcopy(factor_container)
        tiaocang_date = factor_container.tiaocang_date
        neutralize_data = [self.df_mapper[date_].fit_transform(factor_container.data.loc[date_]) for date_ in
                           tiaocang_date]
        neutralize_data_agg = np.vstack(neutralize_data)
        factor_container.data = pd.DataFrame(neutralize_data_agg,
                                             index=factor_container.data.index,
                                             columns=factor_container.data.columns)
        if self.out_container:
            return factor_container
        else:
            return factor_container.data
from alphaware.base import (Factor,
                            FactorContainer)
from alphaware.enums import (FactorType,
                             FactorNormType,
                             NAStrategy)
from alphaware.utils import (fwd_return,
                             load_factor_data_from_csv)
from alphaware.analyzer import FactorIC
from alphaware.preprocess import (FactorNeutralizer,
                                  FactorStandardizer,
                                  FactorWinsorizer,
                                  FactorImputer)
# -*- coding: utf-8 -*-


# 加载MV和PB数据
data_pb = load_factor_data_from_csv('pb.csv')
data_mv = load_factor_data_from_csv('mv.csv') / 100000000

# 创建Factor实例，储存数据以及相关参数
factor_pb = Factor(data=data_pb,
                   name='PB',
                   property_dict={'type': FactorType.ALPHA_FACTOR, 'norm_type': FactorNormType.Industry_Neutral})
factor_mv = Factor(data=data_mv,
                   name='MV',
                   property_dict={'type': FactorType.ALPHA_FACTOR_MV, 'norm_type': FactorNormType.Industry_Neutral})

# 加载月度收益数据
data_return = load_factor_data_from_csv('return.csv')
# 将数据改成未来1月收益
data_return = fwd_return(data_return)
factor_return = Factor(data=data_return, name='1_Fwd_Return', property_dict={'type': FactorType.FWD_RETURN})

# 加载行业数据(早年的wind的行业代码不太全，可能用其他数据源的数据更好，此处仅做示例用)
data_industry_code = load_factor_data_from_csv('sw.csv')
factor_industry_code = Factor(data=data_industry_code,
                              name='industry_code',
                              property_dict={'type': FactorType.INDUSTY_CODE})

# 创建FactorContainer实例，加载所有的因子信息
fc = FactorContainer(start_date='2014-01-01',
                     end_date='2014-03-01',
                     factors=[factor_mv, factor_pb, factor_return, factor_industry_code])

# 第一步，处理极个别N/A, 有中位数替换
fc = FactorImputer(numerical_strategy=NAStrategy.MEDIAN,
                   categorical_strategy=NAStrategy.CUSTOM,
                   custom_value='other',
                   out_container=True).fit_transform(fc)

# 第二部，去极值化

fc = FactorWinsorizer(quantile_range=(5, 95),