# csv data util
import pandas as pd
from tqdm import tqdm


class csvUtil():
    def __init__(self, max_length=1000,encoding='utf8'):
        self.max_length = max_length
        self.encoding = encoding

    def divide_1000_csv_data(self,filename,):
        with open(filename,"rb") as f:
            data = pd.read_csv(f,encoding=self.encoding)
            df = pd.DataFrame(data)
            #df1 = df.iloc[:,[0,1]] #按列提取数据
            df1 = df.loc[0:1000,['WS_TITLE','ID','WS_CT','CREATED_TIME','PUBLISH_TIME']] #获取id与文本
            df1 = df1.dropna()
            list = df1.values.tolist()
            list_data = []
            list_title = ['WS_TITLE','ID','WS_CT','CREATED_TIME','PUBLISH_TIME']
            for i in range(len(list)):
                temp = dict(zip(list_title,list[i]))
                list_data.append(temp)
            return list_data,df1

    def divide_csv_data(self,filename):
        with open(filename,"rb") as f:
            data = pd.read_csv(f,encoding=self.encoding)
            x = data.shape[0]
            num = x//self.max_length
            if x%self.max_length!=0:
                num += 1
            df = pd.DataFrame(data)
            for i in tqdm(range(num)):
                #df1 = df.iloc[:,[0,1]] #按列提取数据
                df1 = df.loc[i*self.max_length:(i+1)*self.max_length,['WS_TITLE','ID','WS_CT','CREATED_TIME','PUBLISH_TIME']] #获取id与文本
                df1 = df1.dropna()
                list = df1.values.tolist()
                list_data = []
                list_title = ['WS_TITLE','ID','WS_CT','CREATED_TIME','PUBLISH_TIME']
                for j in range(len(list)):
                    temp = dict(zip(list_title,list[j]))
                    list_data.append(temp)
                df1.to_csv("EE_data_large_part{}.csv".format(i), encoding='utf8', index=False)
            return list_data,df1

    def extract_txt_data(self,filename):
        with open(filename,'r',encoding=self.encoding) as f:
            df = pd.read_csv(f)
            # col = [id,sentiment,review]
            df1 = df.iloc[:, [3,2]]
            df1 = df1.dropna()
            df1.iloc[:,[1]] = df1.iloc[:,[1]].astype('int')
            num = df.shape[0]
            print("The num of data is",num)
            df_train = df1.iloc[:int(num*0.8), :]
            df_test =  df1.iloc[int(num*0.8):int(num*0.9), :]
            df_dev = df1.iloc[int(num * 0.9):, :]
            df_train.to_csv('train.txt', sep='\t',index=False, header=None)
            df_test.to_csv('test.txt', sep='\t', index=False, header=None)
            df_dev.to_csv('dev.txt', sep='\t', index=False, header=None)
            return df1,df_train,df_test,df_dev

if __name__ == '__main__':
    # util = csvUtil(encoding='gbk')
    # df,df1,df2,df3 = util.extract_txt_data("news_training_data.csv")
    with open('train.txt', 'r', encoding='UTF-8') as f:
        for line in tqdm(f):
            lin = line.strip()
            if not lin:
                continue
            l = lin.split('\t')
            if len(l)!=2:
                print(l)


# csv data util
import pandas as pd
from tqdm import tqdm


class csvUtil():
    def __init__(self, max_length=1000,encoding='utf8'):
        self.max_length = max_length
        self.encoding = encoding

    def divide_1000_csv_data(self,filename,):
        with open(filename,"rb") as f:
            data = pd.read_csv(f,encoding=self.encoding)
            df = pd.DataFrame(data)
            #df1 = df.iloc[:,[0,1]] #按列提取数据
            df1 = df.loc[0:1000,['WS_TITLE','ID','WS_CT','CREATED_TIME','PUBLISH_TIME']] #获取id与文本
            df1 = df1.dropna()
            list = df1.values.tolist()
            list_data = []
            list_title = ['WS_TITLE','ID','WS_CT','CREATED_TIME','PUBLISH_TIME']
            for i in range(len(list)):
                temp = dict(zip(list_title,list[i]))
                list_data.append(temp)
            return list_data,df1

    def divide_csv_data(self,filename):
        with open(filename,"rb") as f:
            data = pd.read_csv(f,encoding=self.encoding)
            x = data.shape[0]
            num = x//self.max_length
            if x%self.max_length!=0:
                num += 1
            df = pd.DataFrame(data)
            for i in tqdm(range(num)):
                #df1 = df.iloc[:,[0,1]] #按列提取数据
                df1 = df.loc[i*self.max_length:(i+1)*self.max_length,['WS_TITLE','ID','WS_CT','CREATED_TIME','PUBLISH_TIME']] #获取id与文本
                df1 = df1.dropna()
                list = df1.values.tolist()
                list_data = []
                list_title = ['WS_TITLE','ID','WS_CT','CREATED_TIME','PUBLISH_TIME']
                for j in range(len(list)):
                    temp = dict(zip(list_title,list[j]))
                    list_data.append(temp)
                df1.to_csv("EE_data_large_part{}.csv".format(i), encoding='utf8', index=False)
            return list_data,df1

    def extract_txt_data(self,filename):
        with open(filename,'r',encoding=self.encoding) as f:
            df = pd.read_csv(f)
            # col = [id,sentiment,review]
            df1 = df.iloc[:, [3,2]]
            df1 = df1.dropna()
            df1.iloc[:,[1]] = df1.iloc[:,[1]].astype('int')
            num = df.shape[0]
            print("The num of data is",num)
            df_train = df1.iloc[:int(num*0.8), :]
            df_test =  df1.iloc[int(num*0.8):int(num*0.9), :]
            df_dev = df1.iloc[int(num * 0.9):, :]
            df_train.to_csv('train.txt', sep='\t',index=False, header=None)
            df_test.to_csv('test.txt', sep='\t', index=False, header=None)
            df_dev.to_csv('dev.txt', sep='\t', index=False, header=None)
            return df1,df_train,df_test,df_dev

if __name__ == '__main__':
    # util = csvUtil(encoding='gbk')
    # df,df1,df2,df3 = util.extract_txt_data("news_training_data.csv")
    with open('train.txt', 'r', encoding='UTF-8') as f:
        for line in tqdm(f):
            lin = line.strip()
            if not lin:
                continue
            l = lin.split('\t')
            if len(l)!=2:
                print(l)


# csv data util #df1 = df.iloc[:,[0,1]] #按列提取数据 #获取id与文本 #df1 = df.iloc[:,[0,1]] #按列提取数据 #获取id与文本 # col = [id,sentiment,review] # util = csvUtil(encoding='gbk') # df,df1,df2,df3 = util.extract_txt_data("news_training_data.csv")