<reponame>KazukiOnodera/Microsoft-Malware-Prediction
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 23 21:49:53 2019

@author: kazuki.onodera

NOT time series feature

"""

import numpy as np
import pandas as pd
#from multiprocessing import Pool
from sklearn.preprocessing import LabelEncoder

import utils

PREF = 'f002'


col_cat = [
         'ProductName',                              # cardinality(6, 6), same distribuyion
         'EngineVersion',                           # maybe time series
         'AppVersion',                              # maybe time series
         'AvSigVersion',                            # maybe time series
         'DefaultBrowsersIdentifier',               # cardinality(1730, 1548)
         'AVProductStatesIdentifier',               # cardinality(28970, 23492)
         'CountryIdentifier',                        # cardinality(222, 222)
         'CityIdentifier',                           # cardinality(107366, 105817)
         'OrganizationIdentifier',                   # cardinality(49, 50)
         'GeoNameIdentifier',                        # cardinality(292, 289)
         'LocaleEnglishNameIdentifier',              # cardinality(276, 278)
         'Platform',                                 # cardinality(4, 4)
         'Processor',                                # cardinality(3, 3)
         'OsVer',                                   # cardinality(58, 44) maybe time series
         'OsPlatformSubRelease',                     # cardinality(58, 44)
         'OsBuildLab',                              # cardinality(663, 673)
         'SkuEdition',                               # cardinality(8, 8)
         'IeVerIdentifier',                         # cardinality(303, 294) maybe time series
         'SmartScreen',                              # cardinality(21, 21)
         'Census_MDC2FormFactor',                    # cardinality(13, 14)
         'Census_DeviceFamily',                      # cardinality(3, 3)
         'Census_OEMNameIdentifier',                # cardinality(3832, 3685)
         'Census_OEMModelIdentifier',               # cardinality(175365, 167776)
         'Census_ProcessorManufacturerIdentifier',   # cardinality(7, 7)
         'Census_ProcessorModelIdentifier',          # cardinality(3428, 3438)
         'Census_ProcessorClass',                    # cardinality(3, 3)
         'Census_PrimaryDiskTypeName',               # cardinality(4, 4)
         'Census_ChassisTypeName',                   # cardinality(52, 48)
         'Census_PowerPlatformRoleName',             # cardinality(10, 10)
         'Census_InternalBatteryType',               # cardinality(78, 63)
         'Census_OSVersion',                        # cardinality(469, 475)
         'Census_OSArchitecture',                    # cardinality(3, 3)
         'Census_OSBranch',                          # cardinality(32, 29)
         'Census_OSEdition',                         # cardinality(33, 36)
         'Census_OSSkuName',                         # cardinality(30, 31)
         'Census_OSInstallTypeName',                 # cardinality(9, 9)
         'Census_OSInstallLanguageIdentifier',       # cardinality(39, 39)
         'Census_OSUILocaleIdentifier',              # cardinality(147, 139)
         'Census_OSWUAutoUpdateOptionsName',         # cardinality(6, 6)
         'Census_GenuineStateName',                  # cardinality(5, 5)
         'Census_ActivationChannel',                 # cardinality(6, 6)
         'Census_FlightRing',                        # cardinality(10, 11)
         'Census_FirmwareManufacturerIdentifier',    # cardinality(712, 722)
         'Census_FirmwareVersionIdentifier',         # cardinality(50494, 49811)
         'Wdft_RegionIdentifier',                    # cardinality(15, 15)
         
         'OsBuild',                                  # cardinality(58, 44)
         'OsSuite',                                  # cardinality(58, 44)
         
         'OsBuildLab_major',
         'OsBuildLab_minor',
         'OsBuildLab_build',
         'OsBuildLab_architecture',
         ]



# =============================================================================
# main
# =============================================================================
if __name__ == "__main__":
    utils.start(__file__)
    
    tr = pd.read_feather('../data/train.f')[col_cat]
    te = pd.read_feather('../data/test.f')[col_cat]
    
    
    le = LabelEncoder()
    
    for c in col_cat:
        print('processing', c)
        
        if tr[c].dtype != 'O':
            min_ = min(tr[c].min(), te[c].min()) -1
            tr[c].fillna(min_, inplace=True)
            te[c].fillna(min_, inplace=True)
        else:
            tr[c].fillna('na dayo', inplace=True)
            te[c].fillna('na dayo', inplace=True)
        le.fit( tr[c].append(te[c]) )
        tr[c] = le.transform(tr[c])
        te[c]  = le.transform(te[c])
    
    utils.reduce_mem_usage(tr)
    utils.reduce_mem_usage(te)
    
    tr.add_prefix(PREF+'_').to_feather(f'../data/train_{PREF}.f')
    te.add_prefix(PREF+'_').to_feather(f'../data/test_{PREF}.f')
    
    
    print('==== categories ====')
    print(tr.head().add_prefix(PREF+'_').columns.tolist())
    
    utils.end(__file__)

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 23 21:49:53 2019

@author: kazuki.onodera

NOT time series feature

"""

import numpy as np
import pandas as pd
#from multiprocessing import Pool
from sklearn.preprocessing import LabelEncoder

import utils

PREF = 'f002'


col_cat = [
         'ProductName',                              # cardinality(6, 6), same distribuyion
         'EngineVersion',                           # maybe time series
         'AppVersion',                              # maybe time series
         'AvSigVersion',                            # maybe time series
         'DefaultBrowsersIdentifier',               # cardinality(1730, 1548)
         'AVProductStatesIdentifier',               # cardinality(28970, 23492)
         'CountryIdentifier',                        # cardinality(222, 222)
         'CityIdentifier',                           # cardinality(107366, 105817)
         'OrganizationIdentifier',                   # cardinality(49, 50)
         'GeoNameIdentifier',                        # cardinality(292, 289)
         'LocaleEnglishNameIdentifier',              # cardinality(276, 278)
         'Platform',                                 # cardinality(4, 4)
         'Processor',                                # cardinality(3, 3)
         'OsVer',                                   # cardinality(58, 44) maybe time series
         'OsPlatformSubRelease',                     # cardinality(58, 44)
         'OsBuildLab',                              # cardinality(663, 673)
         'SkuEdition',                               # cardinality(8, 8)
         'IeVerIdentifier',                         # cardinality(303, 294) maybe time series
         'SmartScreen',                              # cardinality(21, 21)
         'Census_MDC2FormFactor',                    # cardinality(13, 14)
         'Census_DeviceFamily',                      # cardinality(3, 3)
         'Census_OEMNameIdentifier',                # cardinality(3832, 3685)
         'Census_OEMModelIdentifier',               # cardinality(175365, 167776)
         'Census_ProcessorManufacturerIdentifier',   # cardinality(7, 7)
         'Census_ProcessorModelIdentifier',          # cardinality(3428, 3438)
         'Census_ProcessorClass',                    # cardinality(3, 3)
         'Census_PrimaryDiskTypeName',               # cardinality(4, 4)
         'Census_ChassisTypeName',                   # cardinality(52, 48)
         'Census_PowerPlatformRoleName',             # cardinality(10, 10)
         'Census_InternalBatteryType',               # cardinality(78, 63)
         'Census_OSVersion',                        # cardinality(469, 475)
         'Census_OSArchitecture',                    # cardinality(3, 3)
         'Census_OSBranch',                          # cardinality(32, 29)
         'Census_OSEdition',                         # cardinality(33, 36)
         'Census_OSSkuName',                         # cardinality(30, 31)
         'Census_OSInstallTypeName',                 # cardinality(9, 9)
         'Census_OSInstallLanguageIdentifier',       # cardinality(39, 39)
         'Census_OSUILocaleIdentifier',              # cardinality(147, 139)
         'Census_OSWUAutoUpdateOptionsName',         # cardinality(6, 6)
         'Census_GenuineStateName',                  # cardinality(5, 5)
         'Census_ActivationChannel',                 # cardinality(6, 6)
         'Census_FlightRing',                        # cardinality(10, 11)
         'Census_FirmwareManufacturerIdentifier',    # cardinality(712, 722)
         'Census_FirmwareVersionIdentifier',         # cardinality(50494, 49811)
         'Wdft_RegionIdentifier',                    # cardinality(15, 15)
         
         'OsBuild',                                  # cardinality(58, 44)
         'OsSuite',                                  # cardinality(58, 44)
         
         'OsBuildLab_major',
         'OsBuildLab_minor',
         'OsBuildLab_build',
         'OsBuildLab_architecture',
         ]



# =============================================================================
# main
# =============================================================================
if __name__ == "__main__":
    utils.start(__file__)
    
    tr = pd.read_feather('../data/train.f')[col_cat]
    te = pd.read_feather('../data/test.f')[col_cat]
    
    
    le = LabelEncoder()
    
    for c in col_cat:
        print('processing', c)
        
        if tr[c].dtype != 'O':
            min_ = min(tr[c].min(), te[c].min()) -1
            tr[c].fillna(min_, inplace=True)
            te[c].fillna(min_, inplace=True)
        else:
            tr[c].fillna('na dayo', inplace=True)
            te[c].fillna('na dayo', inplace=True)
        le.fit( tr[c].append(te[c]) )
        tr[c] = le.transform(tr[c])
        te[c]  = le.transform(te[c])
    
    utils.reduce_mem_usage(tr)
    utils.reduce_mem_usage(te)
    
    tr.add_prefix(PREF+'_').to_feather(f'../data/train_{PREF}.f')
    te.add_prefix(PREF+'_').to_feather(f'../data/test_{PREF}.f')
    
    
    print('==== categories ====')
    print(tr.head().add_prefix(PREF+'_').columns.tolist())
    
    utils.end(__file__)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- Created on Wed Jan 23 21:49:53 2019  @author: kazuki.onodera  NOT time series feature #from multiprocessing import Pool # cardinality(6, 6), same distribuyion # maybe time series # maybe time series # maybe time series # cardinality(1730, 1548) # cardinality(28970, 23492) # cardinality(222, 222) # cardinality(107366, 105817) # cardinality(49, 50) # cardinality(292, 289) # cardinality(276, 278) # cardinality(4, 4) # cardinality(3, 3) # cardinality(58, 44) maybe time series # cardinality(58, 44) # cardinality(663, 673) # cardinality(8, 8) # cardinality(303, 294) maybe time series # cardinality(21, 21) # cardinality(13, 14) # cardinality(3, 3) # cardinality(3832, 3685) # cardinality(175365, 167776) # cardinality(7, 7) # cardinality(3428, 3438) # cardinality(3, 3) # cardinality(4, 4) # cardinality(52, 48) # cardinality(10, 10) # cardinality(78, 63) # cardinality(469, 475) # cardinality(3, 3) # cardinality(32, 29) # cardinality(33, 36) # cardinality(30, 31) # cardinality(9, 9) # cardinality(39, 39) # cardinality(147, 139) # cardinality(6, 6) # cardinality(5, 5) # cardinality(6, 6) # cardinality(10, 11) # cardinality(712, 722) # cardinality(50494, 49811) # cardinality(15, 15) # cardinality(58, 44) # cardinality(58, 44) # ============================================================================= # main # =============================================================================