<s> from . euctwfreq import ( EUCTWCharToFreqOrder , EUCTW_TABLE_SIZE , <EOL> EUCTW_TYPICAL_DISTRIBUTION_RATIO ) <EOL> from . euckrfreq import ( EUCKRCharToFreqOrder , EUCKR_TABLE_SIZE , <EOL> EUCKR_TYPICAL_DISTRIBUTION_RATIO ) <EOL> from . gb2312freq import ( GB2312CharToFreqOrder , GB2312_TABLE_SIZE , <EOL> GB2312_TYPICAL_DISTRIBUTION_RATIO ) <EOL> from . big5freq import ( Big5CharToFreqOrder , BIG5_TABLE_SIZE , <EOL> BIG5_TYPICAL_DISTRIBUTION_RATIO ) <EOL> from . jisfreq import ( JISCharToFreqOrder , JIS_TABLE_SIZE , <EOL> JIS_TYPICAL_DISTRIBUTION_RATIO ) <EOL> from . compat import wrap_ord <EOL> ENOUGH_DATA_THRESHOLD = <NUM_LIT> <EOL> SURE_YES = <NUM_LIT> <EOL> SURE_NO = <NUM_LIT> <EOL> class CharDistributionAnalysis : <EOL> def __init__ ( self ) : <EOL> self . _mCharToFreqOrder = None <EOL> self . _mTableSize = None <EOL> self . _mTypicalDistributionRatio = None <EOL> self . reset ( ) <EOL> def reset ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _mDone = False <EOL> self . _mTotalChars = <NUM_LIT:0> <EOL> self . _mFreqChars = <NUM_LIT:0> <EOL> def feed ( self , aBuf , aCharLen ) : <EOL> """<STR_LIT>""" <EOL> if aCharLen == <NUM_LIT:2> : <EOL> order = self . get_order ( aBuf ) <EOL> else : <EOL> order = - <NUM_LIT:1> <EOL> if order >= <NUM_LIT:0> : <EOL> self . _mTotalChars += <NUM_LIT:1> <EOL> if order < self . _mTableSize : <EOL> if <NUM_LIT> > self . _mCharToFreqOrder [ order ] : <EOL> self . _mFreqChars += <NUM_LIT:1> <EOL> def get_confidence ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _mTotalChars <= <NUM_LIT:0> : <EOL> return SURE_NO <EOL> if self . _mTotalChars != self . _mFreqChars : <EOL> r = ( self . _mFreqChars / ( ( self . _mTotalChars - self . _mFreqChars ) <EOL> * self . _mTypicalDistributionRatio ) ) <EOL> if r < SURE_YES : <EOL> return r <EOL> return SURE_YES <EOL> def got_enough_data ( self ) : <EOL> return self . _mTotalChars > ENOUGH_DATA_THRESHOLD <EOL> def get_order ( self , aBuf ) : <EOL> return - <NUM_LIT:1> <EOL> class EUCTWDistributionAnalysis ( CharDistributionAnalysis ) : <EOL> def __init__ ( self ) : <EOL> CharDistributionAnalysis . __init__ ( self ) <EOL> self . _mCharToFreqOrder = EUCTWCharToFreqOrder <EOL> self . _mTableSize = EUCTW_TABLE_SIZE <EOL> self . _mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO <EOL> def get_order ( self , aBuf ) : <EOL> first_char = wrap_ord ( aBuf [ <NUM_LIT:0> ] ) <EOL> if first_char >= <NUM_LIT> : <EOL> return <NUM_LIT> * ( first_char - <NUM_LIT> ) + wrap_ord ( aBuf [ <NUM_LIT:1> ] ) - <NUM_LIT> <EOL> else : <EOL> return - <NUM_LIT:1> <EOL> class EUCKRDistributionAnalysis ( CharDistributionAnalysis ) : <EOL> def __init__ ( self ) : <EOL> CharDistributionAnalysis . __init__ ( self ) <EOL> self . _mCharToFreqOrder = EUCKRCharToFreqOrder <EOL> self . _mTableSize = EUCKR_TABLE_SIZE <EOL> self . _mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO <EOL> def get_order ( self , aBuf ) : <EOL> first_char = wrap_ord ( aBuf [ <NUM_LIT:0> ] ) <EOL> if first_char >= <NUM_LIT> : <EOL> return <NUM_LIT> * ( first_char - <NUM_LIT> ) + wrap_ord ( aBuf [ <NUM_LIT:1> ] ) - <NUM_LIT> <EOL> else : <EOL> return - <NUM_LIT:1> <EOL> class GB2312DistributionAnalysis ( CharDistributionAnalysis ) : <EOL> def __init__ ( self ) : <EOL> CharDistributionAnalysis . __init__ ( self ) <EOL> self . _mCharToFreqOrder = GB2312CharToFreqOrder <EOL> self . _mTableSize = GB2312_TABLE_SIZE <EOL> self . _mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO <EOL> def get_order ( self , aBuf ) : <EOL> first_char , second_char = wrap_ord ( aBuf [ <NUM_LIT:0> ] ) , wrap_ord ( aBuf [ <NUM_LIT:1> ] ) <EOL> if ( first_char >= <NUM_LIT> ) and ( second_char >= <NUM_LIT> ) : <EOL> return <NUM_LIT> * ( first_char - <NUM_LIT> ) + second_char - <NUM_LIT> <EOL> else : <EOL> return - <NUM_LIT:1> <EOL> class Big5DistributionAnalysis ( CharDistributionAnalysis ) : <EOL> def __init__ ( self ) : <EOL> CharDistributionAnalysis . __init__ ( self ) <EOL> self . _mCharToFreqOrder = Big5CharToFreqOrder <EOL> self . _mTableSize = BIG5_TABLE_SIZE <EOL> self . _mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO <EOL> def get_order ( self , aBuf ) : <EOL> first_char , second_char = wrap_ord ( aBuf [ <NUM_LIT:0> ] ) , wrap_ord ( aBuf [ <NUM_LIT:1> ] ) <EOL> if first_char >= <NUM_LIT> : <EOL> if second_char >= <NUM_LIT> : <EOL> return <NUM_LIT> * ( first_char - <NUM_LIT> ) + second_char - <NUM_LIT> + <NUM_LIT> <EOL> else : <EOL> return <NUM_LIT> * ( first_char - <NUM_LIT> ) + second_char - <NUM_LIT> <EOL> else : <EOL> return - <NUM_LIT:1> <EOL> class SJISDistributionAnalysis ( CharDistributionAnalysis ) : <EOL> def __init__ ( self ) : <EOL> CharDistributionAnalysis . __init__ ( self ) <EOL> self . _mCharToFreqOrder = JISCharToFreqOrder <EOL> self . _mTableSize = JIS_TABLE_SIZE <EOL> self . _mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO <EOL> def get_order ( self , aBuf ) : <EOL> first_char , second_char = wrap_ord ( aBuf [ <NUM_LIT:0> ] ) , wrap_ord ( aBuf [ <NUM_LIT:1> ] ) <EOL> if ( first_char >= <NUM_LIT> ) and ( first_char <= <NUM_LIT> ) : <EOL> order = <NUM_LIT> * ( first_char - <NUM_LIT> ) <EOL> elif ( first_char >= <NUM_LIT> ) and ( first_char <= <NUM_LIT> ) : <EOL> order = <NUM_LIT> * ( first_char - <NUM_LIT> + <NUM_LIT> ) <EOL> else : <EOL> return - <NUM_LIT:1> <EOL> order = order + second_char - <NUM_LIT> <EOL> if second_char > <NUM_LIT> : <EOL> order = - <NUM_LIT:1> <EOL> return order <EOL> class EUCJPDistributionAnalysis ( CharDistributionAnalysis ) : <EOL> def __init__ ( self ) : <EOL> CharDistributionAnalysis . __init__ ( self ) <EOL> self . _mCharToFreqOrder = JISCharToFreqOrder <EOL> self . _mTableSize = JIS_TABLE_SIZE <EOL> self . _mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO <EOL> def get_order ( self , aBuf ) : <EOL> char = wrap_ord ( aBuf [ <NUM_LIT:0> ] ) <EOL> if char >= <NUM_LIT> : <EOL> return <NUM_LIT> * ( char - <NUM_LIT> ) + wrap_ord ( aBuf [ <NUM_LIT:1> ] ) - <NUM_LIT> <EOL> else : <EOL> return - <NUM_LIT:1> </s>
