<s> import constants , sys <EOL> from charsetprober import CharSetProber <EOL> SAMPLE_SIZE = <NUM_LIT:64> <EOL> SB_ENOUGH_REL_THRESHOLD = <NUM_LIT> <EOL> POSITIVE_SHORTCUT_THRESHOLD = <NUM_LIT> <EOL> NEGATIVE_SHORTCUT_THRESHOLD = <NUM_LIT> <EOL> SYMBOL_CAT_ORDER = <NUM_LIT> <EOL> NUMBER_OF_SEQ_CAT = <NUM_LIT:4> <EOL> POSITIVE_CAT = NUMBER_OF_SEQ_CAT - <NUM_LIT:1> <EOL> class SingleByteCharSetProber ( CharSetProber ) : <EOL> def __init__ ( self , model , reversed = constants . False , nameProber = None ) : <EOL> CharSetProber . __init__ ( self ) <EOL> self . _mModel = model <EOL> self . _mReversed = reversed <EOL> self . _mNameProber = nameProber <EOL> self . reset ( ) <EOL> def reset ( self ) : <EOL> CharSetProber . reset ( self ) <EOL> self . _mLastOrder = <NUM_LIT:255> <EOL> self . _mSeqCounters = [ <NUM_LIT:0> ] * NUMBER_OF_SEQ_CAT <EOL> self . _mTotalSeqs = <NUM_LIT:0> <EOL> self . _mTotalChar = <NUM_LIT:0> <EOL> self . _mFreqChar = <NUM_LIT:0> <EOL> def get_charset_name ( self ) : <EOL> if self . _mNameProber : <EOL> return self . _mNameProber . get_charset_name ( ) <EOL> else : <EOL> return self . _mModel [ '<STR_LIT>' ] <EOL> def feed ( self , aBuf ) : <EOL> if not self . _mModel [ '<STR_LIT>' ] : <EOL> aBuf = self . filter_without_english_letters ( aBuf ) <EOL> aLen = len ( aBuf ) <EOL> if not aLen : <EOL> return self . get_state ( ) <EOL> for c in aBuf : <EOL> order = self . _mModel [ '<STR_LIT>' ] [ ord ( c ) ] <EOL> if order < SYMBOL_CAT_ORDER : <EOL> self . _mTotalChar += <NUM_LIT:1> <EOL> if order < SAMPLE_SIZE : <EOL> self . _mFreqChar += <NUM_LIT:1> <EOL> if self . _mLastOrder < SAMPLE_SIZE : <EOL> self . _mTotalSeqs += <NUM_LIT:1> <EOL> if not self . _mReversed : <EOL> self . _mSeqCounters [ self . _mModel [ '<STR_LIT>' ] [ ( self . _mLastOrder * SAMPLE_SIZE ) + order ] ] += <NUM_LIT:1> <EOL> else : <EOL> self . _mSeqCounters [ self . _mModel [ '<STR_LIT>' ] [ ( order * SAMPLE_SIZE ) + self . _mLastOrder ] ] += <NUM_LIT:1> <EOL> self . _mLastOrder = order <EOL> if self . get_state ( ) == constants . eDetecting : <EOL> if self . _mTotalSeqs > SB_ENOUGH_REL_THRESHOLD : <EOL> cf = self . get_confidence ( ) <EOL> if cf > POSITIVE_SHORTCUT_THRESHOLD : <EOL> if constants . _debug : <EOL> sys . stderr . write ( '<STR_LIT>' % ( self . _mModel [ '<STR_LIT>' ] , cf ) ) <EOL> self . _mState = constants . eFoundIt <EOL> elif cf < NEGATIVE_SHORTCUT_THRESHOLD : <EOL> if constants . _debug : <EOL> sys . stderr . write ( '<STR_LIT>' % ( self . _mModel [ '<STR_LIT>' ] , cf , NEGATIVE_SHORTCUT_THRESHOLD ) ) <EOL> self . _mState = constants . eNotMe <EOL> return self . get_state ( ) <EOL> def get_confidence ( self ) : <EOL> r = <NUM_LIT> <EOL> if self . _mTotalSeqs > <NUM_LIT:0> : <EOL> r = ( <NUM_LIT:1.0> * self . _mSeqCounters [ POSITIVE_CAT ] ) / self . _mTotalSeqs / self . _mModel [ '<STR_LIT>' ] <EOL> r = r * self . _mFreqChar / self . _mTotalChar <EOL> if r >= <NUM_LIT:1.0> : <EOL> r = <NUM_LIT> <EOL> return r </s>
