<s> from hashlib import sha1 <EOL> from . import defaults <EOL> from . . import reverts <EOL> from . tokens import Token , Tokens <EOL> class Version : <EOL> __slots__ = ( '<STR_LIT>' ) <EOL> def __init__ ( self ) : <EOL> self . tokens = None <EOL> class State : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , tokenize = defaults . TOKENIZE , diff = defaults . DIFF , <EOL> revert_radius = reverts . defaults . RADIUS , <EOL> revert_detector = None ) : <EOL> self . tokenize = tokenize <EOL> self . diff = diff <EOL> if revert_detector is None : <EOL> self . revert_detector = reverts . Detector ( int ( revert_radius ) ) <EOL> else : <EOL> self . revert_detector = revert_detector <EOL> self . last = None <EOL> def process ( self , text , revision = None , checksum = None ) : <EOL> """<STR_LIT>""" <EOL> if checksum is None : <EOL> checksum = sha1 ( bytes ( text , '<STR_LIT:utf8>' ) ) . hexdigest ( ) <EOL> version = Version ( ) <EOL> revert = self . revert_detector . process ( checksum , version ) <EOL> if revert is not None : <EOL> tokens_added = Tokens ( ) <EOL> tokens_removed = Tokens ( ) <EOL> _ , _ , reverted_to = revert <EOL> version . tokens = reverted_to . tokens <EOL> else : <EOL> if self . last is None : <EOL> version . tokens = Tokens ( Token ( t ) for t in self . tokenize ( text ) ) <EOL> tokens_added = version . tokens <EOL> tokens_removed = Tokens ( ) <EOL> else : <EOL> version . tokens , tokens_added , tokens_removed = self . last . tokens . compare ( self . tokenize ( text ) , self . diff ) <EOL> version . tokens . persist ( revision ) <EOL> self . last = version <EOL> return version . tokens , tokens_added , tokens_removed </s>
