<s> import re <EOL> import sqlparse <EOL> from sqlparse . tokens import Name <EOL> from collections import defaultdict <EOL> from . pgliterals . main import get_literals <EOL> white_space_regex = re . compile ( '<STR_LIT>' , re . MULTILINE ) <EOL> def _compile_regex ( keyword ) : <EOL> pattern = '<STR_LIT>' + re . sub ( white_space_regex , '<STR_LIT>' , keyword ) + '<STR_LIT>' <EOL> return re . compile ( pattern , re . MULTILINE | re . IGNORECASE ) <EOL> keywords = get_literals ( '<STR_LIT>' ) <EOL> keyword_regexs = dict ( ( kw , _compile_regex ( kw ) ) for kw in keywords ) <EOL> class PrevalenceCounter ( object ) : <EOL> def __init__ ( self ) : <EOL> self . keyword_counts = defaultdict ( int ) <EOL> self . name_counts = defaultdict ( int ) <EOL> def update ( self , text ) : <EOL> self . update_keywords ( text ) <EOL> self . update_names ( text ) <EOL> def update_names ( self , text ) : <EOL> for parsed in sqlparse . parse ( text ) : <EOL> for token in parsed . flatten ( ) : <EOL> if token . ttype in Name : <EOL> self . name_counts [ token . value ] += <NUM_LIT:1> <EOL> def clear_names ( self ) : <EOL> self . name_counts = defaultdict ( int ) <EOL> def update_keywords ( self , text ) : <EOL> for keyword , regex in keyword_regexs . items ( ) : <EOL> for _ in regex . finditer ( text ) : <EOL> self . keyword_counts [ keyword ] += <NUM_LIT:1> <EOL> def keyword_count ( self , keyword ) : <EOL> return self . keyword_counts [ keyword ] <EOL> def name_count ( self , name ) : <EOL> return self . name_counts [ name ] </s>
