def lex(module=None,object=None,debug=0,optimize=0,lextab="lextab",reflags=0,nowarn=0,cls=Lexer):
    global lexer
    ldict = None
    stateinfo  = { 'INITIAL' : 'inclusive'}
    error = 0
    files = { }
    lexobj = cls()
    lexobj.lexdebug = debug
    lexobj.lexoptimize = optimize
    global token,input

    if nowarn: warn = 0
    else: warn = 1
    
    if object: module = object

    if module:
        # User supplied a module object.
        if isinstance(module, types.ModuleType):
            ldict = module.__dict__
        elif isinstance(module, _INSTANCETYPE):
            _items = [(k,getattr(module,k)) for k in dir(module)]
            ldict = { }
            for (i,v) in _items:
                ldict[i] = v
        else:
            raise ValueError,"Expected a module or instance"
        lexobj.lexmodule = module
        
    else:
        # No module given.  We might be able to get information from the caller.
        try:
            raise RuntimeError
        except __HOLE__:
            e,b,t = sys.exc_info()
            f = t.tb_frame
            f = f.f_back           # Walk out to our calling function
            ldict = f.f_globals    # Grab its globals dictionary

    if optimize and lextab:
        try:
            lexobj.readtab(lextab,ldict)
            token = lexobj.token
            input = lexobj.input
            lexer = lexobj
            return lexobj
        
        except ImportError:
            pass
        
    # Get the tokens, states, and literals variables (if any)
    if (module and isinstance(module,_INSTANCETYPE)):
        tokens   = getattr(module,"tokens",None)
        states   = getattr(module,"states",None)
        literals = getattr(module,"literals","")
    else:
        tokens   = ldict.get("tokens",None)
        states   = ldict.get("states",None)
        literals = ldict.get("literals","")
        
    if not tokens:
        raise SyntaxError,"lex: module does not define 'tokens'"
    if not (isinstance(tokens,types.ListType) or isinstance(tokens,types.TupleType)):
        raise SyntaxError,"lex: tokens must be a list or tuple."

    # Build a dictionary of valid token names
    lexobj.lextokens = { }
    if not optimize:
        for n in tokens:
            if not _is_identifier.match(n):
                print "lex: Bad token name '%s'" % n
                error = 1
            if warn and lexobj.lextokens.has_key(n):
                print "lex: Warning. Token '%s' multiply defined." % n
            lexobj.lextokens[n] = None
    else:
        for n in tokens: lexobj.lextokens[n] = None

    if debug:
        print "lex: tokens = '%s'" % lexobj.lextokens.keys()

    try:
         for c in literals:
               if not (isinstance(c,types.StringType) or isinstance(c,types.UnicodeType)) or len(c) > 1:
                    print "lex: Invalid literal %s. Must be a single character" % repr(c)
                    error = 1
                    continue

    except TypeError:
         print "lex: Invalid literals specification. literals must be a sequence of characters."
         error = 1

    lexobj.lexliterals = literals

    # Build statemap
    if states:
         if not (isinstance(states,types.TupleType) or isinstance(states,types.ListType)):
              print "lex: states must be defined as a tuple or list."
              error = 1
         else:
              for s in states:
                    if not isinstance(s,types.TupleType) or len(s) != 2:
                           print "lex: invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')" % repr(s)
                           error = 1
                           continue
                    name, statetype = s
                    if not isinstance(name,types.StringType):
                           print "lex: state name %s must be a string" % repr(name)
                           error = 1
                           continue
                    if not (statetype == 'inclusive' or statetype == 'exclusive'):
                           print "lex: state type for state %s must be 'inclusive' or 'exclusive'" % name
                           error = 1
                           continue
                    if stateinfo.has_key(name):
                           print "lex: state '%s' already defined." % name
                           error = 1
                           continue
                    stateinfo[name] = statetype

    # Get a list of symbols with the t_ or s_ prefix
    tsymbols = [f for f in ldict.keys() if f[:2] == 't_' ]

    # Now build up a list of functions and a list of strings

    funcsym =  { }        # Symbols defined as functions
    strsym =   { }        # Symbols defined as strings
    toknames = { }        # Mapping of symbols to token names

    for s in stateinfo.keys():
         funcsym[s] = []
         strsym[s] = []

    ignore   = { }        # Ignore strings by state
    errorf   = { }        # Error functions by state

    if len(tsymbols) == 0:
        raise SyntaxError,"lex: no rules of the form t_rulename are defined."

    for f in tsymbols:
        t = ldict[f]
        states, tokname = _statetoken(f,stateinfo)
        toknames[f] = tokname

        if callable(t):
            for s in states: funcsym[s].append((f,t))
        elif (isinstance(t, types.StringType) or isinstance(t,types.UnicodeType)):
            for s in states: strsym[s].append((f,t))
        else:
            print "lex: %s not defined as a function or string" % f
            error = 1

    # Sort the functions by line number
    for f in funcsym.values():
        f.sort(lambda x,y: cmp(x[1].func_code.co_firstlineno,y[1].func_code.co_firstlineno))

    # Sort the strings by regular expression length
    for s in strsym.values():
        s.sort(lambda x,y: (len(x[1]) < len(y[1])) - (len(x[1]) > len(y[1])))

    regexs = { }

    # Build the master regular expressions
    for state in stateinfo.keys():
        regex_list = []

        # Add rules defined by functions first
        for fname, f in funcsym[state]:
            line = f.func_code.co_firstlineno
            file = f.func_code.co_filename
            files[file] = None
            tokname = toknames[fname]

            ismethod = isinstance(f, types.MethodType)

            if not optimize:
                nargs = f.func_code.co_argcount
                if ismethod:
                    reqargs = 2
                else:
                    reqargs = 1
                if nargs > reqargs:
                    print "%s:%d: Rule '%s' has too many arguments." % (file,line,f.__name__)
                    error = 1
                    continue

                if nargs < reqargs:
                    print "%s:%d: Rule '%s' requires an argument." % (file,line,f.__name__)
                    error = 1
                    continue

                if tokname == 'ignore':
                    print "%s:%d: Rule '%s' must be defined as a string." % (file,line,f.__name__)
                    error = 1
                    continue
        
            if tokname == 'error':
                errorf[state] = f
                continue

            if f.__doc__:
                if not optimize:
                    try:
                        c = re.compile("(?P<%s>%s)" % (f.__name__,f.__doc__), re.VERBOSE | reflags)
                        if c.match(""):
                             print "%s:%d: Regular expression for rule '%s' matches empty string." % (file,line,f.__name__)
                             error = 1
                             continue
                    except re.error,e:
                        print "%s:%d: Invalid regular expression for rule '%s'. %s" % (file,line,f.__name__,e)
                        if '#' in f.__doc__:
                             print "%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'." % (file,line, f.__name__)                 
                        error = 1
                        continue

                    if debug:
                        print "lex: Adding rule %s -> '%s' (state '%s')" % (f.__name__,f.__doc__, state)

                # Okay. The regular expression seemed okay.  Let's append it to the master regular
                # expression we're building
  
                regex_list.append("(?P<%s>%s)" % (f.__name__,f.__doc__))
            else:
                print "%s:%d: No regular expression defined for rule '%s'" % (file,line,f.__name__)

        # Now add all of the simple rules
        for name,r in strsym[state]:
            tokname = toknames[name]       

            if tokname == 'ignore':
                 ignore[state] = r
                 continue

            if not optimize:
                if tokname == 'error':
                    raise SyntaxError,"lex: Rule '%s' must be defined as a function" % name
                    error = 1
                    continue
        
                if not lexobj.lextokens.has_key(tokname) and tokname.find("ignore_") < 0:
                    print "lex: Rule '%s' defined for an unspecified token %s." % (name,tokname)
                    error = 1
                    continue
                try:
                    c = re.compile("(?P<%s>%s)" % (name,r),re.VERBOSE | reflags)
                    if (c.match("")):
                         print "lex: Regular expression for rule '%s' matches empty string." % name
                         error = 1
                         continue
                except re.error,e:
                    print "lex: Invalid regular expression for rule '%s'. %s" % (name,e)
                    if '#' in r:
                         print "lex: Make sure '#' in rule '%s' is escaped with '\\#'." % name

                    error = 1
                    continue
                if debug:
                    print "lex: Adding rule %s -> '%s' (state '%s')" % (name,r,state)
                
            regex_list.append("(?P<%s>%s)" % (name,r))

        if not regex_list:
             print "lex: No rules defined for state '%s'" % state
             error = 1

        regexs[state] = regex_list


    if not optimize:
        for f in files.keys(): 
           if not _validate_file(f):
                error = 1

    if error:
        raise SyntaxError,"lex: Unable to build lexer."

    # From this point forward, we're reasonably confident that we can build the lexer.
    # No more errors will be generated, but there might be some warning messages.

    # Build the master regular expressions

    for state in regexs.keys():
        lexre, re_text = _form_master_re(regexs[state],reflags,ldict)
        lexobj.lexstatere[state] = lexre
        lexobj.lexstateretext[state] = re_text
        if debug:
            for i in range(len(re_text)):
                 print "lex: state '%s'. regex[%d] = '%s'" % (state, i, re_text[i])

    # For inclusive states, we need to add the INITIAL state
    for state,type in stateinfo.items():
        if state != "INITIAL" and type == 'inclusive':
             lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL'])
             lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL'])

    lexobj.lexstateinfo = stateinfo
    lexobj.lexre = lexobj.lexstatere["INITIAL"]
    lexobj.lexretext = lexobj.lexstateretext["INITIAL"]

    # Set up ignore variables
    lexobj.lexstateignore = ignore
    lexobj.lexignore = lexobj.lexstateignore.get("INITIAL","")

    # Set up error functions
    lexobj.lexstateerrorf = errorf
    lexobj.lexerrorf = errorf.get("INITIAL",None)
    if warn and not lexobj.lexerrorf:
        print "lex: Warning. no t_error rule is defined."

    # Check state information for ignore and error rules
    for s,stype in stateinfo.items():
        if stype == 'exclusive':
              if warn and not errorf.has_key(s):
                   print "lex: Warning. no error rule is defined for exclusive state '%s'" % s
              if warn and not ignore.has_key(s) and lexobj.lexignore:
                   print "lex: Warning. no ignore rule is defined for exclusive state '%s'" % s
        elif stype == 'inclusive':
              if not errorf.has_key(s):
                   errorf[s] = errorf.get("INITIAL",None)
              if not ignore.has_key(s):
                   ignore[s] = ignore.get("INITIAL","")
   

    # Create global versions of the token() and input() functions
    token = lexobj.token
    input = lexobj.input
    lexer = lexobj

    # If in optimize mode, we write the lextab   
    if lextab and optimize:
        lexobj.writetab(lextab)

    return lexobj

# -----------------------------------------------------------------------------
# runmain()
#
# This runs the lexer as a main program
# -----------------------------------------------------------------------------
dataset/ETHPy150Open ardekantur/pyglet/tools/wraptypes/lex.py/lex