title: Haskell Alex - regex matches wrong string?
body: <p>I'm trying to write lexer for an indentation-based grammar and I'm having trouble matching the indentation.</p>
  
  <p>Here's my code:</p>
  
  <pre><code>{
  module Lexer ( main ) where
  
  import System.IO.Unsafe
  }
  
  
  %wrapper "monadUserState"
  
  $whitespace = [\ \t\b]
  $digit      = 0-9                                            -- digits
  $alpha      = [A-Za-z]
  $letter     = [a-zA-Z]                                       -- alphabetic characters
  $ident      = [$letter $digit _]                             -- identifier character
  $indent     = [\ \t]
  
  @number     = [$digit]+
  @identifier = $alpha($alpha|_|$digit)*
  
  error:-
  
  @identifier { mkL LVarId }
  
  \n $whitespace* \n { skip }
  \n $whitespace*    { setIndent }
  $whitespace+       { skip }
  
  {
  
  data Lexeme = Lexeme AlexPosn LexemeClass (Maybe String)
  
  instance Show Lexeme where
      show (Lexeme _ LEOF _)   = "  Lexeme EOF"
      show (Lexeme p cl  mbs) = " Lexeme class=" ++ show cl ++ showap p ++ showst mbs
        where
          showap pp = " posn=" ++ showPosn pp
          showst Nothing  = ""
          showst (Just s) = " string=" ++ show s
  
  instance Eq Lexeme where
      (Lexeme _ cls1 _) == (Lexeme _ cls2 _) = cls1 == cls2
  
  showPosn :: AlexPosn -&gt; String
  showPosn (AlexPn _ line col) = show line ++ ':': show col
  
  tokPosn :: Lexeme -&gt; AlexPosn
  tokPosn (Lexeme p _ _) = p
  
  data LexemeClass
      = LVarId
      | LTIndent Int
      | LTDedent Int
      | LIndent
      | LDedent
      | LEOF
      deriving (Show, Eq)
  
  mkL :: LexemeClass -&gt; AlexInput -&gt; Int -&gt; Alex Lexeme
  mkL c (p, _, _, str) len = return (Lexeme p c (Just (take len str)))
  
  data AlexUserState = AlexUserState { indent :: Int }
  
  alexInitUserState :: AlexUserState
  alexInitUserState = AlexUserState 0
  
  type Action = AlexInput -&gt; Int -&gt; Alex Lexeme
  
  getLexerIndentLevel :: Alex Int
  getLexerIndentLevel = Alex $ \s@AlexState{alex_ust=ust} -&gt; Right (s, indent ust)
  
  setLexerIndentLevel :: Int -&gt; Alex ()
  setLexerIndentLevel i = Alex $ \s@AlexState{alex_ust=ust} -&gt; Right (s{alex_ust=(AlexUserState i)}, ())
  
  setIndent :: Action
  setIndent input@(p, _, _, str) i = do
      --let !x = unsafePerformIO $ putStrLn $ "|matched string: " ++ str ++ "|"
      lastIndent &lt;- getLexerIndentLevel
      currIndent &lt;- countIndent (drop 1 str) 0 -- first char is always \n
      if (lastIndent &lt; currIndent) then
          do setLexerIndentLevel currIndent
             mkL (LTIndent (currIndent - lastIndent)) input i
      else if (lastIndent &gt; currIndent) then
          do setLexerIndentLevel currIndent
             mkL (LTDedent (lastIndent - currIndent)) input i
      else alexMonadScan
    where
      countIndent str total
          | take 1 str == "\t" = do skip input 1
                                    countIndent (drop 1 str) (total+1)
          | take 4 str == "    " = do skip input 4
                                      countIndent (drop 4 str) (total+1)
          | otherwise = return total
  
  alexEOF :: Alex Lexeme
  alexEOF = return (Lexeme undefined LEOF Nothing)
  
  scanner :: String -&gt; Either String [Lexeme]
  scanner str =
      let loop = do
          tok@(Lexeme _ cl _) &lt;- alexMonadScan
          if (cl == LEOF)
              then return [tok]
              else do toks &lt;- loop
                      return (tok:toks)
      in runAlex str loop
  
  addIndentations :: [Lexeme] -&gt; [Lexeme]
  addIndentations (lex@(Lexeme pos (LTIndent c) _):ls) =
      concat [iter lex c, addIndentations ls]
    where iter lex c = if c == 0 then []
                       else (Lexeme pos LIndent Nothing):(iter lex (c-1))
  addIndentations (lex@(Lexeme pos (LTDedent c) _):ls) =
      concat [iter lex c, addIndentations ls]
    where iter lex c = if c == 0 then []
                       else (Lexeme pos LDedent Nothing):(iter lex (c-1))
  addIndentations (l:ls) = l:(addIndentations ls)
  addIndentations [] = []
  
  
  main = do
      s &lt;- getContents
      return ()
      print $ fmap addIndentations (scanner s)
  
  }
  </code></pre>
  
  <p>Problem is that in line <code>\n $whitespace*    { setIndent }</code>, regex matches wrong string and calls <code>setIndent</code> with this wrong string. For debugging purposes, I added <code>unsafePerformIO</code> in <code>setIndent</code> function, here's an example run of the program:</p>
  
  <pre><code>begin       
          first indent
  |matched string: 
          first indent
                  second indent
                  second indent
  dedent
  dedent
  |
  |matched string: 
                  second indent
  dedent
  |
  |matched string: 
  dedent
  |
  |matched string: 
  |
  Right [ Lexeme class=LVarId posn=1:1 string="begin", Lexeme class=LIndent posn=1:6, Lexeme class=LVarId posn=2:15 string="indent", Lexeme class=LIndent posn=2:21, Lexeme class=LDedent posn=3:30, Lexeme class=LDedent posn=3:30, Lexeme class=LVarId posn=4:1 string="dedent",  Lexeme EOF]
  </code></pre>
  
  <p>So <code>setIndent</code> is called with more than just whitespaces. And after it returns the lexeme for indentation, other part of the string is omitted.</p>
  
  <p>Is this a bug in Alex? Or what am I doing wrong?</p>