private void newlineOutsideExpression(){
  int oldPos=pos - 1;
  if (pos > 1) {
    addToken(TokenKind.NEWLINE,oldPos,pos);
    oldPos=pos;
  }
  int indentLen=0;
  while (pos < buffer.length) {
    char c=buffer[pos];
    if (c == ' ') {
      indentLen++;
      pos++;
    }
 else     if (c == '\t') {
      indentLen+=8 - indentLen % 8;
      pos++;
    }
 else     if (c == '\n') {
      indentLen=0;
      pos++;
    }
 else     if (c == '#') {
      if (oldPos != pos) {
        addToken(TokenKind.WHITESPACE,oldPos,pos);
        oldPos=pos;
      }
      while (pos < buffer.length && c != '\n') {
        c=buffer[pos++];
      }
      addToken(TokenKind.COMMENT,oldPos,pos - 1,bufferSlice(oldPos,pos - 1));
      oldPos=pos - 1;
      indentLen=0;
    }
 else {
      break;
    }
  }
  if (oldPos != pos) {
    addToken(TokenKind.WHITESPACE,oldPos,pos);
  }
  if (pos == buffer.length) {
    indentLen=0;
  }
  int peekedIndent=indentStack.peek();
  if (peekedIndent < indentLen) {
    indentStack.push(indentLen);
    addToken(TokenKind.INDENT,pos,pos);
  }
 else   if (peekedIndent > indentLen) {
    while (peekedIndent > indentLen) {
      indentStack.pop();
      addToken(TokenKind.DEDENT,pos,pos);
      peekedIndent=indentStack.peek();
    }
    if (peekedIndent < indentLen) {
      error("indentation error");
    }
  }
}

private void newlineOutsideExpression ( ) { int oldPos = pos - 1 ; if ( pos > 1 ) { addToken ( TokenKind . NEWLINE , oldPos , pos ) ; oldPos = pos ; } int indentLen = 0 ; while ( pos < buffer . length ) { char c = buffer [ pos ] ; if ( c == ' ' ) { indentLen ++ ; pos ++ ; } else if ( c == '\t' ) { indentLen += 8 - indentLen % 8 ; pos ++ ; } else if ( c == '\n' ) { indentLen = 0 ; pos ++ ; } else if ( c == '#' ) { if ( oldPos != pos ) { addToken ( TokenKind . WHITESPACE , oldPos , pos ) ; oldPos = pos ; } while ( pos < buffer . length && c != '\n' ) { c = buffer [ pos ++ ] ; } addToken ( TokenKind . COMMENT , oldPos , pos - 1 , bufferSlice ( oldPos , pos - 1 ) ) ; oldPos = pos - 1 ; indentLen = 0 ; } else { break ; } } if ( oldPos != pos ) { addToken ( TokenKind . WHITESPACE , oldPos , pos ) ; } if ( pos == buffer . length ) { indentLen = 0 ; } int peekedIndent = indentStack . peek ( ) ; if ( peekedIndent < indentLen ) { indentStack . push ( indentLen ) ; addToken ( TokenKind . INDENT , pos , pos ) ; } else if ( peekedIndent > indentLen ) { while ( peekedIndent > indentLen ) { indentStack . pop ( ) ; addToken ( TokenKind . DEDENT , pos , pos ) ; peekedIndent = indentStack . peek ( ) ; } if ( peekedIndent < indentLen ) { error ( "indentation error" ) ; } } }
handle indent and dedent within statements . < p > note these tokens have zero length - - this is because we can have an arbitrary number of dedents squeezed into some number of characters , and the size of all the lexical elements must match the number of characters in the file .
