lib/lexer.js
// Lexing, tokenize the source markdown string
Lexing, tokenize the source markdown string
Lexing tokenize the source markdown string
token(srcStr, top) {
    let src = srcStr.replace(/^ +$/gm, '');
    while (src) {
      let isStrMatched = false
      this.grammars.some((grammar) => {
        const cap = grammar.rules.block.exec(src)
        if (!cap) {
          return false
        }
        const subStr = grammar.tokenize(cap, src, this.tokens, this.token.bind(this), top)
        if (subStr === -1) {
          return false
        }
        src = subStr || src.substring(cap[0].length);
        isStrMatched = true
        return true
      })
      if (!isStrMatched && src) {
        throw new Error(`Infinite loop on byte: ${src.charCodeAt(0)}`)
      }
    }
    return this.tokens
  }
token ( srcStr , top ) { let src = srcStr . replace ( / ^ +$ / gm , '' ) ; while ( src ) { let isStrMatched = false this . grammars . some ( ( grammar ) => { const cap = grammar . rules . block . exec ( src ) if ( ! cap ) { return false } const subStr = grammar . tokenize ( cap , src , this . tokens , this . token . bind ( this ) , top ) if ( subStr === - 1 ) { return false } src = subStr || src . substring ( cap [ 0 ] . length ) ; isStrMatched = true return true } ) if ( ! isStrMatched && src ) { throw new Error ( ` ${ src . charCodeAt ( 0 ) } ` ) } } return this . tokens }
Lexing, tokenize the source markdown string
Lexing tokenize the source markdown string