lib/anystyle/parser/parser.rb
# Returns an array of tokens for each line of input.
#
# If the passed-in string is marked as being tagged, extracts labels
# from the string and returns an array of token/label pairs for each
# line of input.
Returns an array of tokens for each line of input.
If the passed-in string is marked as being tagged, extracts labels
from the string and returns an array of token/label pairs for each
line of input.
Returns an array of tokens for each line of input . If the passed - in string is marked as being tagged extracts labels from the string and returns an array of token / label pairs for each line of input .
def tokenize(string, tagged = false)
        if tagged
          lines(string).each_with_index.map do |s,i|
            tt, tokens, tags = s.split(options[:tagged_separator]), [], []
            tt.each do |token|
              case token
              when /^$/
              when /^<([^\/>][^>]*)>$/
                tags << $1
              when /^<\/([^>]+)>$/
                unless (tag = tags.pop) == $1
                  raise ArgumentError, "mismatched tags on line #{i}: #{$1.inspect} (current tag was #{tag.inspect})"
                end
              else
                tokens << [decode_xml_text(token), (tags[-1] || :unknown).to_sym]
              end
            end
            tokens
          end
        else
          lines(string).map { |s| s.split(options[:separator]).reject(&:empty?) }
        end
      end
def tokenize ( string , tagged = false ) if tagged lines ( string ) . each_with_index . map do | s , i | tt , tokens , tags = s . split ( options [ :tagged_separator ] ) , [ ] , [ ] tt . each do | token | case token when / ^$ / when / ^<([^ \/ >][^>]*)>$ / tags << $1 when / ^< \/ ([^>]+)>$ / unless ( tag = tags . pop ) == $1 raise ArgumentError , "mismatched tags on line #{i}: #{$1.inspect} (current tag was #{tag.inspect})" end else tokens << [ decode_xml_text ( token ) , ( tags [ - 1 ] || :unknown ) . to_sym ] end end tokens end else lines ( string ) . map { | s | s . split ( options [ :separator ] ) . reject ( & :empty? ) } end end
Returns an array of tokens for each line of input.
Returns an array of tokens for each line of input .
# skip
string
<not_specific>
tagged
<not_specific>