@Override
  public List<AnalyzedTokenReadings> tag(List<String> sentenceTokens)
  throws IOException {
    List<AnalyzedToken> taggerTokens;
    List<AnalyzedToken> lowerTaggerTokens;
    List<AnalyzedToken> upperTaggerTokens;
    List<AnalyzedTokenReadings> tokenReadings = new ArrayList<>();
    int pos = 0;

    Matcher matcher;
    for (String word : sentenceTokens) {
      String probeWord = word;

      // This loop happens when we need to retry probing the dictionary
      // which happens rarely when trying to remove suffixes -mañ, -se, etc.
      for (;;) {
        List<AnalyzedToken> l = new ArrayList<>();
        String lowerWord = probeWord.toLowerCase(conversionLocale);
        taggerTokens = asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(probeWord));
        lowerTaggerTokens = asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(lowerWord));
        boolean isLowercase = probeWord.equals(lowerWord);

        // Normal case.
        addTokens(taggerTokens, l);

        if (!isLowercase) {
          // Lowercase.
          addTokens(lowerTaggerTokens, l);
        }

        // Uppercase.
        if (lowerTaggerTokens.isEmpty() && taggerTokens.isEmpty()) {
          if (isLowercase) {
            upperTaggerTokens = asAnalyzedTokenListForTaggedWords(word,
                getWordTagger().tag(StringTools.uppercaseFirstChar(probeWord)));
            if (!upperTaggerTokens.isEmpty()) {
              addTokens(upperTaggerTokens, l);
            }
          }
          if (l.isEmpty()) {
            if ((matcher = patternSuffix.matcher(probeWord)).find()) {
              // Remove the suffix and probe dictionary again.
              // So given a word such as "xxx-mañ", we're going to
              // try to probe the dictionary again with "xxx" this time.
              probeWord = matcher.group(1);
              continue;
            }
            l.add(new AnalyzedToken(word, null, null));
          }
        }
        tokenReadings.add(new AnalyzedTokenReadings(l, pos));
        pos += word.length();
        break;
      }
    }

    return tokenReadings;

  }
@ Override public List < AnalyzedTokenReadings > tag ( List < String > sentenceTokens ) throws IOException { List < AnalyzedToken > taggerTokens ; List < AnalyzedToken > lowerTaggerTokens ; List < AnalyzedToken > upperTaggerTokens ; List < AnalyzedTokenReadings > tokenReadings = new ArrayList < > ( ) ; int pos = 0 ; Matcher matcher ; for ( String word : sentenceTokens ) { String probeWord = word ; for ( ; ; ) { List < AnalyzedToken > l = new ArrayList < > ( ) ; String lowerWord = probeWord . toLowerCase ( conversionLocale ) ; taggerTokens = asAnalyzedTokenListForTaggedWords ( word , getWordTagger ( ) . tag ( probeWord ) ) ; lowerTaggerTokens = asAnalyzedTokenListForTaggedWords ( word , getWordTagger ( ) . tag ( lowerWord ) ) ; boolean isLowercase = probeWord . equals ( lowerWord ) ; addTokens ( taggerTokens , l ) ; if ( ! isLowercase ) { addTokens ( lowerTaggerTokens , l ) ; } if ( lowerTaggerTokens . isEmpty ( ) && taggerTokens . isEmpty ( ) ) { if ( isLowercase ) { upperTaggerTokens = asAnalyzedTokenListForTaggedWords ( word , getWordTagger ( ) . tag ( StringTools . uppercaseFirstChar ( probeWord ) ) ) ; if ( ! upperTaggerTokens . isEmpty ( ) ) { addTokens ( upperTaggerTokens , l ) ; } } if ( l . isEmpty ( ) ) { if ( ( matcher = patternSuffix . matcher ( probeWord ) ) . find ( ) ) { probeWord = matcher . group ( 1 ) ; continue ; } l . add ( new AnalyzedToken ( word , null , null ) ) ; } } tokenReadings . add ( new AnalyzedTokenReadings ( l , pos ) ) ; pos += word . length ( ) ; break ; } } return tokenReadings ; }
-mañ, -se, -hont.