name-parser/src/main/java/org/gbif/nameparser/ParsingJob.java
/**
   * basic careful cleaning, trying to preserve all parsable name parts
   */
basic careful cleaning, trying to preserve all parsable name parts
basic careful cleaning trying to preserve all parsable name parts
public static String preClean(String name, @Nullable Set<String> warnings) throws InterruptedException {
    Matcher m = matcherInterruptable(XML_ENTITY_STRIP, name);
    if (m.find()) {
      name = m.replaceAll("&$1;");
    }
    int length = name.length();
    name = StringEscapeUtils.unescapeHtml4(name);
    if (warnings != null && length > name.length()) {
      warnings.add(Warnings.HTML_ENTITIES);
    }
    m = matcherInterruptable(AMPERSAND_ENTITY, name);
    if (m.find()) {
      name = m.replaceAll("&");
      if (warnings != null) {
        warnings.add(Warnings.HTML_ENTITIES);
      }
    }
    m = matcherInterruptable(XML_TAGS, name);
    if (m.find()) {
      name = m.replaceAll("");
      if (warnings != null) {
        warnings.add(Warnings.XML_TAGS);
      }
    }
    name = name.trim();
    for (char c : QUOTES) {
      int idx = 0;
      while (idx < name.length() && (c == name.charAt(idx) || Character.isWhitespace(name.charAt(idx)))) {
        idx++;
      }
      if (idx > 0) {
        int end = 0;
        while (c == name.charAt(name.length() - 1 - end) && (name.length() - idx - end) > 0) {
          end++;
        }
        name = name.substring(idx, name.length() - end);
      }
    }
    name = replAllInterruptable(NORM_WHITESPACE, name, " ");
    name = replAllInterruptable(NORM_APOSTROPHES, name, "'");
    return StringUtils.trimToEmpty(name);
  }
public static String preClean ( String name , @ Nullable Set < String > warnings ) throws InterruptedException { Matcher m = matcherInterruptable ( XML_ENTITY_STRIP , name ) ; if ( m . find ( ) ) { name = m . replaceAll ( "&$1;" ) ; } int length = name . length ( ) ; name = StringEscapeUtils . unescapeHtml4 ( name ) ; if ( warnings != null && length > name . length ( ) ) { warnings . add ( Warnings . HTML_ENTITIES ) ; } m = matcherInterruptable ( AMPERSAND_ENTITY , name ) ; if ( m . find ( ) ) { name = m . replaceAll ( "&" ) ; if ( warnings != null ) { warnings . add ( Warnings . HTML_ENTITIES ) ; } } m = matcherInterruptable ( XML_TAGS , name ) ; if ( m . find ( ) ) { name = m . replaceAll ( "" ) ; if ( warnings != null ) { warnings . add ( Warnings . XML_TAGS ) ; } } name = name . trim ( ) ; for ( char c : QUOTES ) { int idx = 0 ; while ( idx < name . length ( ) && ( c == name . charAt ( idx ) || Character . isWhitespace ( name . charAt ( idx ) ) ) ) { idx ++ ; } if ( idx > 0 ) { int end = 0 ; while ( c == name . charAt ( name . length ( ) - 1 - end ) && ( name . length ( ) - idx - end ) > 0 ) { end ++ ; } name = name . substring ( idx , name . length ( ) - end ) ; } } name = replAllInterruptable ( NORM_WHITESPACE , name , " " ) ; name = replAllInterruptable ( NORM_APOSTROPHES , name , "'" ) ; return StringUtils . trimToEmpty ( name ) ; }
basic careful cleaning, trying to preserve all parsable name parts
basic careful cleaning trying to preserve all parsable name parts
// remove bad whitespace in html entities // unescape html entities // finally remove still existing bad ampersands missing the closing ; // replace xml tags // trim // remove quotes in beginning and matching ones at the end // check if we also find this char at the end // replace various single quote apostrophes with always '