bleualign/score.py
def normalize(s):
    '''Normalize and tokenize text. This is lifted from NIST mteval-v11a.pl.'''
    # Added to bypass NIST-style pre-processing of hyp and ref files -- wade
    if (nonorm):
        return s.split()
    try:
        s.split()
    except:
        s = " ".join(s)
    # language-independent part:
    for (pattern, replace) in normalize1:
        s = re.sub(pattern, replace, s)
    s = xml.sax.saxutils.unescape(s, {'&quot;':'"'})
    # language-dependent part (assuming Western languages):
    s = " %s " % s
    if not preserve_case:
        s = s.lower()         # this might not be identical to the original
    return [tok for tok in normalize3.split(s) if tok and tok != ' ']
def normalize(s):
    '''Normalize and tokenize text. This is lifted from NIST mteval-v11a.pl.'''
    # Added to bypass NIST-style pre-processing of hyp and ref files -- wade
    if (nonorm):
        return s.split()
    try:
        s.split()
    except:
        s = " ".join(s)
    # language-independent part:
    for (pattern, replace) in normalize1:
        s = re.sub(pattern, replace, s)
    s = xml.sax.saxutils.unescape(s, {'&quot;':'"'})
    # language-dependent part (assuming Western languages):
    s = " %s " % s
    if not preserve_case:
        s = s.lower()         # this might not be identical to the original
    return [tok for tok in normalize3.split(s) if tok and tok != ' ']
def normalize ( s ) : # Added to bypass NIST-style pre-processing of hyp and ref files -- wade if ( nonorm ) : return s . split ( ) try : s . split ( ) except : s = " " . join ( s ) # language-independent part: for ( pattern , replace ) in normalize1 : s = re . sub ( pattern , replace , s ) s = xml . sax . saxutils . unescape ( s , { '&quot;' : '"' } ) # language-dependent part (assuming Western languages): s = " %s " % s if not preserve_case : s = s . lower ( ) # this might not be identical to the original return [ tok for tok in normalize3 . split ( s ) if tok and tok != ' ' ]
Normalize and tokenize text. This is lifted from NIST mteval-v11a.pl.
Normalize and tokenize text . This is lifted from NIST mteval - v11a . pl .