Generates Unicode strings, one for each <seg> in a ParaCrawl data file.

  Also decodes some of the most common HTML entities found in ParaCrawl data.

  Args:
    paracrawl_file: A ParaCrawl V3.0 en-.. data file.
  Yields:
    One Unicode string for each <seg> element in the ParaCrawl data file.

def _raw_sentences(paracrawl_file):
  """Generates Unicode strings, one for each <seg> in a ParaCrawl data file.

  Also decodes some of the most common HTML entities found in ParaCrawl data.

  Args:
    paracrawl_file: A ParaCrawl V3.0 en-.. data file.
  Yields:
    One Unicode string for each <seg> element in the ParaCrawl data file.
  """
  for line_utf8 in paracrawl_file:
    line_uni = line_utf8.decode('UTF-8')
    text_match = re.match(r' +<seg>(.*)</seg>$', line_uni)
    if text_match:
      txt = text_match.group(1)
      txt = re.sub(r'&amp;', r'&', txt)
      txt = re.sub(r'& ?amp;', r'&', txt)
      txt = re.sub(r'& ?apos;', r"'", txt)
      txt = re.sub(r'& ?quot;', r'"', txt)
      txt = re.sub(r'& ?lt;', r'<', txt)
      txt = re.sub(r'& ?gt;', r'>', txt)
      yield txt