<reponame>finguist/ReadAlong-Studio
from unittest import TestCase
from lxml import etree
import unittest

from readalongs.g2p import tokenize_xml

class TestTokenizer(TestCase):
    def test_simple(self):
        txt = """<document>
<s xml:lang="atj">Kwei! Tan e ici matisihin?</s>
</document>
"""
        xml = etree.fromstring(txt)
        tokenized = tokenize_xml.tokenize_xml(xml)
        print(etree.tounicode(tokenized))

    def test_mixed_lang(self):
        txt = """<document>
<s xml:lang="atj">Kwei! Tan e ici matisihin?</s>
<s xml:lang="fra">Bonjour! Comment ça va?</s>
</document>
"""
        xml = etree.fromstring(txt)
        tokenized = tokenize_xml.tokenize_xml(xml)
        print(etree.tounicode(tokenized))

    def test_mixed_words(self):
        txt = """<document>
<s xml:lang="atj">Kwei! (<w xml:lang="fra">Bonjour</w>!)</s>
<s xml:lang="atj">Tan e ici matisihin?</s>
</document>
"""
        xml = etree.fromstring(txt)
        tokenized = tokenize_xml.tokenize_xml(xml)
        print(etree.tounicode(tokenized))

    def test_comments(self):
        txt = """<document>
<s xml:lang="atj">Kwei! (<w xml:lang="fra">Bonjour</w>!)</s>
<s xml:lang="atj">Tan e ici matisihin?</s>
</document>
"""
        xml = etree.fromstring(txt)
        tokenized = tokenize_xml.tokenize_xml(xml)
        print(etree.tounicode(tokenized))

from unittest import TestCase
from lxml import etree
import unittest

from readalongs.g2p import tokenize_xml

class TestTokenizer(TestCase):
    def test_simple(self):
        txt = """<document>
<s xml:lang="atj">Kwei! Tan e ici matisihin?</s>
</document>
"""
        xml = etree.fromstring(txt)
        tokenized = tokenize_xml.tokenize_xml(xml)
        print(etree.tounicode(tokenized))

    def test_mixed_lang(self):
        txt = """<document>
<s xml:lang="atj">Kwei! Tan e ici matisihin?</s>
<s xml:lang="fra">Bonjour! Comment ça va?</s>
</document>
"""
        xml = etree.fromstring(txt)
        tokenized = tokenize_xml.tokenize_xml(xml)
        print(etree.tounicode(tokenized))

    def test_mixed_words(self):
        txt = """<document>
<s xml:lang="atj">Kwei! (<w xml:lang="fra">Bonjour</w>!)</s>
<s xml:lang="atj">Tan e ici matisihin?</s>
</document>
"""
        xml = etree.fromstring(txt)
        tokenized = tokenize_xml.tokenize_xml(xml)
        print(etree.tounicode(tokenized))

    def test_comments(self):
        txt = """<document>
<s xml:lang="atj">Kwei! (<w xml:lang="fra">Bonjour</w>!)</s>
<s xml:lang="atj">Tan e ici matisihin?</s>
</document>
"""
        xml = etree.fromstring(txt)
        tokenized = tokenize_xml.tokenize_xml(xml)
        print(etree.tounicode(tokenized))
<document> <s xml:lang="atj">Kwei! Tan e ici matisihin?</s> </document> <document> <s xml:lang="atj">Kwei! Tan e ici matisihin?</s> <s xml:lang="fra">Bonjour! Comment ça va?</s> </document> <document> <s xml:lang="atj">Kwei! (<w xml:lang="fra">Bonjour</w>!)</s> <s xml:lang="atj">Tan e ici matisihin?</s> </document> <document> <s xml:lang="atj">Kwei! (<w xml:lang="fra">Bonjour</w>!)</s> <s xml:lang="atj">Tan e ici matisihin?</s> </document>