Update hanzidentifier opencc to handle traditionl chinese

app-.py

@@ -1,140 +0,0 @@
-"""Prep __main__.py.
-
-https://share.streamlit.io/deploy
-    Advanced settings...
-        Python version
-            3.7
-            3.8
-            3.9*
-            3.10
-
-https://docs.streamlit.io/knowledge-base/using-streamlit/hide-row-indices-displaying-dataframe
-    Hide row indices when displaying a dataframe
-# CSS to inject contained in a string
-hide_table_row_index = '''
-            <style>
-            tbody th {display:none}
-            .blank {display:none}
-            </style>
-            '''
-# Inject CSS with Markdown
-st.markdown(hide_table_row_index, unsafe_allow_html=True)
-
-# Display a static table
-st.table(df)
-
-# Hide row indices with st.dataframe
-# CSS to inject contained in a string
-hide_dataframe_row_index = '''
-            <style>
-            .row_heading.level0 {display:none}
-            .blank {display:none}
-            </style>
-           '''
-# Inject CSS with Markdown
-st.markdown(hide_dataframe_row_index, unsafe_allow_html=True)
-
-# Display an interactive table
-st.dataframe(df)
-
-https://medium.com/@avra42/streamlit-python-cool-tricks-to-make-your-web-application-look-better-8abfc3763a5b
-hide_menu_style = '''
-        <style>
-        #MainMenu {visibility: hidden; }
-        footer {visibility: hidden;}
-        </style>
-        '''
-st.markdown(hide_menu_style, unsafe_allow_html=True)
-
-"""
-# pylint: disable=invalid-name
-import os
-import sys
-import time
-from pathlib import Path
-from types import SimpleNamespace
-from typing import Optional
-
-import loguru
-import logzero
-import pandas as pd
-import streamlit as st
-from loguru import logger as loggu
-from logzero import logger
-from set_loglevel import set_loglevel
-from streamlit import session_state as state
-
-from litbee import __version__, litbee
-from litbee.options import options
-
-# from litbee.files2df import files2df
-# from litbee.utils import sb_front_cover, instructions, menu_items
-# from litbee.ezbee_page import ezbee_page
-# from litbee.dzbee_page import dzbee_page
-# from litbee.xbee_page import xbee_page
-from litbee.utils import menu_items
-
-# from ezbee import ezbee
-
-curr_py = sys.version[:3]
-msg = f"Some packages litbee depends on can only run with Python 3.8, current python is {curr_py}, sorry..."
-assert curr_py == "3.8", msg
-
-os.environ["TZ"] = "Asia/Shanghai"
-time.tzset()
-os.environ["LOGLEVEL"] = "10"  # uncomment this in dev
-logzero.loglevel(set_loglevel())
-
-loggu.remove()
-_ = (
-    "<green>{time:YY-MM-DD HH:mm:ss}</green> | "
-    "<level>{level: <5}</level> | <level>{message}</level> "
-    "<cyan>{name}</cyan>:<cyan>{line}</cyan>"
-)
-loggu.add(
-    sys.stderr,
-    format=_,
-    level=set_loglevel(),
-    colorize=True,
-)
-
-# from PIL import Image
-# page_icon=Image.open("icon.ico"),
-st.set_page_config(
-    page_title=f"litbee v{__version__}",
-    # page_icon="🧊",
-    page_icon="🐝",
-    # layout="wide",
-    initial_sidebar_state="auto",  # "auto" or "expanded" or "collapsed",
-    menu_items=menu_items,
-)
-
-# pd.set_option("precision", 2)
-pd.set_option("display.precision", 2)
-pd.options.display.float_format = "{:,.2f}".format
-
-_ = dict(
-    beetype="ezbee",
-    src_filename="",
-    tgt_filename="",
-    src_fileio=b"",
-    tgt_fileio=b"",
-    src_file="",
-    tgt_file="",
-    list1=[""],
-    list2=[""],
-    df=None,
-    df_a=None,
-    df_s_a=None,
-)
-if "ns" not in state:
-    state.ns = SimpleNamespace(**_)
-state.ns.list = [*_]
-
-
-def main():
-    """Bootstrap."""
-    options()
-
-
-main()

app.py

@@ -135,6 +135,7 @@ _ = dict(
     beetype="ezbee",
     sourcetype="upload",
     sourcecount=2,
+    sent_ali=None,
     src_filename="",
     tgt_filename="",
     src_fileio=b"",

litbee/__main__.py

@@ -1,5 +1,7 @@
 """Run streamlit run app.py from __main__.py."""
+# pylint: disable=no-value-for-parameter
 import sys
+
 from streamlit import cli
 
 sys.argv = ["streamlit", "run", "app.py"]

litbee/app-.py

@@ -1,140 +0,0 @@
-"""Prep __main__.py.
-
-https://share.streamlit.io/deploy
-    Advanced settings...
-        Python version
-            3.7
-            3.8
-            3.9*
-            3.10
-
-https://docs.streamlit.io/knowledge-base/using-streamlit/hide-row-indices-displaying-dataframe
-    Hide row indices when displaying a dataframe
-# CSS to inject contained in a string
-hide_table_row_index = '''
-            <style>
-            tbody th {display:none}
-            .blank {display:none}
-            </style>
-            '''
-# Inject CSS with Markdown
-st.markdown(hide_table_row_index, unsafe_allow_html=True)
-
-# Display a static table
-st.table(df)
-
-# Hide row indices with st.dataframe
-# CSS to inject contained in a string
-hide_dataframe_row_index = '''
-            <style>
-            .row_heading.level0 {display:none}
-            .blank {display:none}
-            </style>
-           '''
-# Inject CSS with Markdown
-st.markdown(hide_dataframe_row_index, unsafe_allow_html=True)
-
-# Display an interactive table
-st.dataframe(df)
-
-https://medium.com/@avra42/streamlit-python-cool-tricks-to-make-your-web-application-look-better-8abfc3763a5b
-hide_menu_style = '''
-        <style>
-        #MainMenu {visibility: hidden; }
-        footer {visibility: hidden;}
-        </style>
-        '''
-st.markdown(hide_menu_style, unsafe_allow_html=True)
-
-"""
-# pylint: disable=invalid-name
-import os
-import sys
-import time
-from pathlib import Path
-from types import SimpleNamespace
-from typing import Optional
-
-import loguru
-import logzero
-import pandas as pd
-import streamlit as st
-from loguru import logger as loggu
-from logzero import logger
-from set_loglevel import set_loglevel
-from streamlit import session_state as state
-
-from litbee import __version__, litbee
-from litbee.options import options
-
-# from litbee.files2df import files2df
-# from litbee.utils import sb_front_cover, instructions, menu_items
-# from litbee.ezbee_page import ezbee_page
-# from litbee.dzbee_page import dzbee_page
-# from litbee.xbee_page import xbee_page
-from litbee.utils import menu_items
-
-# from ezbee import ezbee
-
-curr_py = sys.version[:3]
-msg = f"Some packages litbee depends on can only run with Python 3.8, current python is {curr_py}, sorry..."
-assert curr_py == "3.8", msg
-
-os.environ["TZ"] = "Asia/Shanghai"
-time.tzset()
-os.environ["LOGLEVEL"] = "10"  # uncomment this in dev
-logzero.loglevel(set_loglevel())
-
-loggu.remove()
-_ = (
-    "<green>{time:YY-MM-DD HH:mm:ss}</green> | "
-    "<level>{level: <5}</level> | <level>{message}</level> "
-    "<cyan>{name}</cyan>:<cyan>{line}</cyan>"
-)
-loggu.add(
-    sys.stderr,
-    format=_,
-    level=set_loglevel(),
-    colorize=True,
-)
-
-# from PIL import Image
-# page_icon=Image.open("icon.ico"),
-st.set_page_config(  # type: ignore
-    page_title=f"litbee v{__version__}",
-    # page_icon="🧊",
-    page_icon="🐝",
-    # layout="wide",
-    initial_sidebar_state="auto",  # "auto" or "expanded" or "collapsed",
-    menu_items=menu_items,
-)
-
-# pd.set_option("precision", 2)
-pd.set_option("display.precision", 2)
-pd.options.display.float_format = "{:,.2f}".format
-
-_ = dict(
-    beetype="ezbee",
-    src_filename="",
-    tgt_filename="",
-    src_fileio=b"",
-    tgt_fileio=b"",
-    src_file="",
-    tgt_file="",
-    list1=[""],
-    list2=[""],
-    df=None,
-    df_a=None,
-    df_s_a=None,
-)
-if "ns" not in state:
-    state.ns = SimpleNamespace(**_)
-state.ns.list = [*_]
-
-
-def main():
-    """Bootstrap."""
-    options()
-
-
-main()

litbee/app.py

@@ -55,13 +55,12 @@ from pathlib import Path
 from types import SimpleNamespace
 from typing import Optional
 
+import debee
+import dzbee
+import ezbee
 import loguru
 import logzero
 import pandas as pd
-import ezbee
-import dzbee
-import debee
-
 import streamlit as st
 from loguru import logger as loggu
 from logzero import logger
@@ -69,25 +68,25 @@ from set_loglevel import set_loglevel
 from streamlit import session_state as state
 
 from litbee import __version__
-# from litbee.options import options
+from litbee.home import home
+from litbee.info import info
+from litbee.multipage import Multipage
+from litbee.settings import settings
 
 # from litbee.files2df import files2df
 # from litbee.utils import sb_front_cover, instructions, menu_items
 # from litbee.ezbee_page import ezbee_page
 # from litbee.dzbee_page import dzbee_page
 # from litbee.xbee_page import xbee_page
-from litbee.utils import menu_items
+from litbee.utils import menu_items, style_css
+
+# from litbee.options import options
 
-from litbee.multipage import Multipage
 
 # from litbee.fetch_upload import fetch_upload
 # from litbee.fetch_paste import fetch_paste
 # from litbee.fetch_urls import fetch_urls
 
-from litbee.home import home
-from litbee.settings import settings
-from litbee.info import info
-from litbee.utils import style_css
 
 # from ezbee import ezbee
 

litbee/dzbee_page.py

@@ -1,20 +0,0 @@
-"""Display dzbee page."""
-import pandas as pd
-import streamlit as st
-from loguru import logger as loggu
-from logzero import logger
-
-
-def dzbee_page():
-    """Display dzbee page."""
-    # st.title('dzbee')
-    # st.write('Welcome to app1')
-
-    try:
-        df = st.session_state.ns.df
-    except Exception as exc:
-        logger.error(exc)
-        df = pd.DataFrame([[""]])
-
-    loggu.debug(" df ")
-    st.table(df)

litbee/ezbee_page.py

@@ -1,129 +0,0 @@
-"""Display ezbee page."""
-from functools import partial
-
-import logzero
-import pandas as pd
-import streamlit as st
-from ezbee import ezbee
-from ezbee.gen_pairs import gen_pairs
-from loguru import logger as loggu
-from logzero import logger
-from set_loglevel import set_loglevel
-from st_aggrid import AgGrid
-from st_aggrid.grid_options_builder import GridOptionsBuilder
-from streamlit import session_state as state
-
-logzero.loglevel(set_loglevel())
-
-
-def st_radio_horizontal(*args, **kwargs):
-    """Trick to have horizontal st radio to simulate tabs."""
-    col, _ = st.columns(2)
-    with col:
-        # st.write('<style> div[data-testid=column] > div > div > div > div.stRadio > div{flex-direction: row;}</style>', unsafe_allow_html=True)
-        # return st.radio(*args, **kwargs)
-        st.write(
-            "<style> div[data-testid=stSidebar] > div > div > div > div > div > div > div.stRadio > div{flex-direction: row;}</style>",
-            unsafe_allow_html=True,
-        )
-        return st.sidebar.radio(*args, **kwargs)
-
-
-def ezbee_page():
-    """Display ezbee page."""
-    # st.title('ezbee')
-    # st.write('### ezbee')
-    # st.write('Welcome to app1')
-
-    _ = """
-    try:
-        df = st.session_state.ns.df
-    except Exception as exc:
-        logger.error(exc)
-        df = pd.DataFrame([[""]])
-    # """
-
-    # st.table(df)  # looks alright
-
-    # stlyed pd dataframe?
-    # bigger, no pagination
-    # st.markdown(df.to_html(), unsafe_allow_html=True)
-
-    # ag_grid smallish, editable, probably slower
-
-    if "df" not in globals():
-        logger.debug(" df not defined, return")
-        return None
-
-    df = pd.DataFrame([["", "", ""]], columns=["text1", "text2", "llh"])
-
-    df_exp = st.expander("to be aligned", expanded=False)
-    with df_exp:
-        st.write(df)  # too small
-
-    _ = """
-    ag_exp = st.expander("done aligned")  # , expanded=False
-    with ag_exp:
-        agdf = AgGrid(
-            df,
-            # fit_columns_on_grid_load=True,
-            editable=True,
-            gridOptions=gridOptions,
-            key="ag_exp",
-        )
-    # """
-
-    list1 = [elm.strip() for elm in df.text1 if elm.strip()]
-    list2 = [elm.strip() for elm in df.text2 if elm.strip()]
-    logger.info("Processing data...")
-    try:
-        aset = ezbee(
-            list1,
-            list2,
-            # eps=eps,
-            # min_samples=min_samples,
-        )
-    except Exception as e:
-        logger.error("aset = ezbee(...) exc: %s", e)
-        aset = ""
-        return None
-
-    # fastlid changed logger.level is changed to 20
-    # turn back to loglevel
-    logzero.loglevel(set_loglevel())
-    if aset:
-        logger.debug("aset: %s...%s", aset[:3], aset[-3:])
-
-    # st.write(aset)
-
-    aligned_pairs = gen_pairs(list1, list2, aset)
-    if aligned_pairs:
-        logger.debug("%s...%s", aligned_pairs[:3], aligned_pairs[-3:])
-
-    df_a = pd.DataFrame(aligned_pairs, columns=["text1", "text2", "llh"])
-
-    # insert seq no
-    df_a.insert(0, "sn", range(len(df_a)))
-
-    gb = GridOptionsBuilder.from_dataframe(df_a)
-    gb.configure_pagination(paginationAutoPageSize=True)
-    options = {
-        "resizable": True,
-        "autoHeight": True,
-        "wrapText": True,
-        "editable": True,
-    }
-    gb.configure_default_column(**options)
-    gridOptions = gb.build()
-
-    st.write("aligned (double-click a cell to edit)")
-    agdf = AgGrid(
-        # df,
-        df_a,
-        gridOptions=gridOptions,
-        key="outside",
-        editable=True,
-        width="100%",
-        height=500,
-        # fit_columns_on_grid_load=True,
-    )

litbee/fetch_upload.py

@@ -3,31 +3,33 @@
 org ezbee_page.py.
 """
 # pylint: disable=invalid-name
-from functools import partial
+# pylint: disable=too-many-return-statements,too-many-branches,too-many-statements, too-many-locals
 import inspect
+
+from functools import partial
 from itertools import zip_longest
 from time import perf_counter
 
+import hanzidentifier
 import logzero
 import numpy as np
 import pandas as pd
 import streamlit as st
-from dzbee import dzbee  # noqa
-from ezbee import ezbee  # noqa
-from debee import debee  # noqa
 
-# from ezbee.gen_pairs import gen_pairs  # aset2pairs?
 from aset2pairs import aset2pairs
+from debee import debee  # noqa
+from dzbee import dzbee  # noqa
+from ezbee import ezbee  # noqa
 from fastlid import fastlid
 from icecream import ic
 from loguru import logger as loggu
 from logzero import logger
 from set_loglevel import set_loglevel
-from st_aggrid import AgGrid, GridUpdateMode, GridOptionsBuilder
-# from st_aggrid.grid_options_builder import GridOptionsBuilder
+from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode
+
 from streamlit import session_state as state
 
-# logzero.loglevel(set_loglevel())
+from litbee.t2s import t2s
 
 
 def fetch_upload():  # noqa
@@ -43,7 +45,7 @@ def fetch_upload():  # noqa
         return None
 
     # src_fileio tgt_fileio
-    with st.form(key='upload_in_form'):
+    with st.form(key="upload_in_form"):
         _ = st.expander(f"{state.ns.beetype}: Pick two files", expanded=True)
         with _:
             col1, col2 = st.columns(2)
@@ -67,7 +69,7 @@ def fetch_upload():  # noqa
                     key="tgt_text",
                     # accept_multiple_files=True,
                 )
-        submitted = st.form_submit_button('Submit')
+        submitted = st.form_submit_button("Submit")
 
     # logger.debug(" len(src_fileio): %s", len(src_fileio))
     # logger.debug(" len(tgt_fileio): %s", len(tgt_fileio))
@@ -127,10 +129,12 @@ def fetch_upload():  # noqa
     if not (filename1 or filename2):
         st.write("|  no file uploaded")
         return None
-    elif not filename1:
+
+    if not filename1:
         st.write("|  file1 not ready")
         return None
-    elif not filename2:
+
+    if not filename2:
         st.write("|  file2 not ready")
         return None
 
@@ -216,16 +220,18 @@ def fetch_upload():  # noqa
         # logger.debug("fn.__doc__: %s", fn.__doc__)
         logger.debug("fn.__name__: %s", fn.__name__)
 
-        from inspect import getabsfile
-        logger.debug("getabsfile(fn): %s", getabsfile(fn))
+        # from inspect import getabsfile
+        # logger.debug("getabsfile(fn): %s", getabsfile(fn))
+
+        # convert to simplified chinese if is_tranditional
 
         with st.spinner(" diggin..."):
             then = perf_counter()
             try:
                 # aset = ezbee/dzbee/debee
                 aset = globals()[state.ns.beetype](
-                    list1,
-                    list2,
+                    t2s(list1),  # t2s, handle trand.chinese
+                    t2s(list2),
                     # eps=eps,
                     # min_samples=min_samples,
                 )
@@ -267,7 +273,9 @@ def fetch_upload():  # noqa
         logger.debug("%s...%s", aligned_pairs[:1], aligned_pairs[-1:])
         # logger.debug("aligned_pairs[:20]: \n%s", aligned_pairs[:20])
 
-    df_a = pd.DataFrame(aligned_pairs, columns=["text1", "text2", "llh"], dtype="object")
+    df_a = pd.DataFrame(
+        aligned_pairs, columns=["text1", "text2", "llh"], dtype="object"
+    )
 
     # if set_loglevel() <= 10:
     _ = st.expander("done aligned")
@@ -301,5 +309,7 @@ def fetch_upload():  # noqa
             # width="100%",  # width parameter is deprecated
             height=750,
             # fit_columns_on_grid_load=True,
-            update_mode=GridUpdateMode.MODEL_CHANGED
+            update_mode=GridUpdateMode.MODEL_CHANGED,
         )
+
+    return None

litbee/home.py

@@ -2,9 +2,10 @@
 
 org ezbee_page.py.
 """
+import inspect
+
 # pylint: disable=invalid-name
 from functools import partial
-import inspect
 from itertools import zip_longest
 from time import perf_counter
 
@@ -12,18 +13,19 @@ import logzero
 import numpy as np
 import pandas as pd
 import streamlit as st
-from dzbee import dzbee  # noqa
-from ezbee import ezbee  # noqa
-from debee import debee  # noqa
 
 # from ezbee.gen_pairs import gen_pairs  # aset2pairs?
 from aset2pairs import aset2pairs
+from debee import debee  # noqa
+from dzbee import dzbee  # noqa
+from ezbee import ezbee  # noqa
 from fastlid import fastlid
 from icecream import ic
 from loguru import logger as loggu
 from logzero import logger
 from set_loglevel import set_loglevel
-from st_aggrid import AgGrid, GridUpdateMode, GridOptionsBuilder
+from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode
+
 # from st_aggrid.grid_options_builder import GridOptionsBuilder
 from streamlit import session_state as state
 
@@ -43,7 +45,7 @@ def home():  # noqa
         return None
 
     # src_fileio tgt_fileio
-    with st.form(key='upload_in_form'):
+    with st.form(key="upload_in_form"):
         _ = st.expander(f"{state.ns.beetype}: Pick two files", expanded=True)
         with _:
             col1, col2 = st.columns(2)
@@ -67,7 +69,7 @@ def home():  # noqa
                     key="tgt_text",
                     # accept_multiple_files=True,
                 )
-        submitted = st.form_submit_button('Submit')
+        submitted = st.form_submit_button("Submit")
 
     # logger.debug(" len(src_fileio): %s", len(src_fileio))
     # logger.debug(" len(tgt_fileio): %s", len(tgt_fileio))
@@ -217,6 +219,7 @@ def home():  # noqa
         logger.debug("fn.__name__: %s", fn.__name__)
 
         from inspect import getabsfile
+
         logger.debug("getabsfile(fn): %s", getabsfile(fn))
 
         with st.spinner(" diggin..."):
@@ -267,7 +270,9 @@ def home():  # noqa
         logger.debug("%s...%s", aligned_pairs[:1], aligned_pairs[-1:])
         # logger.debug("aligned_pairs[:20]: \n%s", aligned_pairs[:20])
 
-    df_a = pd.DataFrame(aligned_pairs, columns=["text1", "text2", "llh"], dtype="object")
+    df_a = pd.DataFrame(
+        aligned_pairs, columns=["text1", "text2", "llh"], dtype="object"
+    )
 
     # if set_loglevel() <= 10:
     _ = st.expander("done aligned")
@@ -301,5 +306,5 @@ def home():  # noqa
             # width="100%",  # width parameter is deprecated
             height=750,
             # fit_columns_on_grid_load=True,
-            update_mode=GridUpdateMode.MODEL_CHANGED
+            update_mode=GridUpdateMode.MODEL_CHANGED,
         )

litbee/info.py

@@ -1,10 +1,11 @@
 """Present info about litbee."""
-import ezbee
-import dzbee
-import debee
-
 from textwrap import dedent
+
+import debee
+import dzbee
+import ezbee
 import streamlit as st
+
 from litbee import __version__
 
 # from litbee.utils import style_css
@@ -23,7 +24,9 @@ msg = dedent(
     <li> xbee/bumblebee: other language pairs, normal para-align
     </ul>
 
-    The algorithm for fast para-align is home-brewn. Two sent-align algorithms are used: one based on Gale-Church, the other machine learning.
+    The algorithm for fast para-align is home-brewn. Two
+    sent-align algorithms are used: one based on Gale-Church,
+    the other machine learning.
     </div>
     """
 ).strip()

litbee/multipage.py

@@ -13,29 +13,24 @@ class Multipage:
     """Framework for combining multiple streamlit applications."""
 
     def __init__(self) -> None:
-        """Constructor class to generate a list which will store all our applications as an instance variable."""
+        """Construct class to generate a list which will store all our applications as an instance variable."""
         self.pages = []
 
     def add_page(self, title, icon, func) -> None:
-        """Class Method to Add pages to the project
+        """Class Method to Add pages to the project.
 
         Args:
             title ([str]): The title of page which we are adding to the list of apps
-
+            icon: icon from streamlit-menu-option
             func: Python function to render this page in Streamlit
         """
-
-        self.pages.append(
-            {
-                "title": title,
-                "icon": icon,
-                "function": func
-            }
-        )
+        self.pages.append({"title": title, "icon": icon, "function": func})
 
     def run(self):
-        # Drodown to select the page to run
-        st.markdown("""
+        """Dropdown to select the page to run."""
+        # Dropdown to select the page to run
+        st.markdown(
+            """
             <style>
                 section[data-testid="stSidebar"] > div:first-of-type {
                     background-color: var(--secondary-background-color);
@@ -51,12 +46,18 @@ class Multipage:
                     padding: 1rem 0;
                 }
             </style>
-        """, unsafe_allow_html=True)
+        """,
+            unsafe_allow_html=True,
+        )
 
         with st.sidebar:
-            selected = option_menu(None, [page["title"] for page in self.pages],
-                                   icons=[page["icon"] for page in self.pages],
-                                   menu_icon="cast", default_index=0)
+            selected = option_menu(
+                None,
+                [page["title"] for page in self.pages],
+                icons=[page["icon"] for page in self.pages],
+                menu_icon="cast",
+                default_index=0,
+            )
 
         # Run the selected page
         for index, item in enumerate(self.pages):

litbee/options.py

@@ -7,12 +7,13 @@ from loguru import logger as loggu
 from logzero import logger
 from streamlit import session_state as state
 
+from litbee.fetch_paste import fetch_paste
+
 # from litbee.ezbee_page import ezbee_page
 # from litbee.dzbee_page import dzbee_page
 # from litbee.xbee_page import xbee_page
 from litbee.fetch_upload import fetch_upload
 from litbee.fetch_urls import fetch_urls
-from litbee.fetch_paste import fetch_paste
 from litbee.files2df import files2df
 from litbee.utils import instructions, sb_front_cover
 
@@ -49,7 +50,7 @@ def options():
     # if beetype not in ["ezbee", "dzbee"]:
     if beetype not in ["ezbee", "dzbee", "debee"]:
         st.write("Coming soon")
-        return None
+        return
 
     # multi-page setup
     menu = {

litbee/settings.py

@@ -1,6 +1,7 @@
 """Prep Settings/Options page."""
 # pylint: disable=invalid-name
 from functools import partial
+
 import streamlit as st
 from loguru import logger as loggu
 from logzero import logger
@@ -10,7 +11,8 @@ from streamlit import session_state as state
 def settings():
     """Prep  Settings/Options page.
 
-    Refer to options.py"""
+    Refer to options.py
+    """
     # horizotal radio
     st.write(
         "<style>div.row-widget.stRadio > div{flex-direction:row;}</style>",
@@ -24,6 +26,7 @@ def settings():
     # col1, col2 = st.columns(2)
 
     # with col1:
+    _ = "ezbee: english-chinese; dzbee: german-chinese, debee: german-english; xbee: other language pairs (slow, approx.1000 pairs/3 min) | ezbee: 英/中; dzbee: 德/中, debee: 德/英; xbee: 其他语言对（慢, 约1000对/3分钟）"
     try:
         index = beetype_list.index(state.ns.beetype)
     except Exception as e:
@@ -34,7 +37,7 @@ def settings():
         beetype_list,
         index=index,
         format_func=lambda x: f"{x:<7} |",
-        help="ezbee: english-chinese; dzbee: german-chinese, debee: german-english",
+        help=_,
     )
     state.ns.beetype = beetype
 
@@ -70,11 +73,27 @@ def settings():
     )
     state.ns.sourcecount = sourcecount
 
+    sentali_list = [None, "fast", "slow"]
+    try:
+        index = sentali_list.index(state.ns.sentali)
+    except Exception as e:
+        logger.error("sentali index error: %s, setting to 0", e)
+        index = 0
+    sentali = st.radio(
+        "Sent Align",
+        sentali_list,
+        index=index,
+        format_func=lambda x: f"{str(x):<4} |",
+        help="None: no sent align; fast: gale-church; slow: machine-learning",
+        disabled=True,
+    )
+    state.ns.sentali = sentali
+
     # show state.ns[:6]
     loggu.debug(f" state.ns.list: {state.ns.list}")
 
     # beetype, sourcetype, sourcecount, filename1, filename2
-    _ = map(partial(getattr, state.ns), state.ns.list[:5])
+    _ = map(partial(getattr, state.ns), state.ns.list[:6])
     logger.debug(" state.ns.list[:3]: %s", str([*_]))
 
     # st.write(f"run: {state.ns.count}")

litbee/t2s.py

@@ -0,0 +1,39 @@
+"""Convert list to simlified Chinese for traditional Chinese, do nothing otherwise."""
+# pylint: disable=invalid-name
+from typing import List
+import hanzidentifier
+from logzero import logger
+from opencc import OpenCC
+
+convert = OpenCC('t2s').convert
+
+
+def t2s(lst: List[str]) -> List[str]:
+    """Convert list to simlified Chinese for traditional Chinese, do nothing otherwise.
+
+    Args:
+        list of strings
+
+    Returns:
+        list of strings
+    """
+    try:
+        # lst[:1000] strim down for extremely large docs
+        _ = hanzidentifier.identify(" ".join(lst[:1000]))
+    except Exception as e:
+        logger.warning("hanzidentifier.is_traditional error: %s, settin to simplified.", e)
+        _ = hanzidentifier.SIMP  # 2: simplified
+
+    if _ not in [hanzidentifier.TRAD, hanzidentifier.MIXED]:
+        return lst
+
+    res = []
+    for line in lst:
+        try:
+            _ = convert(line)
+        except Exception as e:
+            logger.warning("ts2 error: %s, setting to original", e)
+            _ = line
+        res.append(_)
+
+    return res

litbee/utils.py

@@ -40,7 +40,7 @@ msg = dedent(
 
 
 def sb_front_cover():
-    """Prep front cover for sidebar"""
+    """Prep front cover for sidebar."""
     st.sidebar.markdown(f"### litbee {__version__} ")
 
     sb_tit_expander = st.sidebar.expander("More info (click to toggle)", expanded=False)
@@ -65,6 +65,7 @@ intructins = dedent(
 
 
 def instructions():
+    """Prep msg."""
     logger.debug("instructions entry")
     back_cover_expander = st.expander("Instructions")
     with back_cover_expander:

poetry.lock

@@ -519,6 +519,17 @@ python-versions = "*"
 [package.extras]
 test = ["pytest", "sphinx", "sphinx-autobuild", "wheel", "twine"]
 
+[[package]]
+name = "hanzidentifier"
+version = "1.0.2"
+description = "Python module that identifies Chinese text as Simplified or Traditional."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+zhon = ">=1.1.3"
+
 [[package]]
 name = "icecream"
 version = "2.1.2"
@@ -1038,6 +1049,14 @@ category = "main"
 optional = false
 python-versions = ">=3.8"
 
+[[package]]
+name = "opencc-python-reimplemented"
+version = "0.1.6"
+description = "OpenCC made with Python"
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "packaging"
 version = "21.3"
@@ -1129,6 +1148,17 @@ python-versions = ">=3.7"
 docs = ["olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-issues (>=3.0.1)", "sphinx-removed-in", "sphinx-rtd-theme (>=1.0)", "sphinxext-opengraph"]
 tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"]
 
+[[package]]
+name = "plotly"
+version = "5.8.0"
+description = "An open-source, interactive data visualization library for Python"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+tenacity = ">=6.2.0"
+
 [[package]]
 name = "pluggy"
 version = "1.0.0"
@@ -1689,6 +1719,17 @@ python-versions = ">=3.6"
 [package.dependencies]
 streamlit = ">=0.63"
 
+[[package]]
+name = "tenacity"
+version = "8.0.1"
+description = "Retry code until it succeeds"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.extras]
+doc = ["reno", "sphinx", "tornado (>=4.5)"]
+
 [[package]]
 name = "terminado"
 version = "0.15.0"
@@ -1952,6 +1993,14 @@ category = "main"
 optional = false
 python-versions = ">=3.4"
 
+[[package]]
+name = "zhon"
+version = "1.1.5"
+description = "Zhon provides constants used in Chinese text processing."
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "zipp"
 version = "3.8.0"
@@ -1967,7 +2016,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8.3"
-content-hash = "0001273edc2530f75a45f1baa8b689876cba18a12373a5f50776c6d758fbaee2"
+content-hash = "eabc5b9c944b380d2a60b4ec1b1f218f4b1a3aea1426c7fb75fdc51d4889e57e"
 
 [metadata.files]
 about-time = [
@@ -2259,6 +2308,9 @@ gitpython = [
 grapheme = [
     {file = "grapheme-0.6.0.tar.gz", hash = "sha256:44c2b9f21bbe77cfb05835fec230bd435954275267fea1858013b102f8603cca"},
 ]
+hanzidentifier = [
+    {file = "hanzidentifier-1.0.2.tar.gz", hash = "sha256:793a298430aa9a9d6ab344dc0ca0ab4bd1161d88c7da941d6554571093003cba"},
+]
 icecream = [
     {file = "icecream-2.1.2-py2.py3-none-any.whl", hash = "sha256:04b9cea4d9931cf6960db0430ebf11fa34464ce7152e384ddf50f96d25b201b4"},
     {file = "icecream-2.1.2.tar.gz", hash = "sha256:09300b2d1c678712410cbd47c95198eb1b580f66f311a554ccd6b9e758ece0ee"},
@@ -2489,6 +2541,10 @@ numpy = [
     {file = "numpy-1.22.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0791fbd1e43bf74b3502133207e378901272f3c156c4df4954cad833b1380207"},
     {file = "numpy-1.22.4.zip", hash = "sha256:425b390e4619f58d8526b3dcf656dde069133ae5c240229821f01b5f44ea07af"},
 ]
+opencc-python-reimplemented = [
+    {file = "opencc-python-reimplemented-0.1.6.tar.gz", hash = "sha256:6e4eaae2bd6b04d5c1b5bd7f3f87904ba2f1caa982203fdf2610c8261e47ee24"},
+    {file = "opencc_python_reimplemented-0.1.6-py3.8.egg", hash = "sha256:3071d7ddcecc1b5129434e713e35f73aab9f5bd507d728c908acdbb48879194d"},
+]
 packaging = [
     {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"},
     {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"},
@@ -2576,6 +2632,10 @@ pillow = [
     {file = "Pillow-9.1.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:baf3be0b9446a4083cc0c5bb9f9c964034be5374b5bc09757be89f5d2fa247b8"},
     {file = "Pillow-9.1.1.tar.gz", hash = "sha256:7502539939b53d7565f3d11d87c78e7ec900d3c72945d4ee0e2f250d598309a0"},
 ]
+plotly = [
+    {file = "plotly-5.8.0-py2.py3-none-any.whl", hash = "sha256:0e6e2382aafe2b2978d2c1b10ea93e73ad1ec80fa9a195ff6eea62af7905dfdc"},
+    {file = "plotly-5.8.0.tar.gz", hash = "sha256:58cef3292f5994d82154d51fbc7338c48009fc47ea32ffe052ad29aaa15e0df9"},
+]
 pluggy = [
     {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"},
     {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"},
@@ -3107,6 +3167,10 @@ streamlit-option-menu = [
     {file = "streamlit-option-menu-0.3.2.tar.gz", hash = "sha256:69d1aef6f30f83f29eda3dc9667733bc2e28cd640eb17b4b6ca315f633484c52"},
     {file = "streamlit_option_menu-0.3.2-py3-none-any.whl", hash = "sha256:0b7eae3ffdb0276c81d15750465c72957d57d2f766cb027c586d053519731178"},
 ]
+tenacity = [
+    {file = "tenacity-8.0.1-py3-none-any.whl", hash = "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a"},
+    {file = "tenacity-8.0.1.tar.gz", hash = "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f"},
+]
 terminado = [
     {file = "terminado-0.15.0-py3-none-any.whl", hash = "sha256:0d5f126fbfdb5887b25ae7d9d07b0d716b1cc0ccaacc71c1f3c14d228e065197"},
     {file = "terminado-0.15.0.tar.gz", hash = "sha256:ab4eeedccfcc1e6134bfee86106af90852c69d602884ea3a1e8ca6d4486e9bfe"},
@@ -3263,6 +3327,9 @@ xlsxwriter = [
     {file = "XlsxWriter-3.0.3-py3-none-any.whl", hash = "sha256:df0aefe5137478d206847eccf9f114715e42aaea077e6a48d0e8a2152e983010"},
     {file = "XlsxWriter-3.0.3.tar.gz", hash = "sha256:e89f4a1d2fa2c9ea15cde77de95cd3fd8b0345d0efb3964623f395c8c4988b7f"},
 ]
+zhon = [
+    {file = "zhon-1.1.5.tar.gz", hash = "sha256:793723575c46f10ace8846c579ce740b04c73e2aa583e04e000aedbd4a47f87f"},
+]
 zipp = [
     {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"},
     {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"},

pylintrc

@@ -0,0 +1,7 @@
+[EXCEPTIONS]
+# C0301 line too long
+# R0801 dup
+# C0103 invalid-name
+# W0612 unused-variable
+# W0611 unused-import
+disable=W0703,R0801,C0103,C0301,W0612,W0611

pyproject.toml

@@ -22,6 +22,9 @@ streamlit = "^1.9.2"
 debee = "^0.1.0-alpha.2"
 ezbee = "^0.1.0"
 streamlit-option-menu = "^0.3.2"
+plotly = "^5.8.0"
+hanzidentifier = "^1.0.2"
+opencc-python-reimplemented = "^0.1.6"
 
 [tool.poe.executor]
 type = "poetry"
@@ -33,7 +36,9 @@ build = "poetry build"
 _publish = "poetry publish"
 release = ["test", "build", "_publish"]
 lint = { cmd = "pylint litbee" }
-format = "black tests litbee"
+isort = "isort tests litbee"
+black = "black tests litbee"
+format = ["isort", "black"]
 docstyle = "pydocstyle --convention=google tests litbee"
 tunnel = {cmd = "ssh -CN ip_or_hostname_defined_in_hosts -L 9091:127.0.0.1:9091"}
 

run-flake8.sh

@@ -0,0 +1 @@
+flake8 --ignore F401,E501,F841

tests/test_t2s.py

@@ -0,0 +1,15 @@
+"""Test t2s."""
+from litbee.t2s import t2s
+
+
+def test_t2s1():
+    """Test trivial t2s."""
+    assert t2s(["", ""]) == ["", ""]
+    assert t2s(["a\nb", ""]) == ["a\nb", ""]
+    assert t2s(["a\n\nb\n", ""]) == ["a\n\nb\n", ""]
+
+
+def test_t2s2():
+    """Test t2s."""
+    assert t2s(["需攜帶", "需攜帶"]) == ["需携带", "需携带"]
+    assert t2s(["需攜帶\n\n需攜帶\n", "需携带\n"]) == ["需携带\n\n需携带\n", "需携带\n"]