add cas, lzz, anp and edit misc.

app.py

@@ -24,10 +24,10 @@ def render_home_table():
     df_data['ISO Code'] = df_data['ISO Code'].astype(str)  # Convert to string
     df_data['Number of Sites'] = df_data.apply(lambda row: '<a href="/?isocode={}&site=True" target="_self">{}</a>'.format(row['ISO Code'], row['Number of Sites']), axis=1)
     df_data['Number of Links'] = df_data.apply(lambda row: '<a href="/?isocode={}&links=True" target="_self">{}</a>'.format(row['ISO Code'], row['Number of Links']), axis=1)
-    df_data["Supported by MADLAD-400, flores, and Glot500"] = df_data.apply(lambda row: color_mapping([row["Supported by allenai/MADLAD-400"] + row["Supported by facebook/flores"] +  row["Supported by cis-lmu/Glot500"]]), axis =1)
+    df_data["Supported by MADLAD400 & FLORES & GLOT500"] = df_data.apply(lambda row: color_mapping([row["Supported by allenai/MADLAD-400"] + row["Supported by facebook/flores"] +  row["Supported by cis-lmu/Glot500"]]), axis =1)
 
     # Display the table
-    df_data = df_data[['ISO Code', 'Language Name', 'Family', 'Subgrouping', 'Number of Sites', 'Number of Links', 'Number of Speakers', 'Supported by MADLAD-400, flores, and Glot500']]
+    df_data = df_data[['ISO Code', 'Language Name', 'Family', 'Subgrouping', 'Number of Sites', 'Number of Links', 'Number of Speakers', 'Supported by MADLAD400 & FLORES & GLOT500']]
     st.write(df_to_html(df_data), unsafe_allow_html=True)
 
 
@@ -103,6 +103,8 @@ def main():
     else:
         # show home
         render_metadata()
-        st.markdown("**GlotWeb** is an indexing service for low-resource languages. It indexes sites or links written in each language. This list can be used to create raw text or parallel corpora and to study low-resource languages on the web. We also compare the level of support for these languages in the 3 big datasets of low-resource languages (🟥 0/3 < 🟧 1/3 < 🟨 2/3 < 🟩 3/3).\n")
+        st.markdown("**GlotWeb** is an indexing service for low-resource languages. It indexes **non-religous** sites or links written in each language. This list can be used to create raw text or parallel corpora and to study low-resource languages on the web.\n")
         render_home_table()
+        st.markdown("\n\nWe compare the level of support for these languages in the three big datasets ([MADLAD-400](https://huggingface.co/datasets/allenai/MADLAD-400), [FLORES200](https://huggingface.co/datasets/facebook/flores), [GLOT500](https://huggingface.co/datasets/cis-lmu/Glot500)) of low-resource languages (🟥 0/3 < 🟧 1/3 < 🟨 2/3 < 🟩 3/3). Although the support in these datasets for some of these languages could be just the religious texts.", unsafe_allow_html=True)
+
 main()

languages/anp_Deva.json

@@ -0,0 +1,20 @@
+{
+    "Language Name": "Angika",
+    "Family": "Indo-European",
+    "Subgrouping": "Eastern Indo-Aryan",
+    "Number of Speakers": "15_000_000",
+    "Supported by allenai/MADLAD-400": 1,
+    "Supported by facebook/flores": 0,
+    "Supported by cis-lmu/Glot500": 0,
+    "Sites": [
+      {
+        "Site Name": "angika.com",
+        "Site URL": "https://www.angika.com/#angika",
+        "Category": "blog",
+        "Confidence": "🟩",
+        "Info": "confirmed by glotlid and webpage metadata.",
+        "Possible Parallel Languages": "eng_Latn, hin_Deva",
+        "Links": []
+      }
+    ]
+  }

languages/cas_Latn.json

@@ -0,0 +1,23 @@
+{
+    "Language Name": "Tsimané",
+    "Family": "Mosetén-Chimané",
+    "Subgrouping": "",
+    "Number of Speakers": "5_300",
+    "Supported by allenai/MADLAD-400": 0,
+    "Supported by facebook/flores": 0,
+    "Supported by cis-lmu/Glot500": 0,
+    "Sites": [
+      {
+        "Site Name": "tsimanelinguisticouniverso.wordpress.com",
+        "Site URL": "https://tsimanelinguisticouniverso.wordpress.com/",
+        "Category": "blog",
+        "Confidence": "🟩",
+        "Info": "confirmed by glotlid and webpage metadata - some posts have spanish translation.",
+        "Possible Parallel Languages": "spa_Latn",
+        "Links": ["https://tsimanelinguisticouniverso.wordpress.com/2015/07/28/49/",
+                  "https://tsimanelinguisticouniverso.wordpress.com/2015/07/28/conozcamos-la-lengua-tsimane-no-1/",
+                "https://tsimanelinguisticouniverso.wordpress.com/2015/05/07/jun-chuc-carijtacdye-yu/",
+              "https://tsimanelinguisticouniverso.wordpress.com/2015/02/17/patuju/"]
+      }
+    ]
+  }

languages/hac_Arab.json

@@ -8,7 +8,7 @@
     "Supported by cis-lmu/Glot500": 0,
     "Sites": [
       {
-        "Site Name": "anfsorani.com/هۆرامی",
+        "Site Name": "anfsorani.com",
         "Site URL": "https://anfsorani.com/هۆرامی",
         "Category": "news",
         "Confidence": "🟩",

languages/lzz_Latn.json

@@ -0,0 +1,21 @@
+{
+    "Language Name": "Laz",
+    "Family": "Kartvelian",
+    "Subgrouping": "Zan",
+    "Number of Speakers": "22_000",
+    "Supported by allenai/MADLAD-400": 0,
+    "Supported by facebook/flores": 0,
+    "Supported by cis-lmu/Glot500": 0,
+    "Sites": [
+      {
+        "Site Name": "kolkhoba.org",
+        "Site URL": "https://www.kolkhoba.org/lazuri.htm",
+        "Category": "articles",
+        "Confidence": "🟩",
+        "Info": "confirmed by webpage metadata",
+        "Possible Parallel Languages": "tur_Latn",
+        "Links": []
+      }
+    ]
+  }
+  

languages/snk_Latn.json

@@ -8,7 +8,7 @@
     "Supported by cis-lmu/Glot500": 0,
     "Sites": [
       {
-        "Site Name": "soninkara.com/snk/",
+        "Site Name": "soninkara.com",
         "Site URL": "http://www.soninkara.com/snk/",
         "Category": "news,forums",
         "Confidence": "🟩",

languages/tet_Latn.json

@@ -17,13 +17,13 @@
         "Links": []
       },
       {
-        "Site Name": "timor-leste.gov.tl/?lang=tp",
+        "Site Name": "timor-leste.gov.tl",
         "Site URL": "http://timor-leste.gov.tl/?lang=tp",
         "Category": "government",
         "Confidence": "🟩",
         "Info": "confirmed by webpage metadata as tet_Latn",
         "Possible Parallel Languages": "eng_Latn, por_Latn",
-        "Links": []
+        "Links": ["http://timor-leste.gov.tl/wp-content/uploads/2021/08/TT-2021-08-24-debate-PN_autorizacao_EE17.pdf"]
       },
       {
         "Site Name": "belun.tl",
@@ -32,7 +32,7 @@
         "Confidence": "🟩",
         "Info": "confirmed by webpage metadata as tet_Latn",
         "Possible Parallel Languages": "eng_Latn",
-        "Links": []
+        "Links": ["https://belun.tl/wp-content/uploads/2015/12/Relatoriu-Politika-CPD-RDTL.pdf"]
       },
       {
         "Site Name": "tempotimor.com",