Upload 11 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+catmus-medieval/language_model.binary filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,211 @@
+import streamlit as st
+import warnings
+warnings.simplefilter("ignore", UserWarning)
+
+from uuid import uuid4
+from laia.scripts.htr.decode_ctc import run as decode
+from laia.common.arguments import CommonArgs, DataArgs, TrainerArgs, DecodeArgs
+import sys
+from tempfile import NamedTemporaryFile, mkdtemp
+from pathlib import Path
+from contextlib import redirect_stdout
+import re
+from PIL import Image
+from bidi.algorithm import get_display
+import multiprocessing
+from ultralytics import YOLO
+import cv2
+import numpy as np
+import pandas as pd
+import logging
+from typing import List, Optional
+
+# Configure logging
+logging.getLogger("lightning.pytorch").setLevel(logging.ERROR)
+
+# Load YOLOv8 model
+model = YOLO('model.pt')
+images = Path(mkdtemp())
+DEFAULT_HEIGHT = 128
+TEXT_DIRECTION = "LTR"
+NUM_WORKERS = multiprocessing.cpu_count()
+
+# Regex pattern for extracting results
+IMAGE_ID_PATTERN = r"(?P<image_id>[-a-z0-9]{36})"
+CONFIDENCE_PATTERN = r"(?P<confidence>[0-9.]+)"  # For line
+TEXT_PATTERN = r"\s*(?P<text>.*)\s*"
+LINE_PREDICTION = re.compile(rf"{IMAGE_ID_PATTERN} {CONFIDENCE_PATTERN} {TEXT_PATTERN}")
+
+def get_width(image, height=DEFAULT_HEIGHT):
+    aspect_ratio = image.width / image.height
+    return height * aspect_ratio
+
+def simplify_polygons(polygons: List[np.ndarray], approx_level: float = 0.01) -> List[Optional[np.ndarray]]:
+    """Simplify polygon contours using Douglas-Peucker algorithm.
+    
+    Args:
+        polygons: List of polygon contours
+        approx_level: Approximation level (0-1), lower values mean more simplification
+        
+    Returns:
+        List of simplified polygons (or None for invalid polygons)
+    """
+    result = []
+    for polygon in polygons:
+        if len(polygon) < 4:
+            result.append(None)
+            continue
+
+        perimeter = cv2.arcLength(polygon, True)
+        approx = cv2.approxPolyDP(polygon, approx_level * perimeter, True)
+        if len(approx) < 4:
+            result.append(None)
+            continue
+
+        result.append(approx.squeeze())
+    return result
+
+def predict(model_name, input_img):
+    model_dir = 'catmus-medieval'
+    temperature = 2.0
+    batch_size = 1
+
+    weights_path = f"{model_dir}/weights.ckpt"
+    syms_path = f"{model_dir}/syms.txt"
+    language_model_params = {"language_model_weight": 1.0}
+    use_language_model = True
+    if use_language_model:
+        language_model_params.update({
+            "language_model_path": f"{model_dir}/language_model.binary",
+            "lexicon_path": f"{model_dir}/lexicon.txt",
+            "tokens_path": f"{model_dir}/tokens.txt",
+        })
+
+    common_args = CommonArgs(
+        checkpoint="weights.ckpt",
+        train_path=f"{model_dir}",
+        experiment_dirname="",
+    )
+
+    data_args = DataArgs(batch_size=batch_size, color_mode="L")
+    trainer_args = TrainerArgs(progress_bar_refresh_rate=0)
+    decode_args = DecodeArgs(
+        include_img_ids=True,
+        join_string="",
+        convert_spaces=True,
+        print_line_confidence_scores=True,
+        print_word_confidence_scores=False,
+        temperature=temperature,
+        use_language_model=use_language_model,
+        **language_model_params,
+    )
+
+    with NamedTemporaryFile() as pred_stdout, NamedTemporaryFile() as img_list:
+        image_id = uuid4()
+        input_img = input_img.resize((int(get_width(input_img)), DEFAULT_HEIGHT))
+        input_img.save(f"{images}/{image_id}.jpg")
+        Path(img_list.name).write_text("\n".join([str(image_id)]))
+
+        with redirect_stdout(open(pred_stdout.name, mode="w")):
+            decode(
+                syms=str(syms_path),
+                img_list=img_list.name,
+                img_dirs=[str(images)],
+                common=common_args,
+                data=data_args,
+                trainer=trainer_args,
+                decode=decode_args,
+                num_workers=1,
+            )
+            sys.stdout.flush()
+        predictions = Path(pred_stdout.name).read_text().strip().splitlines()
+
+    _, score, text = LINE_PREDICTION.match(predictions[0]).groups()
+    if TEXT_DIRECTION == "RTL":
+        return input_img, {"text": get_display(text), "score": score}
+    else:
+        return input_img, {"text": text, "score": score}
+
+def process_image(image):
+    # Perform inference on an image, select textline only
+    results = model(image, classes=0)
+    
+    img_cv2 = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+    masks = results[0].masks
+    polygons = []
+    texts = []
+    
+    if masks is not None:
+        # Get masks data and original image dimensions
+        masks = masks.data.cpu().numpy()
+        img_height, img_width = img_cv2.shape[:2]
+        
+        # Get bounding boxes in xyxy format
+        boxes = results[0].boxes.xyxy.cpu().numpy()
+        
+        # Sort by y-coordinate of the top-left corner
+        sorted_indices = np.argsort(boxes[:, 1])
+        masks = masks[sorted_indices]
+        boxes = boxes[sorted_indices]
+        
+        for i, (mask, box) in enumerate(zip(masks, boxes)):
+            # Scale the mask to original image size
+            mask = cv2.resize(mask.squeeze(), (img_width, img_height), interpolation=cv2.INTER_LINEAR)
+            mask = (mask > 0.5).astype(np.uint8) * 255  # Apply threshold
+            
+            # Convert mask to polygon
+            contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+            
+            if contours:
+                # Get the largest contour
+                largest_contour = max(contours, key=cv2.contourArea)
+                simplified_polygon = simplify_polygons([largest_contour])[0]
+                
+                if simplified_polygon is not None:
+                    # Crop the image using the bounding box for text recognition
+                    x1, y1, x2, y2 = map(int, box)
+                    crop_img = img_cv2[y1:y2, x1:x2]
+                    crop_pil = Image.fromarray(cv2.cvtColor(crop_img, cv2.COLOR_BGR2RGB))
+                    
+                    # Recognize text using PyLaia model
+                    predicted = predict('pylaia-samaritan_v1', crop_pil)
+                    texts.append(predicted[1]["text"])
+                    
+                    # Convert polygon to list of points for display
+                    poly_points = simplified_polygon.reshape(-1, 2).astype(int).tolist()
+                    polygons.append(f"Line {i+1}: {poly_points}")
+                    
+                    # Draw polygon on the image
+                    cv2.polylines(img_cv2, [simplified_polygon.reshape(-1, 1, 2).astype(int)], 
+                                 True, (0, 255, 0), 2)
+    
+    # Convert image back to RGB for display in Streamlit
+    img_result = cv2.cvtColor(img_cv2, cv2.COLOR_BGR2RGB)
+    
+    # Combine polygons and texts into a DataFrame for table display
+    table_data = pd.DataFrame({"Polygons": polygons, "Recognized Text": texts})
+    return Image.fromarray(img_result), table_data
+
+def segment_and_recognize(image):
+    segmented_image, table_data = process_image(image)
+    return segmented_image, table_data
+
+# Streamlit app layout
+st.title("YOLOv11 Text Line Segmentation & PyLaia Text Recognition on CATMuS/medieval")
+
+# File uploader
+uploaded_image = st.file_uploader("Upload an image", type=["png", "jpg", "jpeg"])
+
+# Process the image if uploaded
+if uploaded_image is not None:
+    image = Image.open(uploaded_image)
+
+    if st.button("Segment and Recognize"):
+        # Perform segmentation and recognition
+        segmented_image, table_data = segment_and_recognize(image)
+
+        # Display the segmented image
+        st.image(segmented_image, caption="Segmented Image with Polygon Masks", use_container_width=True)
+
+        # Display the table with polygons and recognized text
+        st.table(table_data)

catmus-medieval/language_model.arpa.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:754aca0dfb5b8c96b922e7ae96ad515a4bdd131760a8b59a3648b2c33d41329f
+size 21600345

catmus-medieval/language_model.binary

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d825caa68d8a1d7db10196c9de6591550fc04dec020ddb2c502786330d11c91
+size 50512864

catmus-medieval/lexicon.txt

@@ -0,0 +1,254 @@
+<ctc> <ctc>
+# #
+& &
+' '
+( (
+) )
+* *
++ +
+, ,
+- -
+. .
+/ /
+0 0
+1 1
+2 2
+3 3
+4 4
+5 5
+6 6
+7 7
+8 8
+9 9
+: :
+; ;
+= =
+? ?
+A A
+B B
+C C
+D D
+E E
+F F
+G G
+H H
+I I
+J J
+K K
+L L
+M M
+N N
+O O
+P P
+Q Q
+R R
+S S
+T T
+U U
+V V
+W W
+X X
+Y Y
+Z Z
+[ [
+] ]
+^ ^
+_ _
+a a
+b b
+c c
+d d
+e e
+f f
+g g
+h h
+i i
+j j
+k k
+l l
+m m
+n n
+o o
+p p
+q q
+r r
+s s
+t t
+u u
+v v
+w w
+x x
+y y
+z z
+| |
+~ ~
+¬ ¬
+° °
+¶ ¶
+Ø Ø
+Þ Þ
+ß ß
+æ æ
+ð ð
+÷ ÷
+þ þ
+đ đ
+ħ ħ
+ł ł
+ŧ ŧ
+ƀ ƀ
+ƿ ƿ
+Ƿ Ƿ
+ɂ ɂ
+ɨ ɨ
+ʰ ʰ
+ʳ ʳ
+ˡ ˡ
+ˢ ˢ
+ˣ ˣ
+̀ ̀
+́ ́
+̂ ̂
+̃ ̃
+̇ ̇
+̈ ̈
+̌ ̌
+̓ ̓
+̔ ̔
+̧ ̧
+̨ ̨
+̵ ̵
+̶ ̶
+̽ ̽
+̾ ̾
+͂ ͂
+ͣ ͣ
+ͤ ͤ
+ͥ ͥ
+ͦ ͦ
+ͧ ͧ
+ͨ ͨ
+ͩ ͩ
+ͪ ͪ
+ͫ ͫ
+ͬ ͬ
+ͭ ͭ
+ͮ ͮ
+ͯ ͯ
+Α Α
+Β Β
+Γ Γ
+Δ Δ
+Ε Ε
+Ζ Ζ
+Η Η
+Θ Θ
+Ι Ι
+Κ Κ
+Λ Λ
+Μ Μ
+Ν Ν
+Ξ Ξ
+Ο Ο
+Π Π
+Ρ Ρ
+Σ Σ
+Τ Τ
+Υ Υ
+Φ Φ
+Χ Χ
+Ψ Ψ
+Ω Ω
+α α
+β β
+γ γ
+δ δ
+ε ε
+η η
+θ θ
+ι ι
+κ κ
+λ λ
+μ μ
+ν ν
+ο ο
+π π
+ρ ρ
+ς ς
+σ σ
+τ τ
+υ υ
+χ χ
+ω ω
+ᛞ ᛞ
+ᵃ ᵃ
+ᵇ ᵇ
+ᵈ ᵈ
+ᵉ ᵉ
+ᵍ ᵍ
+ᵐ ᵐ
+ᵒ ᵒ
+ᵖ ᵖ
+ᵗ ᵗ
+ᵘ ᵘ
+ᶜ ᶜ
+ᶞ ᶞ
+ᶠ ᶠ
+ᶻ ᶻ
+᷑ ᷑
+᷒ ᷒
+ᷚ ᷚ
+ᷜ ᷜ
+ᷝ ᷝ
+ᷠ ᷠ
+ᷤ ᷤ
+ᷦ ᷦ
+ᷨ ᷨ
+ᷫ ᷫ
+ᷮ ᷮ
+ẜ ẜ
+ẞ ẞ
+† †
+‸ ‸
+⁊ ⁊
+⁋ ⁋
+⁜ ⁜
+⁰ ⁰
+ⁱ ⁱ
+⁴ ⁴
+⁷ ⁷
+⁹ ⁹
+ⁿ ⁿ
+℥ ℥
+♡ ♡
+❧ ❧
+⟦ ⟦
+⟧ ⟧
+Ꝑ Ꝑ
+ꝑ ꝑ
+ꝓ ꝓ
+ꝗ ꝗ
+Ꝙ Ꝙ
+ꝙ ꝙ
+Ꝟ Ꝟ
+ꝟ ꝟ
+ꝥ ꝥ
+ꝭ ꝭ
+Ꝯ Ꝯ
+ꝯ ꝯ
+ꝰ ꝰ
+ꝵ ꝵ
+ꝷ ꝷ
+Ꞧ Ꞧ
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+𐞥 𐞥
+<unk> <unk>
+<space> <space>

catmus-medieval/syms.txt

@@ -0,0 +1,254 @@
+<ctc> 0
+# 1
+& 2
+' 3
+( 4
+) 5
+* 6
++ 7
+, 8
+- 9
+. 10
+/ 11
+0 12
+1 13
+2 14
+3 15
+4 16
+5 17
+6 18
+7 19
+8 20
+9 21
+: 22
+; 23
+= 24
+? 25
+A 26
+B 27
+C 28
+D 29
+E 30
+F 31
+G 32
+H 33
+I 34
+J 35
+K 36
+L 37
+M 38
+N 39
+O 40
+P 41
+Q 42
+R 43
+S 44
+T 45
+U 46
+V 47
+W 48
+X 49
+Y 50
+Z 51
+[ 52
+] 53
+^ 54
+_ 55
+a 56
+b 57
+c 58
+d 59
+e 60
+f 61
+g 62
+h 63
+i 64
+j 65
+k 66
+l 67
+m 68
+n 69
+o 70
+p 71
+q 72
+r 73
+s 74
+t 75
+u 76
+v 77
+w 78
+x 79
+y 80
+z 81
+| 82
+~ 83
+¬ 84
+° 85
+¶ 86
+Ø 87
+Þ 88
+ß 89
+æ 90
+ð 91
+÷ 92
+þ 93
+đ 94
+ħ 95
+ł 96
+ŧ 97
+ƀ 98
+ƿ 99
+Ƿ 100
+ɂ 101
+ɨ 102
+ʰ 103
+ʳ 104
+ˡ 105
+ˢ 106
+ˣ 107
+̀ 108
+́ 109
+̂ 110
+̃ 111
+̇ 112
+̈ 113
+̌ 114
+̓ 115
+̔ 116
+̧ 117
+̨ 118
+̵ 119
+̶ 120
+̽ 121
+̾ 122
+͂ 123
+ͣ 124
+ͤ 125
+ͥ 126
+ͦ 127
+ͧ 128
+ͨ 129
+ͩ 130
+ͪ 131
+ͫ 132
+ͬ 133
+ͭ 134
+ͮ 135
+ͯ 136
+Α 137
+Β 138
+Γ 139
+Δ 140
+Ε 141
+Ζ 142
+Η 143
+Θ 144
+Ι 145
+Κ 146
+Λ 147
+Μ 148
+Ν 149
+Ξ 150
+Ο 151
+Π 152
+Ρ 153
+Σ 154
+Τ 155
+Υ 156
+Φ 157
+Χ 158
+Ψ 159
+Ω 160
+α 161
+β 162
+γ 163
+δ 164
+ε 165
+η 166
+θ 167
+ι 168
+κ 169
+λ 170
+μ 171
+ν 172
+ο 173
+π 174
+ρ 175
+ς 176
+σ 177
+τ 178
+υ 179
+χ 180
+ω 181
+ᛞ 182
+ᵃ 183
+ᵇ 184
+ᵈ 185
+ᵉ 186
+ᵍ 187
+ᵐ 188
+ᵒ 189
+ᵖ 190
+ᵗ 191
+ᵘ 192
+ᶜ 193
+ᶞ 194
+ᶠ 195
+ᶻ 196
+᷑ 197
+᷒ 198
+ᷚ 199
+ᷜ 200
+ᷝ 201
+ᷠ 202
+ᷤ 203
+ᷦ 204
+ᷨ 205
+ᷫ 206
+ᷮ 207
+ẜ 208
+ẞ 209
+† 210
+‸ 211
+⁊ 212
+⁋ 213
+⁜ 214
+⁰ 215
+ⁱ 216
+⁴ 217
+⁷ 218
+⁹ 219
+ⁿ 220
+℥ 221
+♡ 222
+❧ 223
+⟦ 224
+⟧ 225
+Ꝑ 226
+ꝑ 227
+ꝓ 228
+ꝗ 229
+Ꝙ 230
+ꝙ 231
+Ꝟ 232
+ꝟ 233
+ꝥ 234
+ꝭ 235
+Ꝯ 236
+ꝯ 237
+ꝰ 238
+ꝵ 239
+ꝷ 240
+Ꞧ 241
+ 242
+ 243
+ 244
+ 245
+ 246
+ 247
+ 248
+ 249
+ 250
+𐞥 251
+<unk> 252
+<space> 253

catmus-medieval/tokens.txt

@@ -0,0 +1,254 @@
+<ctc>
+#
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+:
+;
+=
+?
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+[
+]
+^
+_
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+|
+~
+¬
+°
+¶
+Ø
+Þ
+ß
+æ
+ð
+÷
+þ
+đ
+ħ
+ł
+ŧ
+ƀ
+ƿ
+Ƿ
+ɂ
+ɨ
+ʰ
+ʳ
+ˡ
+ˢ
+ˣ
+̀
+́
+̂
+̃
+̇
+̈
+̌
+̓
+̔
+̧
+̨
+̵
+̶
+̽
+̾
+͂
+ͣ
+ͤ
+ͥ
+ͦ
+ͧ
+ͨ
+ͩ
+ͪ
+ͫ
+ͬ
+ͭ
+ͮ
+ͯ
+Α
+Β
+Γ
+Δ
+Ε
+Ζ
+Η
+Θ
+Ι
+Κ
+Λ
+Μ
+Ν
+Ξ
+Ο
+Π
+Ρ
+Σ
+Τ
+Υ
+Φ
+Χ
+Ψ
+Ω
+α
+β
+γ
+δ
+ε
+η
+θ
+ι
+κ
+λ
+μ
+ν
+ο
+π
+ρ
+ς
+σ
+τ
+υ
+χ
+ω
+ᛞ
+ᵃ
+ᵇ
+ᵈ
+ᵉ
+ᵍ
+ᵐ
+ᵒ
+ᵖ
+ᵗ
+ᵘ
+ᶜ
+ᶞ
+ᶠ
+ᶻ
+᷑
+᷒
+ᷚ
+ᷜ
+ᷝ
+ᷠ
+ᷤ
+ᷦ
+ᷨ
+ᷫ
+ᷮ
+ẜ
+ẞ
+†
+‸
+⁊
+⁋
+⁜
+⁰
+ⁱ
+⁴
+⁷
+⁹
+ⁿ
+℥
+♡
+❧
+⟦
+⟧
+Ꝑ
+ꝑ
+ꝓ
+ꝗ
+Ꝙ
+ꝙ
+Ꝟ
+ꝟ
+ꝥ
+ꝭ
+Ꝯ
+ꝯ
+ꝰ
+ꝵ
+ꝷ
+Ꞧ
+
+
+
+
+
+
+
+
+
+𐞥
+<unk>
+<space>

catmus-medieval/weights.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d86b655a3f4095026a5a8c73241a308935dc22ad81e6b3bef4fe5765ace6a400
+size 43382236

model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25996a37e4842b7203abe0f5c969c514aafacbb9864023cb5f057e79e282fe31
+size 45140975

requirements.txt

@@ -0,0 +1,22 @@
+imagesize==1.4.1
+jsonargparse[signatures]==4.7
+matplotlib==3.9.2
+natsort==8.4.0
+pytorch-lightning==1.4.2
+scipy==1.11.3
+textdistance==4.6.3
+torch>=1.13,<1.14
+torchaudio>=0.13,<0.14
+torchmetrics<0.8.0
+torchvision>=0.14,<0.15
+mdutils==1.6.0
+prettytable==3.11.0
+python-bidi==0.6.0
+pylaia==1.1.2
+numpy==1.26.4
+opencv-python==4.11.0.86
+pillow==11.2.0
+python-bidi==0.6.0
+streamlit==1.44.0
+transformers==4.50.3
+ultralytics==8.3.99

runtime.txt

@@ -0,0 +1 @@
+python-3.10