fix docker file and add nlp

Dockerfile

@@ -27,8 +27,8 @@ RUN ln -fs /usr/share/zoneinfo/Asia/Bangkok /etc/localtime && \
     dpkg-reconfigure -f noninteractive tzdata
 
 # สร้าง directory cache ต่าง ๆ
-RUN mkdir -p /tmp/hf_cache /tmp/torch_cache /tmp/matplotlib /tmp/xdg_cache /tmp/home /tmp/uploads \
-    && chmod -R 777 /tmp/hf_cache /tmp/torch_cache /tmp/matplotlib /tmp/xdg_cache /tmp/home /tmp/uploads
+RUN mkdir -p /tmp/hf_cache /tmp/torch_cache /tmp/matplotlib /tmp/xdg_cache /tmp/home /tmp/uploads /tmp/pythainlp_data \
+    && chmod -R 777 /tmp/hf_cache /tmp/torch_cache /tmp/matplotlib /tmp/xdg_cache /tmp/home /tmp/uploads /tmp/pythainlp_data
 
 # เพิ่ม PATH สำหรับ cuDNN 9 ให้เจอ .so
 ENV HUGGINGFACE_HUB_CACHE=/tmp/hf_cache \
@@ -42,7 +42,8 @@ ENV HUGGINGFACE_HUB_CACHE=/tmp/hf_cache \
     TMPDIR=/tmp \
     TEMP=/tmp \
     TMP=/tmp \
-    LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
+    LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH \
+    PYTHAINLP_DATA_DIR=/tmp/pythainlp_data
 
 # ติดตั้ง Python dependencies
 COPY requirements.txt .

requirements.txt

@@ -28,7 +28,7 @@ pandas==2.1.4
 numpy==1.24.4
 omegaconf==2.3.0
 pyyaml==6.0.1
-
+pythainlp==5.1.2
 # Utilities
 nest_asyncio==1.5.8
 python-dotenv==1.0.0

utils.py

@@ -12,6 +12,7 @@ from collections import Counter
 import time
 from config import UPLOAD_FOLDER
 from models import pipelines, models, together
+from pythainlp.spell import correct_sent
 
 def save_uploaded_file(file: UploadFile) -> str:
     os.makedirs(UPLOAD_FOLDER, exist_ok=True)
@@ -20,6 +21,11 @@ def save_uploaded_file(file: UploadFile) -> str:
         shutil.copyfileobj(file.file, f)
     return filepath
 
+def correct_text_with_tokenizer(text: str) -> str:
+    tokens = word_tokenize(text, engine="newmm")
+    corrected_tokens = [correct(word) for word in tokens]
+    return ''.join(corrected_tokens)
+
 def extract_and_normalize_audio(file_path: str) -> str:
     ext = os.path.splitext(file_path)[1].lower()
     audio_path = os.path.join(UPLOAD_FOLDER, "extracted_audio.wav")
@@ -81,25 +87,22 @@ def transcribe_segments(segment_folder: str) -> pd.DataFrame:
 
             if words:
                 full_text = ''.join([w.word for w in words])
+                nlp_corrected_text = correct_sent(full_text)
                 probs = [w.probability for w in words if w.probability is not None]
                 avg_prob = round(np.mean(probs), 4) if probs else 0.0
-                start_time = round(min(w.start for w in words if w.start is not None), 2)
-                end_time = round(max(w.end for w in words if w.end is not None), 2)
 
                 results.append({
                     "filename": filename,
                     "text": full_text,
+                    "nlp_corrected_text":nlp_corrected_text,
                     "avg_probability": avg_prob,
-                    "start": start_time,
-                    "end": end_time
                 })
             else:
                 results.append({
                     "filename": filename,
                     "text": "",
+                    "nlp_corrected_text":"",
                     "avg_probability": 0.0,
-                    "start": 0.0,
-                    "end": 0.0
                 })
 
         except Exception as e:
@@ -107,9 +110,8 @@ def transcribe_segments(segment_folder: str) -> pd.DataFrame:
             results.append({
                 "filename": filename,
                 "text": "",
+                "nlp_corrected_text":"",
                 "avg_probability": 0.0,
-                "start": 0.0,
-                "end": 0.0,
                 "error": str(e)
             })