handle both ar and eng

Dockerfile

@@ -1,17 +1,28 @@
-FROM python:3.11-slim
+# Use Python 3.9 as the base image
+FROM python:3.9
 
+# Set working directory in the container
 WORKDIR /app
 
-COPY requirements.txt .
+# Create a non-root user and set permissions
+RUN useradd -m myuser && chown -R myuser:myuser /app
+USER myuser
+
+# Set Hugging Face cache directory
+ENV HF_HOME=/app/.cache/huggingface
+
+# Update PATH for uvicorn
+ENV PATH="/home/myuser/.local/bin:${PATH}"
+
+# Copy requirements.txt and install dependencies
+COPY --chown=myuser:myuser requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Create Hugging Face cache directory with permissions
-ENV HF_HOME=/app/.cache
-ENV TRANSFORMERS_CACHE=/app/.cache
-ENV HF_HUB_CACHE=/app/.cache
-RUN mkdir -p /app/.cache && chmod -R 777 /app/.cache
+# Clear cache and pre-download models
+RUN rm -rf /app/.cache/huggingface/* && python -c "from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM; pipeline('text-classification', model='Hello-SimpleAI/chatgpt-detector-roberta'); pipeline('text-classification', model='openai-community/roberta-large-openai-detector'); pipeline('text-classification', model='sabaridsnfuji/arabic-ai-text-detector'); AutoTokenizer.from_pretrained('gpt2'); AutoModelForCausalLM.from_pretrained('gpt2'); AutoTokenizer.from_pretrained('aubmindlab/araGPT2'); AutoModelForCausalLM.from_pretrained('aubmindlab/araGPT2')"
 
-COPY . .
+# Copy the application code
+COPY --chown=myuser:myuser . .
 
-EXPOSE 7860
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+# Run the FastAPI app with Uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -1,35 +1,82 @@
-from fastapi import FastAPI
-from pydantic import BaseModel
-from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForCausalLM
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, validator
+import re
 import torch
-import math
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForCausalLM, pipeline
+from collections import Counter
+import logging
+import numpy as np
 
-app = FastAPI(title="Improved AI Text Detector")
-
-# 1. Classifier model (better than akshayvkt)
-clf_model_name = "Hello-SimpleAI/chatgpt-detector-roberta"
-clf_tokenizer = AutoTokenizer.from_pretrained(clf_model_name)
-clf_model = AutoModelForSequenceClassification.from_pretrained(clf_model_name)
-
-# 2. Perplexity model (GPT-2)
-ppl_model_name = "gpt2"
-ppl_tokenizer = AutoTokenizer.from_pretrained(ppl_model_name)
-ppl_model = AutoModelForCausalLM.from_pretrained(ppl_model_name)
+# Configure logging with more detail
+logging.basicConfig(filename="predictions.log", level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s")
 
-class InputText(BaseModel):
-    text: str
+app = FastAPI(title="Improved AI Text Detector")
 
-def get_classifier_score(text: str) -> float:
-    inputs = clf_tokenizer(text, return_tensors="pt", truncation=True, max_length=512)
-    with torch.no_grad():
-        outputs = clf_model(**inputs)
-        probs = torch.softmax(outputs.logits, dim=-1)
-        ai_prob = probs[0][1].item()  # label 1 = AI
-    return ai_prob
-
-def get_perplexity(text: str) -> float:
-    encodings = ppl_tokenizer(text, return_tensors="pt")
-    max_length = ppl_model.config.n_positions
+# Enable GPU if available, else use CPU
+device = 0 if torch.cuda.is_available() else -1
+torch.manual_seed(42)
+
+# Load classifier models
+english_detectors = [
+    pipeline("text-classification", model="Hello-SimpleAI/chatgpt-detector-roberta", device=device, truncation=True, max_length=512),
+    pipeline("text-classification", model="openai-community/roberta-large-openai-detector", device=device, truncation=True, max_length=512)
+]
+arabic_detector = pipeline("text-classification", model="sabaridsnfuji/arabic-ai-text-detector", device=device, truncation=True, max_length=512)
+
+# Load perplexity models
+ppl_english = {
+    "tokenizer": AutoTokenizer.from_pretrained("gpt2"),
+    "model": AutoModelForCausalLM.from_pretrained("gpt2").to(device)
+}
+ppl_arabic = {
+    "tokenizer": AutoTokenizer.from_pretrained("aubmindlab/araGPT2"),
+    "model": AutoModelForCausalLM.from_pretrained("aubmindlab/araGPT2").to(device)
+}
+
+def detect_language(text: str) -> str:
+    """Detect if text is Arabic or English based on Unicode character ranges."""
+    arabic_chars = len(re.findall(r'[\u0600-\u06FF]', text))
+    latin_chars = len(re.findall(r'[A-Za-z]', text))
+    total_chars = arabic_chars + latin_chars
+    if total_chars == 0:
+        return 'en'
+    arabic_ratio = arabic_chars / total_chars
+    return 'ar' if arabic_ratio > 0.5 else 'en'
+
+def calculate_burstiness(text: str) -> float:
+    """Calculate burstiness (std/mean of sentence lengths) to bias toward human text."""
+    sentences = re.split(r'[.!?]', text)
+    lengths = [len(s.split()) for s in sentences if s]
+    return np.std(lengths) / (np.mean(lengths) + 1e-6) if lengths else 0
+
+def calculate_ttr(text: str) -> float:
+    """Calculate type-token ratio (lexical diversity) to bias toward human text."""
+    words = text.split()
+    if not words:
+        return 0
+    unique_words = len(set(words))
+    total_words = len(words)
+    return unique_words / total_words
+
+def clean_text(text: str, language: str) -> str:
+    """Clean text by removing special characters and normalizing spaces. Skip lowercase for Arabic."""
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'[^\w\s.,!?]', '', text)
+    text = text.strip()
+    if language == 'en':
+        text = text.lower()
+    return text
+
+def get_classifier_score(text: str, detector) -> float:
+    """Get classifier probability for AI label."""
+    result = detector(text, truncation=True, max_length=512)[0]
+    score = result['score']
+    return score if result['label'] in ['AI', 'Fake'] else 1 - score
+
+def get_perplexity(text: str, tokenizer, model) -> float:
+    """Calculate perplexity using a language model."""
+    encodings = tokenizer(text, return_tensors="pt", truncation=True, max_length=512).to(device)
+    max_length = model.config.n_positions
     stride = 512
     seq_len = encodings.input_ids.size(1)
 
@@ -38,12 +85,12 @@ def get_perplexity(text: str) -> float:
     for begin_loc in range(0, seq_len, stride):
         end_loc = min(begin_loc + stride, seq_len)
         trg_len = end_loc - prev_end_loc
-        input_ids = encodings.input_ids[:, begin_loc:end_loc]
+        input_ids = encodings.input_ids[:, begin_loc:end_loc].to(device)
         target_ids = input_ids.clone()
         target_ids[:, :-trg_len] = -100
 
         with torch.no_grad():
-            outputs = ppl_model(input_ids, labels=target_ids)
+            outputs = model(input_ids, labels=target_ids)
             neg_log_likelihood = outputs.loss * trg_len
 
         nlls.append(neg_log_likelihood)
@@ -52,30 +99,138 @@ def get_perplexity(text: str) -> float:
         if end_loc == seq_len:
             break
 
-    ppl = torch.exp(torch.stack(nlls).sum() / end_loc)
-    return ppl.item()
-
-@app.post("/detect")
-def detect(input_text: InputText):
-    text = input_text.text.strip()
-
-    # Run classifier
-    clf_score = get_classifier_score(text)
+    ppl = torch.exp(torch.stack(nlls).sum() / end_loc if nlls else torch.tensor(0)).item()
+    return ppl
+
+def split_text(text: str, max_chars: int = 5000) -> list:
+    """Split text into chunks of max_chars, preserving sentence boundaries."""
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+    chunks = []
+    current_chunk = ""
+    for sentence in sentences:
+        if len(current_chunk) + len(sentence) <= max_chars:
+            current_chunk += sentence + " "
+        else:
+            if current_chunk:
+                chunks.append(current_chunk.strip())
+            current_chunk = sentence + " "
+    if current_chunk:
+        chunks.append(current_chunk.strip())
+    return chunks
+
+class TextInput(BaseModel):
+    text: str
 
-    # Run perplexity
-    ppl = get_perplexity(text)
+    @validator("text")
+    def validate_text(cls, value):
+        """Validate input text for minimum length and content."""
+        word_count = len(value.split())
+        if word_count < 50:
+            raise ValueError(f"Text too short ({word_count} words). Minimum 50 words required.")
+        if not re.search(r'[\w]', value):
+            raise ValueError("Text must contain alphabetic characters.")
+        return value
 
-    # Decision rule: combine both
-    # Lower perplexity (<50) + high classifier_score (>0.7) = AI
-    if clf_score > 0.6 and ppl < 70:
-        final = "AI"
-    elif clf_score < 0.4 and ppl > 60:
-        final = "Human"
+@app.post("/detect")
+def detect(input_text: TextInput):
+    detected_lang = detect_language(input_text.text)
+    note_lang = f"Detected language: {'Arabic' if detected_lang == 'ar' else 'English'}"
+    cleaned_text = clean_text(input_text.text, detected_lang)
+    burstiness = calculate_burstiness(cleaned_text)
+    ttr = calculate_ttr(cleaned_text)
+    note_features = f"Burstiness: {burstiness:.2f} (high suggests human), TTR: {ttr:.2f} (low suggests human)"
+
+    # Select appropriate models
+    detectors = english_detectors if detected_lang == 'en' else [arabic_detector]
+    ppl_model = ppl_english if detected_lang == 'en' else ppl_arabic
+    is_ensemble = detected_lang == 'en'
+
+    if len(cleaned_text) > 10000:
+        chunks = split_text(cleaned_text, max_chars=5000)
+        labels = []
+        clf_scores = []
+        ppls = []
+
+        for chunk_idx, chunk in enumerate(chunks):
+            chunk_labels = []
+            chunk_clf_scores = []
+            for det_idx, detector in enumerate(detectors):
+                clf_score = get_classifier_score(chunk, detector)
+                label = "AI" if clf_score >= 0.99 else "Human" if clf_score < 0.60 else "Uncertain"
+                chunk_labels.append(label)
+                chunk_clf_scores.append(clf_score)
+                logging.debug(f"Chunk {chunk_idx}, Model {det_idx}: Label={label}, Classifier Score={clf_score:.4f}")
+            ppl = get_perplexity(chunk, ppl_model["tokenizer"], ppl_model["model"])
+            chunk_final_label = Counter(chunk_labels).most_common(1)[0][0]
+            avg_clf_score = np.mean(chunk_clf_scores)
+
+            # Combine classifier, perplexity, burstiness, and TTR
+            if chunk_final_label == "Uncertain" or len(set(chunk_labels)) == len(detectors) or any(l == "Human" for l in chunk_labels):
+                if ppl > 60 or burstiness > 1.2 or ttr < 0.12:
+                    chunk_final_label = "Human"
+            elif chunk_final_label == "AI" and (ppl > 60 or burstiness > 1.2 or ttr < 0.12):
+                chunk_final_label = "Human"
+            labels.append(chunk_final_label)
+            clf_scores.append(avg_clf_score)
+            ppls.append(ppl)
+            logging.debug(f"Chunk {chunk_idx} Final: Label={chunk_final_label}, Avg Classifier Score={avg_clf_score:.4f}, Perplexity={ppl:.2f}, Burstiness={burstiness:.2f}, TTR={ttr:.2f}")
+
+        label_counts = Counter(labels)
+        final_label = label_counts.most_common(1)[0][0]
+        if final_label == "Uncertain" or len(set(labels)) == len(detectors) or any(l == "Human" for l in labels):
+            if any(ppl > 60 for ppl in ppls) or burstiness > 1.2 or ttr < 0.12:
+                final_label = "Human"
+        avg_clf_score = sum(clf_scores) / len(clf_scores) if clf_scores else 0.0
+        avg_ppl = sum(ppls) / len(ppls) if ppls else 0.0
+        logging.info(f"Language: {detected_lang} | Text Length: {len(cleaned_text)} | Chunks: {len(chunks)} | Prediction: {final_label} | Avg Classifier Score: {avg_clf_score:.4f} | Avg Perplexity: {avg_ppl:.2f} | {note_features}")
+        return {
+            "prediction": final_label,
+            "classifier_score": round(avg_clf_score, 4),
+            "perplexity": round(avg_ppl, 2),
+            "note": f"{note_lang}. Text was split into {len(chunks)} chunks due to length > 10,000 characters. {note_features}.",
+            "chunk_results": [
+                {"chunk": chunk[:50] + "...", "label": labels[i], "classifier_score": clf_scores[i], "perplexity": ppls[i], "burstiness": burstiness, "ttr": ttr}
+                for i, chunk in enumerate(chunks)
+            ]
+        }
     else:
-        final = "Uncertain"
-
-    return {
-        "classifier_score": round(clf_score, 4),
-        "perplexity": round(ppl, 2),
-        "final_label": final
-    }
+        if is_ensemble:
+            clf_scores = []
+            labels = []
+            for det_idx, detector in enumerate(detectors):
+                clf_score = get_classifier_score(cleaned_text, detector)
+                label = "AI" if clf_score >= 0.99 else "Human" if clf_score < 0.60 else "Uncertain"
+                labels.append(label)
+                clf_scores.append(clf_score)
+                logging.debug(f"Model {det_idx}: Label={label}, Classifier Score={clf_score:.4f}")
+            ppl = get_perplexity(cleaned_text, ppl_model["tokenizer"], ppl_model["model"])
+            label_counts = Counter(labels)
+            final_label = label_counts.most_common(1)[0][0]
+            if final_label == "Uncertain" or len(set(labels)) == len(detectors) or any(l == "Human" for l in labels):
+                if ppl > 60 or burstiness > 1.2 or ttr < 0.12:
+                    final_label = "Human"
+            elif final_label == "AI" and (ppl > 60 or burstiness > 1.2 or ttr < 0.12):
+                final_label = "Human"
+            avg_clf_score = sum(clf_scores) / len(clf_scores) if clf_scores else 0.0
+            note = f"{note_lang}. Ensemble used: {len(detectors)} models. {note_features}. Perplexity: {ppl:.2f}."
+            if 0.60 <= avg_clf_score < 0.99:
+                note += " Warning: Close to threshold, result may be uncertain."
+            logging.info(f"Language: {detected_lang} | Text Length: {len(cleaned_text)} | Prediction: {final_label} | Avg Classifier Score: {avg_clf_score:.4f} | Perplexity: {ppl:.2f} | Model Scores: {clf_scores} | {note_features}")
+        else:
+            clf_score = get_classifier_score(cleaned_text, arabic_detector)
+            ppl = get_perplexity(cleaned_text, ppl_model["tokenizer"], ppl_model["model"])
+            final_label = "AI" if clf_score >= 0.97 else "Human" if clf_score < 0.60 else "Uncertain"
+            if final_label == "Uncertain" or final_label == "Human":
+                if ppl > 60 or burstiness > 0.8 or ttr < 0.12:
+                    final_label = "Human"
+            avg_clf_score = clf_score
+            note = f"{note_lang}. {note_features}. Perplexity: {ppl:.2f}."
+            if 0.60 <= clf_score < 0.97:
+                note += " Warning: Close to threshold, result may be uncertain."
+            logging.info(f"Language: {detected_lang} | Text Length: {len(cleaned_text)} | Prediction: {final_label} | Classifier Score: {avg_clf_score:.4f} | Perplexity: {ppl:.2f} | {note_features}")
+        return {
+            "prediction": final_label,
+            "classifier_score": round(avg_clf_score, 4),
+            "perplexity": round(ppl, 2),
+            "note": note
+        }