Ajouter la prise en charge des fichiers PDF et PyPDF2

.gitignore

@@ -0,0 +1 @@
+__pycache__

app.py

@@ -4,16 +4,21 @@ from fastapi import FastAPI, UploadFile
 from typing import Union 
 import json 
 import csv
-from modeles import bert, squeezebert, deberta, loadSqueeze
+from modeles import bert, squeezebert, deberta
 from uploadFile import file_to_text
 from typing import List
 from transformers import pipeline
 from pydantic import BaseModel
-
-
+from typing import Optional
 
 app = FastAPI()
 
+class Request(BaseModel):
+    context: str
+    question: str
+    model: Optional[str] = None
+    # files: Optional[List[UploadFile]] = None
+
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -22,53 +27,58 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-class SqueezeBERTRequest(BaseModel):
-    context: str
-    question: str
-
-class BERTRequest(BaseModel):
-    context: str
-    question: str
-
-class DeBERTaRequest(BaseModel):
-    context: str
-    question: str
-
-pipBert = pipeline('question-answering', model="ALOQAS/bert-large-uncased-finetuned-squad-v2", tokenizer="ALOQAS/bert-large-uncased-finetuned-squad-v2")
-pipDeberta = pipeline('question-answering', model="ALOQAS/deberta-large-finetuned-squad-v2", tokenizer="ALOQAS/deberta-large-finetuned-squad-v2")
-tokenizer, model = loadSqueeze()
+pipSqueezeBert = pipeline("question-answering", model="ALOQAS/squeezebert-uncased-finetuned-squad-v2")
+pipBert = pipeline('question-answering', model="ALOQAS/bert-large-uncased-finetuned-squad-v2")
+pipDeberta = pipeline('question-answering', model="ALOQAS/deberta-large-finetuned-squad-v2")
 
 @app.get("/")
 async def root():
     return {"message": "Hello World"}
 
+@app.post("/contextText/")
+async def create_upload_file(request: Request):
+    try:
+        if request.model == "squeezebert":
+            answer = squeezebert(request.context, request.question, pipSqueezeBert)
+        elif request.model == "bert":
+            answer = bert(request.context, request.question, pipBert)
+        elif request.model == "deberta":
+            answer = deberta(request.context, request.question, pipDeberta)
+        else:
+            raise HTTPException(status_code=400, detail="Model not found.")
+        return answer
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")
 
 @app.post("/uploadfile/")
-async def create_upload_file(files: List[UploadFile], question: str, model: str):
-    res = []
+async def create_upload_file(files: List[UploadFile] = File(...), question: str = Form(...), model: str = Form(...)):
+    res = ""
     for file in files:
-        fileToText = await file_to_text(file)
-
-        res.append({"model": model, "texte": question, "filename": file.filename, "file_to_text": fileToText})
-
-    return res
-
-
-@app.post("/contextText/")
-async def create_upload_file(context: str, texte: str, model: str):
-
-    return {"model": model, "texte": texte, "context": context}
-
-
-@app.post("/withoutFile/")
-async def create_upload_file(texte: str, model: str):
-
-    return {"model": model, "texte": texte}
+        try:
+            res += await file_to_text(file)
+        except Exception as e:
+            print(f"Failed to process file {file.filename}: {e}") 
+            continue 
+
+    if res == "":
+        raise HTTPException(status_code=400, detail="All files failed to process.")
+    
+    answer = None
+    if model == "squeezebert":
+        answer = squeezebert(res, question, pipSqueezeBert)
+    elif model == "bert":
+        answer = bert(res, question, pipBert)
+    elif model == "deberta":
+        answer = deberta(res, question, pipDeberta)
+    else:
+        raise HTTPException(status_code=400, detail="Model not found.")
+    
+    return answer
 
 @app.post("/squeezebert/")
-async def qasqueezebert(request: SqueezeBERTRequest):
+async def qasqueezebert(request: Request):
     try:
-        squeezebert_answer = squeezebert(request.context, request.question, model, tokenizer)
+        squeezebert_answer = squeezebert(request.context, request.question, pipSqueezeBert)
         if squeezebert_answer:
             return squeezebert_answer
         else:
@@ -77,7 +87,7 @@ async def qasqueezebert(request: SqueezeBERTRequest):
         raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")
 
 @app.post("/bert/")
-async def qabert(request: BERTRequest):
+async def qabert(request: Request):
     try:
         bert_answer = bert(request.context, request.question, pipBert)
         if bert_answer:
@@ -87,8 +97,8 @@ async def qabert(request: BERTRequest):
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")
 
-@app.post("/deberta-v2/")
-async def qadeberta(request: DeBERTaRequest):
+@app.post("/deberta/")
+async def qadeberta(request: Request):
     try:
         deberta_answer = deberta(request.context, request.question, pipDeberta)
         if deberta_answer:
@@ -97,18 +107,3 @@ async def qadeberta(request: DeBERTaRequest):
             raise HTTPException(status_code=404, detail="No answer found")
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")
-
-def extract_data(file: UploadFile) -> Union[str, dict, list]:
-    if file.filename.endswith(".txt"):
-        data = file.file.read()
-        return data.decode("utf-8")
-    elif file.filename.endswith(".csv"):
-        data = file.file.read().decode("utf-8")
-        rows = data.split("\n")
-        reader = csv.DictReader(rows)
-        return [dict(row) for row in reader]
-    elif file.filename.endswith(".json"):
-        data = file.file.read().decode("utf-8")
-        return json.loads(data)
-    else:
-        return "Invalid file format"

modeles.py

@@ -1,54 +1,5 @@
-from transformers import AutoTokenizer, AutoModelForQuestionAnswering
-import torch
-
-def loadSqueeze():
-    tokenizer = AutoTokenizer.from_pretrained("ALOQAS/squeezebert-uncased-finetuned-squad-v2")
-    model = AutoModelForQuestionAnswering.from_pretrained("ALOQAS/squeezebert-uncased-finetuned-squad-v2")
-    return tokenizer, model
-
-def squeezebert(context, question, model, tokenizer):
-    # Tokenize the input question-context pair
-    inputs = tokenizer.encode_plus(question, context, max_length=512, truncation=True, padding=True, return_tensors='pt') 
-
-    # Send inputs to the same device as your model
-    inputs = {k: v.to(model.device) for k, v in inputs.items()}
-
-    with torch.no_grad():
-        # Forward pass, get model outputs
-        outputs = model(**inputs)
-
-    # Extract the start and end positions of the answer in the tokens
-    answer_start_scores, answer_end_scores = outputs.start_logits, outputs.end_logits
-
-    # Calculate probabilities from logits
-    answer_start_prob = torch.softmax(answer_start_scores, dim=-1)
-    answer_end_prob = torch.softmax(answer_end_scores, dim=-1)
-
-    # Find the most likely start and end positions
-    answer_start_index = torch.argmax(answer_start_prob)  # Most likely start of answer
-    answer_end_index = torch.argmax(answer_end_prob) + 1  # Most likely end of answer; +1 for inclusive slicing
-
-    # Extract the highest probability scores
-    start_score = answer_start_prob.max().item()  # Highest probability of start
-    end_score = answer_end_prob.max().item()  # Highest probability of end
-
-    # Combine the scores into a singular score
-    combined_score = (start_score * end_score) ** 0.5  # Geometric mean of start and end scores
-
-    # Convert token indices to the actual answer text
-    answer_tokens = inputs['input_ids'][0, answer_start_index:answer_end_index]
-    answer = tokenizer.decode(answer_tokens, skip_special_tokens=True)
-
-    # Return the answer, its positions, and the combined score
-    return {
-        "answer": answer,
-        "start": answer_start_index.item(),
-        "end": answer_end_index.item(),
-        "score": combined_score
-    }
-
-
-
+def squeezebert(context, question, pip):
+    return pip(context=context, question=question)
 
 def bert(context, question, pip):
     return pip(context=context, question=question)

requirements.txt

@@ -13,4 +13,5 @@ chardet
 frontend
 typing
 torch
-pydantic
+pydantic
+PyPDF2

uploadFile.py

@@ -2,11 +2,11 @@ import chardet
 from fastapi import UploadFile, HTTPException
 from io import BytesIO
 from docx import Document
-import fitz
+import PyPDF2
 
 async def file_to_text(file: UploadFile):
     file_extension = file.filename.split('.')[-1].lower()
-
+    # Fait 
     if file_extension == 'csv':
             csv_data = await file.read()
             encoding = chardet.detect(csv_data)['encoding']
@@ -16,12 +16,12 @@ async def file_to_text(file: UploadFile):
             except UnicodeDecodeError:
                 raise HTTPException(status_code=400, detail="Le fichier CSV contient des caractères qui ne peuvent pas être décodés.")
 
-    # Fait
+    # Fait 
     elif file_extension == 'json':
         json_data = await file.read()
         return json_data.decode()
 
-    # Fait
+    # Fait 
     elif file_extension == 'docx':
         doc_data = await file.read()
         # Utilisez un flux mémoire pour passer les données au Document
@@ -29,23 +29,21 @@ async def file_to_text(file: UploadFile):
         doc = Document(doc_stream)
         doc_text = [paragraph.text for paragraph in doc.paragraphs]
         return '\n'.join(doc_text)
-
-    # Fait
+    
+    # Fait 
     elif file_extension == 'txt':
         txt_data = await file.read()
         return txt_data.decode()
-
+    
     # Fait 
     elif file_extension == 'pdf':
         try:
             pdf_data = await file.read()
             # Chargez les données binaires dans un objet fitz.Document
-            pdf_document = fitz.open("pdf", pdf_data)
-            text = ''
-            # Extrait le texte de chaque page
-            for page in pdf_document:
-                text += page.get_text()
-            pdf_document.close()
+            pdf_document = PyPDF2.PdfReader(BytesIO(pdf_data))
+            text = ""
+            for page_number in range(len(pdf_document.pages)):
+                text += pdf_document.pages[page_number].extract_text()
             return text
         except Exception as e:
             raise HTTPException(status_code=500, detail=f"Erreur de lecture du fichier PDF : {e}")