Allow listing other document types than xxxxCRs

api/docs.py

@@ -8,6 +8,7 @@ import traceback
 import zipfile
 import json
 import os
+from httpx import AsyncClient
 from pydantic import BaseModel
 import requests
 import subprocess
@@ -23,7 +24,7 @@ from dependencies import DOC_FINDER_BASE_URL, get_http_client, get_llm_router
 from fastapi.responses import StreamingResponse
 from litellm.router import Router
 
-from schemas import DataRequest, DataResponse, DocRequirements, DownloadRequest, MeetingsRequest, MeetingsResponse, RequirementsRequest, RequirementsResponse
+from schemas import DataRequest, DataResponse, DocRequirements, DocDownloadRequest, MeetingsRequest, MeetingsResponse, ExtractRequirementsRequest, ExtractRequirementsResponse
 
 # API router for requirement extraction from docs / doc list retrieval / download
 router = APIRouter(tags=["document extraction"])
@@ -173,22 +174,26 @@ def docx_to_txt(doc_id: str, url: str):
 # ============================================= Doc routes =========================================================
 
 @router.post("/get_meetings", response_model=MeetingsResponse)
-def get_meetings(req: MeetingsRequest):
+async def get_meetings(req: MeetingsRequest, http_client: AsyncClient = Depends(get_http_client)):
+    # Extracting WG
     working_group = req.working_group
     tsg = re.sub(r"\d+", "", working_group)
     wg_number = re.search(r"\d", working_group).group(0)
 
+    # building corresponding FTP url
     logging.debug(tsg, wg_number)
     url = "https://www.3gpp.org/ftp/tsg_" + tsg
     logging.debug(url)
 
-    resp = requests.get(url, verify=False)
-    soup = BeautifulSoup(resp.text, "html.parser")
+    ftp_request = await http_client.get(url)
+    soup = BeautifulSoup(ftp_request.text, "html.parser")
 
     meeting_folders = []
     all_meetings = []
     wg_folders = [item.get_text() for item in soup.select("tr td a")]
     selected_folder = None
+
+    # sanity check to ensure the requested workgroup is present in the ftp directories
     for folder in wg_folders:
         if "wg" + str(wg_number) in folder.lower():
             selected_folder = folder
@@ -198,7 +203,7 @@ def get_meetings(req: MeetingsRequest):
     logging.debug(url)
 
     if selected_folder:
-        resp = requests.get(url, verify=False)
+        resp = await http_client.get(url)
         soup = BeautifulSoup(resp.text, "html.parser")
         meeting_folders = [item.get_text() for item in soup.select("tr td a") if item.get_text(
         ).startswith("TSG") or (item.get_text().startswith("CT") and "-" in item.get_text())]
@@ -211,14 +216,19 @@ def get_meetings(req: MeetingsRequest):
 
 
 @router.post("/get_dataframe", response_model=DataResponse)
-def get_change_request_dataframe(req: DataRequest):
+async def get_docs_df(req: DataRequest, http_client: AsyncClient = Depends(get_http_client)):
+    """
+    Downloads the document list dataframe for a given meeting 
+    """
+
+    # Extracting WG
     working_group = req.working_group
     tsg = re.sub(r"\d+", "", working_group)
     wg_number = re.search(r"\d", working_group).group(0)
     url = "https://www.3gpp.org/ftp/tsg_" + tsg
     logging.info("Fetching TDocs dataframe")
 
-    resp = requests.get(url, verify=False)
+    resp = await http_client.get(url)
     soup = BeautifulSoup(resp.text, "html.parser")
     wg_folders = [item.get_text() for item in soup.select("tr td a")]
     selected_folder = None
@@ -228,7 +238,7 @@ def get_change_request_dataframe(req: DataRequest):
             break
 
     url += "/" + selected_folder + "/" + req.meeting + "/docs"
-    resp = requests.get(url, verify=False)
+    resp = await http_client.get(url)
     soup = BeautifulSoup(resp.text, "html.parser")
     files = [item.get_text() for item in soup.select("tr td a")
              if item.get_text().endswith(".xlsx")]
@@ -240,7 +250,7 @@ def get_change_request_dataframe(req: DataRequest):
         return f"{url}/{tdoc}.zip"
 
     df = pd.read_excel(str(url + "/" + files[0]).replace("#", "%23"))
-    filtered_df = df[(((df["Type"] == "CR") & ((df["CR category"] == "B") | (df["CR category"] == "C"))) | (df["Type"] == "pCR")) & ~(
+    filtered_df = df[~(
         df["Uploaded"].isna())][["TDoc", "Title", "CR category", "Source", "Type", "Agenda item", "Agenda item description", "TDoc Status"]]
     filtered_df["URL"] = filtered_df["TDoc"].apply(gen_url)
 
@@ -251,11 +261,11 @@ def get_change_request_dataframe(req: DataRequest):
 
 
 @router.post("/download_tdocs")
-def download_tdocs(req: DownloadRequest):
+def download_tdocs(req: DocDownloadRequest):
     """Download the specified TDocs and zips them in a single archive"""
 
     # Document IDs to download
-    document_ids = req.documents
+    document_ids = [doc.document for doc in req.documents]
 
     logging.info(f"Downloading TDocs: {document_ids}")
 
@@ -279,7 +289,7 @@ def download_tdocs(req: DownloadRequest):
 
         raise HTTPException(
             status_code=501, detail="Got no URL results for docs {documents}. 3GPP index may not be up to date")
-    
+
     documents_content: Dict[str, bytes] = {}
     failed_documents: List[str] = []
 
@@ -330,66 +340,6 @@ def download_tdocs(req: DownloadRequest):
         headers={"Content-Disposition": "attachment; filename=tdocs.zip"}
     )
 
-
-@router.post("/generate_requirements", response_model=RequirementsResponse)
-async def gen_reqs(req: RequirementsRequest, background_tasks: BackgroundTasks, llm_router: Router = Depends(get_llm_router)):
-    """Extract requirements from the specified TDocs using a LLM"""
-
-    documents = req.documents
-    n_docs = len(documents)
-
-    logging.info("Generating requirements for documents: {}".format(
-        [doc.document for doc in documents]))
-
-    def prompt(doc_id, full):
-        return f"Here's the document whose ID is {doc_id} : {full}\n\nExtract all requirements and group them by context, returning a list of objects where each object includes a document ID, a concise description of the context where the requirements apply (not a chapter title or copied text), and a list of associated requirements; always return the result as a list, even if only one context is found. Remove the errors"
-
-    async def process_document(doc):
-        doc_id = doc.document
-        url = doc.url
-        try:
-            full = "\n".join(docx_to_txt(doc_id, url))
-        except Exception as e:
-            logging.error(f"Failed to process doc {doc_id}", e)
-            return RequirementsResponse(requirements=[DocRequirements(document=doc_id, context="Error LLM", requirements=[])]).requirements
-
-        try:
-            resp_ai = await llm_router.acompletion(
-                model="gemini-v2",
-                messages=[
-                    {"role": "user", "content": prompt(doc_id, full)}],
-                response_format=RequirementsResponse
-            )
-
-            return RequirementsResponse.model_validate_json(resp_ai.choices[0].message.content).requirements
-
-        except Exception as e:
-            logging.error(
-                f"Failed to process document {doc_id}", e, stack_info=True)
-            return RequirementsResponse(requirements=[DocRequirements(document=doc_id, context="Error LLM", requirements=[])]).requirements
-
-    async def process_batch(batch):
-        results = await asyncio.gather(*(process_document(doc) for doc in batch))
-        return [item for sublist in results for item in sublist]
-
-    all_requirements = []
-
-    if n_docs <= 30:
-        batch_results = await process_batch(documents)
-        all_requirements.extend(batch_results)
-    else:
-        batch_size = 30
-        batches = [documents[i:i + batch_size]
-                   for i in range(0, n_docs, batch_size)]
-
-        for i, batch in enumerate(batches):
-            batch_results = await process_batch(batch)
-            all_requirements.extend(batch_results)
-
-            if i < len(batches) - 1:
-                background_tasks.add_task(asyncio.sleep, 60)
-    return RequirementsResponse(requirements=all_requirements)
-
 # ======================================================================================================================================================================================
 
 
@@ -402,14 +352,13 @@ class ProgressUpdate(BaseModel):
 
 
 @router.post("/generate_requirements/sse")
-async def gen_reqs(req: RequirementsRequest, con: Request, llm_router: Router = Depends(get_llm_router)):
-    """Extract requirements from the specified TDocs using a LLM and returns SSE events about the progress of ongoing operations"""
+async def gen_reqs(req: ExtractRequirementsRequest, llm_router: Router = Depends(get_llm_router)):
+    """Extract requirements from the specified xxxxCR docs using a LLM and returns SSE events about the progress of ongoing operations"""
 
     documents = req.documents
     n_docs = len(documents)
 
-    logging.info("Generating requirements for documents: {}".format(
-        [doc.document for doc in documents]))
+    logging.info("Generating requirements for documents: {}".format(req.documents))
 
     # limit max concurrency of LLM requests to prevent a huge pile of errors because of small rate limits
     concurrency_sema = asyncio.Semaphore(4)
@@ -437,9 +386,9 @@ async def gen_reqs(req: RequirementsRequest, con: Request, llm_router: Router =
                 model=model_used,
                 messages=[
                     {"role": "user", "content": prompt(doc_id, full)}],
-                response_format=RequirementsResponse
+                response_format=ExtractRequirementsResponse
             )
-            return RequirementsResponse.model_validate_json(resp_ai.choices[0].message.content).requirements
+            return ExtractRequirementsResponse.model_validate_json(resp_ai.choices[0].message.content).requirements
         except Exception as e:
             return [DocRequirements(document=doc_id, context="Error LLM", requirements=[])]
         finally:
@@ -464,7 +413,7 @@ async def gen_reqs(req: RequirementsRequest, con: Request, llm_router: Router =
             n_processed += 1
             yield progress_update(ProgressUpdate(status="progress", data={}, total_docs=n_docs, processed_docs=n_processed))
 
-            final_response = RequirementsResponse(requirements=items)
+            final_response = ExtractRequirementsResponse(requirements=items)
 
         yield progress_update(ProgressUpdate(status="complete", data=final_response.model_dump(), total_docs=n_docs, processed_docs=n_processed))
 

dependencies.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from httpx import AsyncClient
 from litellm.router import Router
@@ -11,6 +12,7 @@ from jinja2 import Environment, StrictUndefined, FileSystemLoader
 INSIGHT_FINDER_BASE_URL = "https://organizedprogrammers-insight-finder.hf.space/"
 DOC_FINDER_BASE_URL = "https://organizedprogrammers-docfinder.hf.space/"
 
+
 def init_dependencies():
     """Initialize the application global dependencies"""
 
@@ -47,8 +49,9 @@ def init_dependencies():
     prompt_templates = Environment(loader=FileSystemLoader(
         "prompts"), enable_async=True, undefined=StrictUndefined)
 
-    http_client = AsyncClient(verify=os.environ.get(
-        "NO_SSL", "0") == "1", timeout=None)
+    enable_ssl = not os.environ.get("NO_SSL", "0") == "1"
+    logging.debug(f"SSL cert check is {enable_ssl}")
+    http_client = AsyncClient(verify=enable_ssl, timeout=None)
 
 
 def get_llm_router() -> Router:

schemas.py

@@ -23,11 +23,18 @@ class DataResponse(BaseModel):
 
 
 class DocInfo(BaseModel):
+    """
+        Schema for describing a document to download.
+    """
+    # Document name
     document: str
+    # Document URL
     url: str
+    # Document type
+    type: str
 
 
-class RequirementsRequest(BaseModel):
+class ExtractRequirementsRequest(BaseModel):
     documents: List[DocInfo]
 
 
@@ -37,7 +44,7 @@ class DocRequirements(BaseModel):
     requirements: List[str]
 
 
-class RequirementsResponse(BaseModel):
+class ExtractRequirementsResponse(BaseModel):
     requirements: List[DocRequirements]
 
 # --------------------------------------
@@ -66,9 +73,9 @@ class ReqSearchResponse(BaseModel):
 # --------------------------------------
 
 
-class DownloadRequest(BaseModel):
-    documents: List[str] = Field(
-        description="List of document IDs to download")
+class DocDownloadRequest(BaseModel):
+    documents: List[DocInfo] = Field(
+        description="List of documents to download")
 
 
 class ReqGroupingCategory(BaseModel):

static/index.html

@@ -156,21 +156,17 @@
             <!-- Data Table Informations -->
             <div class="flex justify-between items-center mb-2 pt-5" id="data-table-info-container">
                 <div class="flex gap-2 items-center">
-                    <div class="tooltip" data-tip="Extract requirements from selected documents">
+                    <div class="tooltip" data-tip="Extract requirements from selected pCR / CR documents">
                         <button id="extract-requirements-btn"
-                            class="bg-orange-300 text-white text-sm rounded px-3 py-1 shadow hover:bg-orange-600">
-                            <svg class="w-6 h-6 text-gray-800 dark:text-white" aria-hidden="true"
-                                xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none"
-                                viewBox="0 0 24 24">
-                                <path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"
-                                    stroke-width="2"
-                                    d="M9 8h6m-6 4h6m-6 4h6M6 3v18l2-2 2 2 2-2 2 2 2-2 2 2V3l-2 2-2-2-2 2-2-2-2 2-2-2Z" />
-                            </svg>Extract Requirements
+                            class="bg-orange-300 text-white text-sm rounded px-3 py-1 shadow hover:bg-orange-600">💉
+                            Extract Requirements
+                        </button>
+                    </div>
+                    <div class="tooltip" data-tip="Download all selected TDocs as text files">
+                        <button id="download-tdocs-btn" class="text-sm rounded px-3 py-1 shadow cursor-pointer">
+                            📦 Download Selected TDocs
                         </button>
                     </div>
-                    <button id="download-tdocs-btn" class="text-sm rounded px-3 py-1 shadow cursor-pointer">
-                        📦 Download Selected TDocs
-                    </button>
                 </div>
 
                 <!-- document counts -->

static/js/app.js

@@ -161,14 +161,6 @@ function setupFilters(data) {
     document.getElementById('status-filter-label').textContent = 'Status (Tous)';
     document.getElementById('agenda-filter-label').textContent = 'Agenda Item (Tous)';
 }
-/**
- * Configure les événements des filtres
- */
-function setupFilterEvents() {
-    ['doc-type-filter', 'doc-status-filter', 'agenda-item-filter'].forEach(filterId => {
-        document.getElementById(filterId).addEventListener('change', applyFilters);
-    });
-}
 
 function updateSelectedAndDisplayedCount() {
     // Lignes visibles (après filtrage)
@@ -233,22 +225,25 @@ function setupTableEvents() {
 }
 
 /**
- * Télécharge les TDocs sélectionnés
+ * Télécharge les pCR / CR / draftCR dans TDocs sélectionnés.
+ * Le JS filtre les tdocs sélectionnés pour recup uniquement les xxxxxCR
  */
 async function downloadTDocs() {
     showLoadingOverlay('Downloading TDocs...');
     toggleElementsEnabled(['download-tdocs-btn', 'extract-requirements-btn'], false);
 
     try {
-        // Extraire les données du tableau avec TDoc et URL
-        const selectedData = extractTableData({ 'TDoc': 'document', 'URL': 'url' });
+        // Extraire les données du tableau avec le format suivant pour la requete backend
+        // { document: "nom_doc", url: "url_doc", type: "type_de_doc"}
+        const selectedData = extractTableData({ 'TDoc': 'document', 'URL': 'url', 'Type': "type" });
+
         if (selectedData.length === 0) {
             alert('Please select at least one document');
             return;
         }
 
-        // Transformer au format requis: [{tdoc_id: url}, ...]
-        const documents = selectedData.map(obj => obj.document)
+        // on prend tout
+        const documents = selectedData;
 
         const response = await fetch('/docs/download_tdocs', {
             method: 'POST',
@@ -324,17 +319,24 @@ function downloadBlob(blob, filename) {
  * Extrait les requirements des documents sélectionnés
  */
 async function extractRequirements() {
-    const selectedData = extractTableData({ 'TDoc': 'document', 'URL': 'url' });
+    const selectedData = extractTableData({ 'TDoc': 'document', 'URL': 'url', 'Type': 'type' });
+
+    console.log("Selected docs data");
+    console.log(selectedData);
+
     if (selectedData.length === 0) {
         alert('Please select at least one document');
         return;
     }
 
+    // ne prendre que les documents avec un type qui contient xxxxCR en minuscules
+    const documents = selectedData.filter(d => d.type.toLowerCase().includes("cr"));
+
     showLoadingOverlay('Extracting requirements...');
     toggleElementsEnabled(['extract-requirements-btn'], false);
 
     try {
-        const response = await postWithSSE('/docs/generate_requirements/sse', { documents: selectedData }, {
+        const response = await postWithSSE('/docs/generate_requirements/sse', { documents: documents }, {
             onMessage: (msg) => {
                 console.log("SSE message:");
                 console.log(msg);