finish backend

.gitignore

@@ -5,7 +5,7 @@ venv/
 .idea/
 *.log
 *.egg-info/
-pip-wheel-metadata/
+pip-wheel-EntityData/
 .env
 .DS_Store
 static/

requirements.txt

@@ -33,4 +33,4 @@ typing_extensions==4.12.2
 uvicorn==0.32.1
 uvloop==0.21.0
 watchfiles==1.0.0
-websockets==14.1
+websockets==14.1

test.py

@@ -0,0 +1,28 @@
+import pandas as pd
+from translate import Translator
+
+
+def convert_and_translate_headers(input_file: str, output_file: str, sheet_name: str = None):
+    try:
+        # Читаем файл Excel
+        data = pd.read_excel(input_file, sheet_name=sheet_name)
+
+        # Инициализируем переводчик
+        translator = Translator(from_lang='nl', to_lang='en')
+
+        # Переводим названия колонок
+        translated_columns = {col: translator.translate(col) for col in data.columns}
+        data.rename(columns=translated_columns, inplace=True)
+
+        # Сохраняем преобразованные данные в CSV
+        data.to_csv(output_file, index=False)
+        print(f"Файл успешно конвертирован и сохранен: {output_file}")
+    except Exception as e:
+        print(f"Произошла ошибка: {e}")
+
+
+input_xlsx = "test.xlsx"  # Путь к входному .xlsx файлу
+output_csv = "translated_output.csv"  # Путь к выходному .csv файлу
+sheet = "Sheet1"  # Укажите имя листа, если нужно
+
+convert_and_translate_headers(input_xlsx, output_csv, sheet)

trauma/__init__.py

@@ -12,17 +12,12 @@ from trauma.core.wrappers import TraumaResponseWrapper, ErrorTraumaResponse
 def create_app() -> FastAPI:
     app = FastAPI()
 
-    from trauma.api.account import account_router
-    app.include_router(account_router, tags=['account'])
-
     from trauma.api.chat import chat_router
     app.include_router(chat_router, tags=['chat'])
 
     from trauma.api.message import message_router
     app.include_router(message_router, tags=['message'])
 
-    from trauma.api.security import security_router
-    app.include_router(security_router, tags=['security'])
 
     app.add_middleware(
         CORSMiddleware,

trauma/api/account/__init__.py

@@ -1,7 +0,0 @@
-from fastapi import APIRouter
-
-account_router = APIRouter(
-    prefix='/api/account'
-)
-
-from . import views

trauma/api/account/dto.py

@@ -1,6 +0,0 @@
-from pydantic import BaseModel
-
-
-class AccessToken(BaseModel):
-    type: str = "Bearer"
-    value: str

trauma/api/account/model.py

@@ -1,32 +0,0 @@
-from datetime import datetime
-
-from pydantic import field_validator, Field, EmailStr
-from passlib.context import CryptContext
-
-from trauma.core.database import MongoBaseModel
-
-
-class AccountModel(MongoBaseModel):
-    email: EmailStr
-    password: str | None = Field(exclude=True, default=None)
-
-    datetimeInserted: datetime = Field(default_factory=datetime.now)
-    datetimeUpdated: datetime = Field(default_factory=datetime.now)
-
-    @field_validator("datetimeUpdated", mode="before", check_fields=False)
-    @classmethod
-    def validate_datetimeUpdated(cls, v):
-        return v or datetime.now()
-
-    @field_validator('password', mode='before', check_fields=False)
-    @classmethod
-    def set_password_hash(cls, v):
-        if not v.startswith("$2b$"):
-            return CryptContext(schemes=["bcrypt"], deprecated="auto").hash(v)
-        return v
-
-    class Config:
-        arbitrary_types_allowed = True
-        json_encoders = {
-            datetime: lambda dt: dt.isoformat()
-        }

trauma/api/account/schemas.py

@@ -1,6 +0,0 @@
-from trauma.api.account.model import AccountModel
-from trauma.core.wrappers import TraumaResponseWrapper
-
-
-class AccountWrapper(TraumaResponseWrapper[AccountModel]):
-    pass

trauma/api/account/views.py

@@ -1,13 +0,0 @@
-from fastapi import Depends
-
-from trauma.api.account import account_router
-from trauma.api.account.model import AccountModel
-from trauma.api.account.schemas import AccountWrapper
-from trauma.core.security import PermissionDependency
-
-
-@account_router.get('')
-async def get_account(
-        account: AccountModel = Depends(PermissionDependency())
-) -> AccountWrapper:
-    return AccountWrapper(data=account)

trauma/api/chat/db_requests.py

@@ -2,62 +2,65 @@ import asyncio
 
 from fastapi import HTTPException
 
-from trauma.api.account.model import AccountModel
 from trauma.api.chat.model import ChatModel
 from trauma.api.chat.schemas import CreateChatRequest, ChatTitleRequest
 from trauma.core.config import settings
+from trauma.core.wrappers import background_task
 
 
-async def get_chat_obj(chat_id: str, account: AccountModel | None) -> ChatModel:
+async def get_chat_obj(chat_id: str) -> ChatModel:
     chat = await settings.DB_CLIENT.chats.find_one({"id": chat_id})
     if not chat:
         raise HTTPException(status_code=404, detail="Chat not found")
     chat = ChatModel.from_mongo(chat)
-    if account and chat.account != account:
-        raise HTTPException(status_code=403, detail="Chat account not match")
     return chat
 
 
-async def create_chat_obj(chat_request: CreateChatRequest, account: AccountModel | None) -> ChatModel:
-    chat = ChatModel(model=chat_request.model, account=account)
+async def create_chat_obj(chat_request: CreateChatRequest) -> ChatModel:
+    chat = ChatModel(model=chat_request.model)
     await settings.DB_CLIENT.chats.insert_one(chat.to_mongo())
     return chat
 
 
-async def delete_chat_obj(chat_id: str, account: AccountModel | None) -> None:
+async def delete_chat_obj(chat_id: str) -> None:
     chat = await settings.DB_CLIENT.chats.find_one({"id": chat_id})
     if not chat:
         raise HTTPException(status_code=404, detail="Chat not found")
     chat = ChatModel.from_mongo(chat)
-    if account and chat.account != account:
-        raise HTTPException(status_code=403, detail="Chat account not match")
     await settings.DB_CLIENT.chats.delete_one({"id": chat_id})
 
 
-async def update_chat_obj_title(chatId: str, chat_request: ChatTitleRequest, account: AccountModel | None) -> ChatModel:
+async def update_chat_obj_title(chatId: str, chat_request: ChatTitleRequest) -> ChatModel:
     chat = await settings.DB_CLIENT.chats.find_one({"id": chatId})
     if not chat:
         raise HTTPException(status_code=404, detail="Chat not found")
 
     chat = ChatModel.from_mongo(chat)
-    if account and chat.account != account:
-        raise HTTPException(status_code=403, detail="Chat account not match")
 
     chat.title = chat_request.title
     await settings.DB_CLIENT.chats.update_one({"id": chatId}, {"$set": chat.to_mongo()})
     return chat
 
 
-async def get_all_chats_obj(page_size: int, page_index: int, account: AccountModel) -> tuple[list[ChatModel], int]:
-    query = {"account.id": account.id}
+async def get_all_chats_obj(page_size: int, page_index: int) -> tuple[list[ChatModel], int]:
     skip = page_size * page_index
     objects, total_count = await asyncio.gather(
         settings.DB_CLIENT.chats
-        .find(query)
+        .find({})
         .sort("_id", -1)
         .skip(skip)
         .limit(page_size)
         .to_list(length=page_size),
-        settings.DB_CLIENT.chats.count_documents(query),
+        settings.DB_CLIENT.chats.count_documents({}),
     )
     return objects, total_count
+
+
+@background_task()
+async def update_entity_data_obj(entity_data: dict, chat_id: str) -> None:
+    await settings.DB_CLIENT.chats.update_one(
+        {"id": chat_id},
+        {"$set": {
+            "entityData": entity_data,
+        }}
+    )

trauma/api/chat/dto.py

@@ -1,8 +1,16 @@
 from enum import Enum
 
+from pydantic import BaseModel
+
 
 class ModelType(Enum):
     gpt_4o = "gpt-4o"
     gpt_4o_mini = "gpt-4o-mini"
     o1_mini = "o1-mini"
     o1_preview = "o1-preview"
+
+class EntityData(BaseModel):
+    ageMin: int | None = None
+    ageMax: int | None = None
+    treatmentAreas: str | None = None
+    treatmentMethods: str | None = None

trauma/api/chat/model.py

@@ -2,14 +2,13 @@ from datetime import datetime
 
 from pydantic import Field
 
-from trauma.api.account.model import AccountModel
-from trauma.api.chat.dto import ModelType
+from trauma.api.chat.dto import ModelType, EntityData
 from trauma.core.database import MongoBaseModel
 
 
 class ChatModel(MongoBaseModel):
     title: str = 'New Chat'
     model: ModelType
-    account: AccountModel | None = None
+    entityData: EntityData | None = None
     datetimeInserted: datetime = Field(default_factory=datetime.now)
     datetimeUpdated: datetime = Field(default_factory=datetime.now)

trauma/api/chat/views.py

@@ -1,9 +1,7 @@
 from typing import Optional
 
 from fastapi import Query
-from fastapi.params import Depends
 
-from trauma.api.account.model import AccountModel
 from trauma.api.chat import chat_router
 from trauma.api.chat.db_requests import (get_chat_obj,
                                          create_chat_obj,
@@ -12,7 +10,6 @@ from trauma.api.chat.db_requests import (get_chat_obj,
                                          get_all_chats_obj)
 from trauma.api.chat.schemas import ChatWrapper, AllChatWrapper, CreateChatRequest, AllChatResponse, ChatTitleRequest
 from trauma.api.common.dto import Paging
-from trauma.core.security import PermissionDependency
 from trauma.core.wrappers import TraumaResponseWrapper
 
 
@@ -20,9 +17,8 @@ from trauma.core.wrappers import TraumaResponseWrapper
 async def get_all_chats(
         pageSize: Optional[int] = Query(10, description="Number of objects to return per page"),
         pageIndex: Optional[int] = Query(0, description="Page index to retrieve"),
-        account: AccountModel = Depends(PermissionDependency())
 ) -> AllChatWrapper:
-    chats, total_count = await get_all_chats_obj(pageSize, pageIndex, account)
+    chats, total_count = await get_all_chats_obj(pageSize, pageIndex)
     response = AllChatResponse(
         paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total_count),
         data=chats
@@ -32,29 +28,29 @@ async def get_all_chats(
 
 @chat_router.get('/{chatId}')
 async def get_chat(
-        chatId: str, account: AccountModel = Depends(PermissionDependency(is_public=True))
+        chatId: str
 ) -> ChatWrapper:
-    chat = await get_chat_obj(chatId, account)
+    chat = await get_chat_obj(chatId)
     return ChatWrapper(data=chat)
 
 
 @chat_router.post('')
 async def create_chat(
-        chat_data: CreateChatRequest, account: AccountModel = Depends(PermissionDependency(is_public=True))
+        chat_data: CreateChatRequest
 ) -> ChatWrapper:
-    chat = await create_chat_obj(chat_data, account)
+    chat = await create_chat_obj(chat_data)
     return ChatWrapper(data=chat)
 
 
 @chat_router.delete('/{chatId}')
-async def delete_chat(chatId: str, account: AccountModel = Depends(PermissionDependency())) -> TraumaResponseWrapper:
-    await delete_chat_obj(chatId, account)
+async def delete_chat(chatId: str) -> TraumaResponseWrapper:
+    await delete_chat_obj(chatId)
     return TraumaResponseWrapper()
 
 
 @chat_router.patch('/{chatId}/title')
 async def update_chat_title(
-        chatId: str, chat: ChatTitleRequest, account: AccountModel = Depends(PermissionDependency())
+        chatId: str, chat: ChatTitleRequest
 ) -> ChatWrapper:
-    chat = await update_chat_obj_title(chatId, chat, account)
+    chat = await update_chat_obj_title(chatId, chat)
     return ChatWrapper(data=chat)

trauma/api/data/dto.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel
+
+
+class AgeGroup(BaseModel):
+    ageMin: int
+    ageMax: int
+
+class ContactDetails(BaseModel):
+    phone: str | None = None
+    email: str | None = None
+    website: str | None = None
+    address: str | None = None
+    postalCode: str | None = None

trauma/api/data/model.py

@@ -0,0 +1,43 @@
+import datetime
+
+from trauma.api.data.dto import AgeGroup, ContactDetails
+from trauma.core.database import MongoBaseModel
+
+
+class DataModel(MongoBaseModel):
+    startDate: datetime.datetime
+    endDate: datetime.datetime
+    email: str | None = None
+    name: str | None = None
+    datetimeUpdated: datetime.datetime | None = None
+    organizationName: str | None = None
+    organizationLocation: str
+    youthCareRegion: str
+    postalCode: str
+    hasOrganizationMultipleLocations: bool
+    publicEmail: str
+    website: str
+    isTraumaTreatmentVisible: bool
+    injuryTreatmentForms: list[str]
+    offers: list[str]
+    intensities: list[str]
+    injuryTreatmentDuration: list[str]
+    ageGroups: list[AgeGroup]
+    traumaIndication: list[str]
+    supplier: str
+    treatmentFinancing: list[str]
+    isContractedCare: list[bool]
+    traumaTeamCharacteristics: list[str]
+    traumaDisciplines: list[str]
+    practitionersQualifications: list[str]
+    AQAs: list[str]
+    additions: list[str]
+
+
+class EntityModel(MongoBaseModel):
+    id: int
+    name: str
+    ageGroups: list[AgeGroup]
+    treatmentAreas: list[str]
+    treatmentMethods: list[str]
+    contactDetails: ContactDetails

trauma/api/data/prepare_data.py

@@ -0,0 +1,83 @@
+import asyncio
+
+import faiss
+import numpy as np
+
+from trauma.api.data.model import EntityModel
+from trauma.api.message.ai.openai_request import convert_value_to_embeddings
+# import re
+#
+# import pandas as pd
+#
+# from trauma.api.data.dto import AgeGroup
+# from trauma.api.data.model import EntityModel
+from trauma.core.config import settings
+
+
+#
+# file_path = 'shorted_data.csv'
+# df = pd.read_csv(file_path)
+#
+#
+# async def main():
+#     for _, row in df.iterrows():
+#         row = row.to_dict()
+#         age_groups_str = row['Age Groups'].split(';')
+#         age_groups = []
+#         for age_group in age_groups_str:
+#             match_ = re.search(r'(\d+)-(\d+)', age_group)
+#             if match_:
+#                 min_age, max_age = match_.groups()
+#                 age_groups.append(AgeGroup(ageMin=min_age, ageMax=max_age))
+#
+#         treatment_areas = row['Youths treatment'].split(';')
+#         treatment_areas = list(filter(lambda x: len(x) > 2, [i.strip().strip('\n').strip() for i in treatment_areas]))
+#
+#         treatment_methods = row['Offers'].split(';')
+#         treatment_methods = list(
+#             filter(lambda x: len(x) > 2, [i.strip().strip('\n').strip() for i in treatment_methods]))
+#
+#         email = re.search(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", row['Email'])
+#         if email:
+#             email = email.group(0)
+#         else:
+#             email = None
+#         website = row['Website']
+#         if website and not isinstance(website, float):
+#             website = re.sub(r'\n+', '', website)
+#             website = website.strip().strip('\n').strip()
+#             if not website.startswith(('https://', "http://")):
+#                 website = 'https://' + website.lower()
+#         else:
+#             website = None
+#         model_data = {
+#             "id": row['ID'],
+#             "name": row['Organization'].strip().strip('\n').strip(),
+#             "ageGroups": age_groups,
+#             "treatmentAreas": treatment_areas,
+#             "treatmentMethods": treatment_methods,
+#             "contactDetails": {
+#                 "phone": None,
+#                 "email": email,
+#                 "website": website,
+#                 "address": row['Location'].strip().strip('\n').strip(),
+#                 "postalCode": row['Postal code'].strip().strip('\n').strip(),
+#             }
+#         }
+#         entity_model = EntityModel.from_mongo(model_data)
+#         await settings.DB_CLIENT.entities.insert_one(entity_model.to_mongo())
+
+def prepare_entities_str(entities: list[EntityModel]) -> list[str]:
+    entities_str = []
+    for entity in entities:
+        age_groups_str = ', '.join([f"{age_group.ageMin} - {age_group.ageMax}" for age_group in entity.ageGroups])
+        treatment_areas_str = ', '.join(entity.treatmentAreas)
+        treatment_methods_str = ', '.join(entity.treatmentMethods)
+        entity_str = (f"Company name: {entity.name}. We focus on people in the following age groups: {age_groups_str}. "
+                      f"We specialize in treating the following disorder: {treatment_areas_str}."
+                      f" For this, we use the following treatment methods: {treatment_methods_str}.")
+        entities_str.append(entity_str)
+    return entities_str
+
+if __name__ == '__main__':
+    asyncio.run(add_index_to_documents())

trauma/api/message/ai/engine.py

@@ -0,0 +1,35 @@
+import asyncio
+
+from trauma.api.chat.db_requests import update_entity_data_obj
+from trauma.api.chat.model import ChatModel
+from trauma.api.message.ai.openai_request import update_entity_data_with_ai, generate_next_question, \
+    generate_search_request, generate_final_response
+from trauma.api.message.db_requests import save_assistant_user_message, filter_entities_by_age, search_semantic_entities
+from trauma.api.message.schemas import CreateMessageResponse
+from trauma.api.message.utils import retrieve_empty_field_from_entity_data, prepare_user_messages_str, \
+    prepare_final_entities_str
+
+
+async def search_entities(
+        user_message: str, messages: list[dict], chat: ChatModel
+) -> CreateMessageResponse:
+    entity_data = await update_entity_data_with_ai(chat.entityData, user_message)
+    asyncio.create_task(update_entity_data_obj(entity_data, chat.id))
+
+    empty_field = retrieve_empty_field_from_entity_data(entity_data)
+    final_entities = None
+
+    if empty_field:
+        response = await generate_next_question(empty_field, user_message, messages)
+    else:
+        user_messages_str = prepare_user_messages_str(user_message, messages)
+        possible_entity_indexes, search_request = await asyncio.gather(
+            filter_entities_by_age(entity_data),
+            generate_search_request(user_messages_str, entity_data)
+        )
+        final_entities = await search_semantic_entities(search_request, possible_entity_indexes)
+        final_entities_str = prepare_final_entities_str(final_entities)
+        response = await generate_final_response(final_entities_str, user_message, messages)
+
+    asyncio.create_task(save_assistant_user_message(user_message, response, chat.id))
+    return CreateMessageResponse(text=response, entities=final_entities)

trauma/api/message/ai/openai_request.py

@@ -1,47 +1,73 @@
-import base64
-import io
-
-from trauma.api.chat.model import ChatModel
-from trauma.api.message.ai.prompts import Prompts
-from trauma.api.message.dto import Author
-from trauma.api.message.model import MessageModel
+from trauma.api.chat.dto import EntityData
+from trauma.api.message.ai.prompts import TraumaPrompts
 from trauma.core.config import settings
-from trauma.core.wrappers import TraumaResponseWrapper
-
-
-async def prepare_content(user_message: MessageModel) -> list | str:
-    if user_message.fileUrl is None:
-        return user_message.text
-    else:
-        path = str(settings.BASE_DIR) + user_message.fileUrl.replace(settings.Issuer, '')
-        file = await settings.OPENAI_CLIENT.files.create(
-            file=open(path, 'rb'),
-            purpose='vision'
-        )
-        return [{"type": "image_file", "image_file": {"file_id": file.id, "detail": "low"}}]
-
-
-async def response_generator(chat: ChatModel, user_message: MessageModel):
-    content = await prepare_content(user_message)
-    await settings.OPENAI_CLIENT.beta.threads.messages.create(
-        thread_id=chat.threadId,
-        role=Author.User.value,
-        content=content
-    )
+from trauma.core.wrappers import openai_wrapper
+
+
+@openai_wrapper(is_json=True)
+async def update_entity_data_with_ai(entity_data: EntityData, user_message: str):
+    messages = [
+        {
+            "role": "system",
+            "content": TraumaPrompts.update_entity_data_with_ai
+            .replace("{entity_data}", entity_data.model_dump_json())
+            .replace("{user_message}", user_message)
+        }
+    ]
+    return messages
+
+
+@openai_wrapper(temperature=0.8)
+async def generate_next_question(empty_field: str, user_message: str, message_history: list[dict]):
+    messages = [
+        {
+            "role": "system",
+            "content": TraumaPrompts.generate_next_question
+            .replace("{empty_field}", empty_field)
+        },
+        *message_history,
+        {
+            "role": "user",
+            "content": user_message
+        }
+    ]
+    return messages
 
-    full_response = ''
-
-    async with settings.OPENAI_CLIENT.beta.threads.runs.create_and_stream(
-        thread_id=chat.threadId,
-        assistant_id=settings.ASSISTANT_ID,
-        instructions=Prompts.generate_response if not user_message.fileUrl else Prompts.generate_response_image,
-        model=chat.model.value
-    ) as stream:
-        async for chunk in stream.text_deltas:
-            if chunk:
-                full_response += chunk
-                mini_data = {"text": chunk}
-                yield f"data: {TraumaResponseWrapper(data=mini_data).model_dump_json()}\n\n"
-
-    message_obj = MessageModel(chatId=chat.id, author=Author.Assistant, text=full_response)
-    await settings.DB_CLIENT.messages.insert_one(message_obj.to_mongo())
+
+@openai_wrapper(temperature=0.4)
+async def generate_search_request(user_messages_str: str, entity_data: EntityData):
+    messages = [
+        {
+            "role": "system",
+            "content": TraumaPrompts.generate_search_request
+            .replace("{entity_data}", entity_data.model_dump_json())
+            .replace("{user_messages_str}", user_messages_str)
+        }
+    ]
+    return messages
+
+
+@openai_wrapper(temperature=0.4)
+async def generate_final_response(final_entities: str, user_message: str, message_history: list[dict]):
+    messages = [
+        {
+            "role": "system",
+            "content": TraumaPrompts.generate_recommendation_decision
+            .replace("{final_entities}", final_entities)
+
+        },
+        *message_history,
+        {
+            "role": "user",
+            "content": user_message
+        }
+    ]
+    return messages
+
+async def convert_value_to_embeddings(value: str) -> list[float]:
+    embeddings = await settings.OPENAI_CLIENT.embeddings.create(
+        input=value,
+        model='text-embedding-3-large',
+        dimensions=1536,
+    )
+    return embeddings.data[0].embedding

trauma/api/message/ai/prompts.py

@@ -1,216 +1,5 @@
-class Prompts:
-    generate_response = """## Objective 
-    
-You are Hector, the Mitutoyo virtual assistant, engineered for precision in the measurement industry. Your core function is two-fold: first, to match users with ideal products based on their specific requirements, and second, to intelligently recommend compatible accessories and complementary products for upselling opportunities.
-
-## Context
-
-Mitutoyo's B2B platform encompasses precision measurement tools and accessories. Your knowledge base consists of structured JSON product data specifically focused on Mitutoyo's digital and analogue micrometers, including their detailed specifications, features, and accessory relationships. While you have broad knowledge of Mitutoyo's full product range, your detailed product data and recommendation capabilities are currently optimized for the digital and analogue micrometer categories. Each recommendation must be based on exact matches between user requirements and product specifications.
-
-## Data Processing Protocol
-
-1. **Primary Product Matching**:
-   * Parse user requirements against product specifications
-   * Match technical requirements (e.g., "carbide tipped jaws") to FEATURE elements
-   * Validate matches using PRODUCT_DETAILS specifications
-   * Confirm accuracy through DESCRIPTION_LONG technical details
-2. **Product Data Extraction**:
-   * SUPPLIER_PID (for exact product identification)
-   * DESCRIPTION_SHORT (en/nl) for product names
-   * DESCRIPTION_LONG (en/nl) for technical details
-   * PRODUCT_DETAILS for specifications
-   * FEATURE elements for specific capabilities
-   * PRODUCT_ORDER_DETAILS for availability
-   * PRODUCT_PRICE_DETAILS for pricing
-3. **Accessory Matching Protocol**:
-   * Extract all PRODUCT_REFERENCE entries
-   * Validate PROD_ID_TO compatibility
-   * Cross-reference accessory specifications
-   * Verify physical compatibility parameters
-
-## Interaction Flow
-
-1. **Requirement Analysis**:
-   * Parse user's technical requirements
-   * Identify specific feature requests
-   * Match requirements to product specifications
-   * Validate technical compatibility
-2. **Primary Product Recommendation**:
-   * Present matching products with complete details:
-     * Product name (both languages)
-     * Article number
-     * Key specifications
-     * Relevant features
-     * Price information
-3. **Strategic Upselling**:
-   * Analyze PRODUCT_REFERENCE data
-   * Identify value-adding accessories
-   * Present complementary products
-   * Explain benefits and compatibility
-4. **Verification Process**:
-   * Double-check all technical matches
-   * Verify compatibility of all recommendations
-   * Confirm pricing and availability
-   * Validate all product relationships
-
-## Response Format Requirements
-
-```markdown
-## Primary Recommendation
-
-- Product: [Name EN/NL]
-- Article: [SUPPLIER_PID]
-- Key Features: [Matched Requirements]
-- Price: [From PRODUCT_PRICE_DETAILS]
-
-## Recommended Accessories
-
-1. [Primary Accessory]
-   - Purpose: [Specific Benefit]
-   - Article: [SUPPLIER_PID]
-   - Compatibility: [Verification Details]
-2. [Secondary Accessories]
-   - Purpose: [Specific Benefit]
-   - Article: [SUPPLIER_PID]
-   - Compatibility: [Verification Details]
-```
-
-## Error Prevention Protocol
-
-1. **Technical Matching**:
-   * Verify exact feature matches
-   * Confirm dimensional compatibility
-   * Validate technical specifications
-   * Cross-reference all requirements
-2. **Compatibility Verification**:
-   * Check PRODUCT_REFERENCE links
-   * Verify physical specifications
-   * Confirm accessory compatibility
-   * Validate system requirements
-3. **Data Accuracy**:
-   * Double-check all article numbers
-   * Verify price information
-   * Confirm availability status
-   * Validate technical specifications
-
-## Prohibitions
-
-* No assumptions about compatibility
-* No recommendations without PRODUCT_REFERENCE validation
-* No incomplete technical specifications
-* No unverified product relationships
-* NO discussion of competitor products or brands under any circumstances
-* NO comparative analysis with other manufacturers
-* NO recommendations outside of Mitutoyo's product range
-* NO redirecting to other brands, even if Mitutoyo doesn't offer a solution
-* NO market comparisons or industry benchmarking against other brands
-
-## Example Interaction
-
-User: "Need a micrometer with carbide tipped jaws"
-Response Protocol:
-1. Match "carbide tipped jaws" with FEATURE elements
-2. Verify products meeting specification
-3. Extract complete product details
-4. Identify compatible accessories through PRODUCT_REFERENCE
-5. Present primary recommendation with upselling options
-6. Verify all technical relationships
-
-## Key Performance Requirements
-
-* 100% accuracy in technical matching
-* Complete verification of all recommendations
-* Precise accessory compatibility checking
-* Clear, structured response format
-* Professional, technical communication style"""
-    generate_response_image = """### **Prompt Purpose**
-
-You are Hector, Mitutoyo's visual recognition expert, designed to identify Mitutoyo precision measurement instruments from images with unmatched precision and adherence to Mitutoyo’s high standards.
-
-### **Core Functionality**
-- **Primary Role:** Analyze images to confidently identify Mitutoyo products.
-- **Communication:** Clearly indicate the confidence level and provide actionable feedback for incomplete identification cases.
-
-### **Recognition Protocol**
-
-#### **Visual Analysis Sequence**
-
-1. Confirm Mitutoyo product authenticity.
-2. Identify the product category (e.g., micrometer, caliper, indicator).
-3. Recognize specific features and characteristics.
-4. Match visual data against known Mitutoyo design elements.
-5. Determine confidence level based on recognition certainty.
-
-#### **Confidence Level Classification**
-
-- **Level 1: High Confidence (100% Certain)**
-  - **Product Identification:** 
-    - Confirmed as Mitutoyo [Product Name].
-    - **Article Number:** [SUPPLIER_PID].
-  - **Product Details:**
-    - Include complete specifications based on available data.
-
-- **Level 2: Medium Confidence (Visual Recognition)**
-  - **Product Identification:** 
-    - Recognized as Mitutoyo [Product Category/Series].
-    - Requires additional training data to confirm the exact article number.
-  - **Identified Features:**
-    - List identifiable features and series characteristics.
-
-- **Level 3: Limited Confidence**
-  - **Initial Recognition:** 
-    - Confirmed as a Mitutoyo [Product Category].
-    - Requires additional information for precise identification.
-  - **Recommendations:**
-    1. Share the article number if available.
-    2. Provide details about specific requirements.
-    3. Include additional product visuals or specifications.
-
-### **Absolute Prohibitions**
-- **No guessing:** Avoid speculation on article numbers.
-- **Non-Mitutoyo products:** Do not identify or compare with competitor products.
-- **Uncertainty:** Avoid assumptions about specifications without solid evidence.
-- **Precision:** Only communicate confirmed and accurate information.
-
-### **Communication Guidelines**
-
-#### **Certainty Communication**
-
-- Clearly express confidence levels.
-- Explicitly state identification limitations.
-- Outline additional information needed for improved accuracy.
-
-#### **Training Transparency**
-
-- Acknowledge when training data is insufficient.
-- Professionally explain limitations.
-- Offer constructive steps for better identification.
-
-#### **Response Structure**
-
-1. Start with the confidence level.
-2. Provide available identification details.
-3. Highlight limitations and missing data.
-4. Suggest next steps or provide recommendations.
-
-#### **Unclear Cases Protocol**
-
-- Confirm receipt of the image.
-- State if the product is Mitutoyo.
-- Specify the confidence level.
-- Detail limitations and request additional details.
-- Propose alternative assistance options if applicable.
-
-### **Quality Assurance**
-
-- Provide article numbers only when 100% certain.
-- Validate all visual markers against Mitutoyo's known patterns.
-- Clearly communicate confidence levels.
-- Suggest actionable steps for uncertain cases.
-
-### **Key Performance Metrics**
-
-- **Accuracy:** Ensure precise product category identification.
-- **Clarity:** Effectively communicate certainty levels and next steps.
-- **Professionalism:** Handle limitations constructively.
-- **Adherence:** Uphold Mitutoyo’s precision standards at all times."""
+class TraumaPrompts:
+    update_entity_data_with_ai = """"""
+    generate_next_question = """"""
+    generate_search_request = """"""
+    generate_recommendation_decision = """"""

trauma/api/message/ai/utils.py

@@ -1,30 +0,0 @@
-from trauma.api.message.ai.prompts import Prompts
-from trauma.api.message.model import MessageModel
-
-
-def transform_messages_to_openai(messages: list[MessageModel]) -> list[dict]:
-    openai_messages = [{"role": "system", "content": Prompts.generate_response}]
-    for message in messages:
-
-        if message.file:
-            content = [
-                {
-                    "type": "text", "text": message.text
-                },
-                {
-                    "type": "image_url",
-                    "image_url": {
-                        "url": f"data:image/jpeg;base64,{message.file.base64String}",
-                        "detail": "low"
-                    }
-                },
-            ]
-        else:
-            content = message.text
-
-        openai_messages.append({
-            "role": message.author.value,
-            "content": content
-        })
-
-    return openai_messages

trauma/api/message/db_requests.py

@@ -1,18 +1,34 @@
+import faiss
+import numpy as np
 import asyncio
 
 from fastapi import HTTPException
 
-from trauma.api.account.model import AccountModel
+from trauma.api.chat.dto import EntityData
 from trauma.api.chat.model import ChatModel
+from trauma.api.data.model import EntityModel
+from trauma.api.message.ai.openai_request import convert_value_to_embeddings
 from trauma.api.message.dto import Author
 from trauma.api.message.model import MessageModel
 from trauma.api.message.schemas import CreateMessageRequest
 from trauma.core.config import settings
+from trauma.core.wrappers import background_task
 
 
-async def get_all_chat_messages_obj(
-        chat_id: str, account: AccountModel
-) -> list[MessageModel]:
+async def create_message_obj(
+        chat_id: str, message_data: CreateMessageRequest
+) -> tuple[MessageModel, ChatModel]:
+    chat = await settings.DB_CLIENT.chats.find_one({"id": chat_id})
+    if not chat:
+        raise HTTPException(status_code=404, detail="Chat not found.")
+
+    message = MessageModel(**message_data.model_dump(), chatId=chat_id, author=Author.User)
+    await settings.DB_CLIENT.messages.insert_one(message.to_mongo())
+
+    return message, chat
+
+
+async def get_all_chat_messages_obj(chat_id: str) -> tuple[list[MessageModel], ChatModel]:
     messages, chat = await asyncio.gather(
         settings.DB_CLIENT.messages.find({"chatId": chat_id}).to_list(length=None),
         settings.DB_CLIENT.chats.find_one({"id": chat_id})
@@ -23,23 +39,40 @@ async def get_all_chat_messages_obj(
         raise HTTPException(status_code=404, detail="Chat not found")
 
     chat = ChatModel.from_mongo(chat)
-    if account and chat.account != account:
-        raise HTTPException(status_code=403, detail="Chat account not match")
+    return messages, chat
 
-    return messages
 
-async def create_message_obj(
-        chat_id: str, message_data: CreateMessageRequest, account: AccountModel
-) -> tuple[MessageModel, ChatModel]:
-    chat = await settings.DB_CLIENT.chats.find_one({"id": chat_id})
-    if not chat:
-        raise HTTPException(status_code=404, detail="Chat not found")
+@background_task()
+async def save_assistant_user_message(user_message: str, assistant_message: str, chat_id: str) -> None:
+    user_message = MessageModel(chatId=chat_id, author=Author.User, text=user_message)
+    assistant_message = MessageModel(chatId=chat_id, author=Author.User, text=assistant_message)
+    await settings.DB_CLIENT.messages.insert_one(user_message.to_mongo())
+    await settings.DB_CLIENT.messages.insert_one(assistant_message.to_mongo())
 
-    chat = ChatModel.from_mongo(chat)
-    if account and chat.account != account:
-        raise HTTPException(status_code=403, detail="Chat account not match")
 
-    message = MessageModel(**message_data.model_dump(), chatId=chat_id, author=Author.User)
-    await settings.DB_CLIENT.messages.insert_one(message.to_mongo())
+async def filter_entities_by_age(entity: EntityData) -> list[int]:
+    query = {
+        "ageGroups": {
+            "$elemMatch": {
+                "ageMin": {"$lte": entity.ageMax},
+                "ageMax": {"$gte": entity.ageMin}
+            }
+        }
+    }
+    entities = await settings.DB_CLIENT.entities.find(query).to_list(length=None)
+    return [entity['index'] for entity in entities]
+
 
-    return message, chat
+async def search_semantic_entities(search_request: str, entities_indexes: list[int]):
+    embedding = await convert_value_to_embeddings(search_request)
+    query_embedding = np.array([embedding], dtype=np.float32)
+    distances, indices = settings.SEMANTIC_INDEX.search(query_embedding, k=settings.SEMANTIC_INDEX.ntotal)
+    distances = distances[0]
+    indices = indices[0]
+    filtered_results = [
+        {"index": int(idx), "distance": float(dist)}
+        for idx, dist in zip(indices, distances)
+        if idx in entities_indexes and dist <= 0.6
+    ]
+    filtered_results = sorted(filtered_results, key=lambda x: x["distance"])
+    return filtered_results[:5]

trauma/api/message/model.py

@@ -2,7 +2,7 @@ from datetime import datetime
 
 from pydantic import Field
 
-from trauma.api.message.dto import Author, File
+from trauma.api.message.dto import Author
 from trauma.core.database import MongoBaseModel
 
 
@@ -10,6 +10,5 @@ class MessageModel(MongoBaseModel):
     chatId: str
     author: Author
     text: str
-    fileUrl: str | None = None
     datetimeInserted: datetime = Field(default_factory=datetime.now)
     datetimeUpdated: datetime = Field(default_factory=datetime.now)

trauma/api/message/schemas.py

@@ -1,6 +1,7 @@
 from pydantic import BaseModel
 
 from trauma.api.common.dto import Paging
+from trauma.api.data.model import EntityModel
 from trauma.api.message.model import MessageModel
 from trauma.core.wrappers import TraumaResponseWrapper
 
@@ -21,3 +22,8 @@ class AllMessageResponse(BaseModel):
 
 class AllMessageWrapper(TraumaResponseWrapper[AllMessageResponse]):
     pass
+
+
+class CreateMessageResponse(BaseModel):
+    text: str
+    entities: list[EntityModel] | None = None

trauma/api/message/utils.py

@@ -0,0 +1,39 @@
+import json
+
+from trauma.api.data.model import EntityModel
+from trauma.api.message.model import MessageModel
+
+
+def transform_messages_to_openai(messages: list[MessageModel]) -> list[dict]:
+    openai_messages = []
+    for message in messages:
+        content = message.text
+        openai_messages.append({
+            "role": message.author.value,
+            "content": content
+        })
+
+    return openai_messages
+
+
+def retrieve_empty_field_from_entity_data(entity_data: dict) -> str | None:
+    for k, v in entity_data.items():
+        if not v:
+            return k
+    return None
+
+
+def prepare_user_messages_str(user_message: str, messages: list[dict]) -> str:
+    user_message_str = ''
+    for message in messages:
+        if message['role'] == 'user':
+            user_message_str += f'- {message['content']}\n'
+    user_message_str += f'- {user_message}'
+    return user_message_str
+
+
+def prepare_final_entities_str(entities: list[EntityModel]) -> str:
+    entities_list = []
+    for entity in entities:
+        entities_list.append(entity.model_dump(mode='json', exclude={'id', 'contactDetails'}))
+    return json.dumps({"entities": entities_list})

trauma/api/message/views.py

@@ -1,22 +1,20 @@
-from fastapi.params import Depends
-from starlette.responses import StreamingResponse
-
-from trauma.api.account.model import AccountModel
 from trauma.api.common.dto import Paging
 from trauma.api.message import message_router
-from trauma.api.message.ai.openai_request import response_generator
-from trauma.api.message.db_requests import get_all_chat_messages_obj, create_message_obj
+from trauma.api.message.ai.engine import search_entities
+from trauma.api.message.db_requests import get_all_chat_messages_obj
 from trauma.api.message.schemas import (AllMessageWrapper,
                                         AllMessageResponse,
-                                        CreateMessageRequest)
-from trauma.core.security import PermissionDependency
+                                        CreateMessageRequest,
+                                        CreateMessageResponse)
+from trauma.api.message.utils import transform_messages_to_openai
+from trauma.core.wrappers import TraumaResponseWrapper
 
 
 @message_router.get('/{chatId}/all')
 async def get_all_chat_messages(
-        chatId: str, account: AccountModel = Depends(PermissionDependency(is_public=True))
+        chatId: str
 ) -> AllMessageWrapper:
-    messages = await get_all_chat_messages_obj(chatId, account)
+    messages, _ = await get_all_chat_messages_obj(chatId)
     response = AllMessageResponse(
         paging=Paging(pageSize=len(messages), pageIndex=0, totalCount=len(messages)),
         data=messages
@@ -28,7 +26,8 @@ async def get_all_chat_messages(
 async def create_message(
         chatId: str,
         message_data: CreateMessageRequest,
-        account: AccountModel = Depends(PermissionDependency(is_public=True))
-) -> StreamingResponse:
-    user_message, chat = await create_message_obj(chatId, message_data, account)
-    return StreamingResponse(response_generator(chat, user_message), media_type='text/event-stream')
+) -> TraumaResponseWrapper[CreateMessageResponse]:
+    messages, chat = await get_all_chat_messages_obj(chatId)
+    message_history = transform_messages_to_openai(messages)
+    response = await search_entities(message_data.text, message_history, chat)
+    return TraumaResponseWrapper(data=response)

trauma/api/security/__init__.py

@@ -1,7 +0,0 @@
-from fastapi import APIRouter
-
-security_router = APIRouter(
-    prefix='/api/security'
-)
-
-from . import views

trauma/api/security/db_requests.py

@@ -1,34 +0,0 @@
-import asyncio
-
-from fastapi import HTTPException
-
-from trauma.api.account.model import AccountModel
-from trauma.api.common.db_requests import check_unique_fields_existence
-from trauma.api.security.schemas import RegisterAccountRequest, LoginAccountRequest
-from trauma.core.config import settings
-from trauma.core.security import verify_password
-
-
-async def save_account(data: RegisterAccountRequest) -> AccountModel:
-    await asyncio.gather(
-        check_unique_fields_existence("accounts", "email", data.email),
-    )
-    account = AccountModel(
-        **data.model_dump()
-    )
-    await settings.DB_CLIENT.accounts.insert_one(account.to_mongo())
-    return account
-
-
-async def authenticate_account(data: LoginAccountRequest) -> AccountModel:
-    account = await settings.DB_CLIENT.accounts.find_one(
-        {"email": data.email},
-        collation={"locale": "en", "strength": 2})
-    if account is None:
-        raise HTTPException(status_code=404, detail="Invalid email or password.")
-
-    account = AccountModel.from_mongo(account)
-
-    if not verify_password(data.password, account.password):
-        raise HTTPException(status_code=401, detail="Invalid email or password.")
-    return account

trauma/api/security/schemas.py

@@ -1,28 +0,0 @@
-from pydantic import BaseModel, EmailStr
-
-from trauma.api.account.dto import AccessToken
-from trauma.api.account.model import AccountModel
-from trauma.core.wrappers import TraumaResponseWrapper
-
-
-class RegisterAccountRequest(BaseModel):
-    email: EmailStr
-    password: str
-
-
-class RegisterAccountWrapper(TraumaResponseWrapper[AccountModel]):
-    pass
-
-
-class LoginAccountRequest(BaseModel):
-    email: EmailStr
-    password: str
-
-
-class LoginAccountResponse(BaseModel):
-    accessToken: AccessToken
-    account: AccountModel
-
-
-class LoginAccountWrapper(TraumaResponseWrapper[LoginAccountResponse]):
-    pass

trauma/api/security/views.py

@@ -1,27 +0,0 @@
-from typing import Literal
-
-from trauma.api.account.dto import AccessToken
-from trauma.api.security import security_router
-from trauma.api.security.db_requests import authenticate_account, save_account
-from trauma.api.security.schemas import RegisterAccountRequest, RegisterAccountWrapper, LoginAccountResponse, \
-    LoginAccountWrapper, LoginAccountRequest
-from trauma.core.security import create_access_token
-
-model: Literal["accounts"] = "accounts"
-
-
-@security_router.post('/register')
-async def register_user(data: RegisterAccountRequest) -> RegisterAccountWrapper:
-    account = await save_account(data)
-    return RegisterAccountWrapper(data=account)
-
-
-@security_router.post('/login')
-async def login(data: LoginAccountRequest) -> LoginAccountWrapper:
-    account = await authenticate_account(data)
-    access_token = create_access_token(account.email, str(account.id))
-    response = LoginAccountResponse(
-        accessToken=AccessToken(value=access_token),
-        account=account,
-    )
-    return LoginAccountWrapper(data=response)

trauma/core/config.py

@@ -2,6 +2,7 @@ import os
 import pathlib
 from functools import lru_cache
 
+import faiss
 import motor.motor_asyncio
 from dotenv import load_dotenv
 from openai import AsyncClient
@@ -14,7 +15,7 @@ class BaseConfig:
     SECRET_KEY = os.getenv('SECRET')
     DB_CLIENT = motor.motor_asyncio.AsyncIOMotorClient(os.getenv("MONGO_DB_URL")).trauma
     OPENAI_CLIENT = AsyncClient(api_key=os.getenv('OPENAI_API_KEY'))
-
+    SEMANTIC_INDEX = faiss.read_index(str(pathlib.Path(__file__).parent.parent.parent / 'indexes' / 'entities.index'))
 
 class DevelopmentConfig(BaseConfig):
     Issuer = "http://localhost:8000"

trauma/core/wrappers.py

@@ -1,6 +1,8 @@
+import json
 from functools import wraps
 from typing import Generic, Optional, TypeVar
 
+import pydash
 from fastapi import HTTPException
 from pydantic import BaseModel
 from starlette.responses import JSONResponse
@@ -42,3 +44,42 @@ def exception_wrapper(http_error: int, error_message: str):
         return wrapper
 
     return decorator
+
+
+def openai_wrapper(
+        temperature: int | float = 0, model: str = "gpt-4o-mini", is_json: bool = False, return_: str = None
+):
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs) -> str:
+            messages = await func(*args, **kwargs)
+            completion = await settings.OPENAI_CLIENT.chat.completions.create(
+                messages=messages,
+                temperature=temperature,
+                n=1,
+                model=model,
+                response_format={"type": "json_object"} if is_json else {"type": "text"}
+            )
+            response = completion.choices[0].message.content
+            if is_json:
+                response = json.loads(response)
+                if return_:
+                    return pydash.get(response, return_)
+            return response
+
+        return wrapper
+
+    return decorator
+
+def background_task():
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs) -> str:
+            try:
+                result = await func(*args, **kwargs)
+                return result
+            except Exception as e:
+                pass
+        return wrapper
+
+    return decorator