initial commit

.gitignore

@@ -169,5 +169,11 @@ code/.chainlit/translations/
 storage/logs/*
 vectorstores/*
 
-*/.files/*
+**/.files/*
 code/storage/models/
+
+**/translations/en-US.json
+**/translations/zh-CN.json
+
+
+**/vectorstores/*

{code → apps/ai_tutor}/.chainlit/config.toml

@@ -20,10 +20,10 @@ allow_origins = ["*"]
 
 [features]
 # Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
-unsafe_allow_html = true
+unsafe_allow_html = false
 
 # Process and display mathematical expressions. This can clash with "$" characters in messages.
-latex = true
+latex = false
 
 # Automatically tag threads with the current chat profile (if a chat profile is used)
 auto_tag_thread = true
@@ -53,7 +53,7 @@ edit_message = true
 
 [UI]
 # Name of the assistant.
-name = "AI Tutor"
+name = "Assistant"
 
 # Description of the assistant. This is used for HTML tags.
 # description = ""
@@ -62,14 +62,14 @@ name = "AI Tutor"
 default_collapse_content = true
 
 # Chain of Thought (CoT) display mode. Can be "hidden", "tool_call" or "full".
-cot = "hidden"
+cot = "full"
 
 # Link to your github repo. This will add a github button in the UI's header.
-github = "https://github.com/DL4DS/dl4ds_tutor"
+# github = ""
 
 # Specify a CSS file that can be used to customize the user interface.
 # The CSS file can be served from the public directory or via an external link.
-custom_css = "/public/test.css"
+# custom_css = "/public/test.css"
 
 # Specify a Javascript file that can be used to customize the user interface.
 # The Javascript file can be served from the public directory.
@@ -79,7 +79,7 @@ custom_css = "/public/test.css"
 # custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap"
 
 # Specify a custom meta image url.
-custom_meta_image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/f/f5/Boston_University_seal.svg/1200px-Boston_University_seal.svg.png"
+# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png"
 
 # Specify a custom build directory for the frontend.
 # This can be used to customize the frontend code.
@@ -87,7 +87,7 @@ custom_meta_image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/f/
 # custom_build = "./public/build"
 
 [UI.theme]
-    default = "light"
+    default = "dark"
     #layout = "wide"
     #font_family = "Inter, sans-serif"
 # Override default MUI light theme. (Check theme.ts)

{code → apps/ai_tutor}/app.py

@@ -1,3 +1,10 @@
+import sys
+import os
+
+# Add the parent directory of dl4ds_tutor to sys.path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
+
+
 from fastapi import FastAPI, Request, Response, HTTPException
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates

{code → apps/ai_tutor}/main.py

@@ -38,7 +38,7 @@ AGENT = "Agent"
 YOU = "User"
 ERROR = "Error"
 
-with open("modules/config/config.yml", "r") as f:
+with open("../../modules/config/config.yml", "r") as f:
     config = yaml.safe_load(f)
 
 
@@ -85,7 +85,7 @@ class Chatbot:
         """
         Load the configuration from a YAML file.
         """
-        with open("modules/config/config.yml", "r") as f:
+        with open("../../modules/config/config.yml", "r") as f:
             return yaml.safe_load(f)
 
     @no_type_check

apps/chainlit_base/.chainlit/config.toml

@@ -0,0 +1,120 @@
+[project]
+# Whether to enable telemetry (default: true). No personal data is collected.
+enable_telemetry = true
+
+
+# List of environment variables to be provided by each user to use the app.
+user_env = []
+
+# Duration (in seconds) during which the session is saved when the connection is lost
+session_timeout = 3600
+
+# Enable third parties caching (e.g LangChain cache)
+cache = false
+
+# Authorized origins
+allow_origins = ["*"]
+
+# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317)
+# follow_symlink = false
+
+[features]
+# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
+unsafe_allow_html = false
+
+# Process and display mathematical expressions. This can clash with "$" characters in messages.
+latex = false
+
+# Automatically tag threads with the current chat profile (if a chat profile is used)
+auto_tag_thread = true
+
+# Authorize users to spontaneously upload files with messages
+[features.spontaneous_file_upload]
+    enabled = true
+    accept = ["*/*"]
+    max_files = 20
+    max_size_mb = 500
+
+[features.audio]
+    # Threshold for audio recording
+    min_decibels = -45
+    # Delay for the user to start speaking in MS
+    initial_silence_timeout = 3000
+    # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop.
+    silence_timeout = 1500
+    # Above this duration (MS), the recording will forcefully stop.
+    max_duration = 15000
+    # Duration of the audio chunks in MS
+    chunk_duration = 1000
+    # Sample rate of the audio
+    sample_rate = 44100
+
+edit_message = true
+
+[UI]
+# Name of the assistant.
+name = "Assistant"
+
+# Description of the assistant. This is used for HTML tags.
+# description = ""
+
+# Large size content are by default collapsed for a cleaner ui
+default_collapse_content = true
+
+# Chain of Thought (CoT) display mode. Can be "hidden", "tool_call" or "full".
+cot = "full"
+
+# Link to your github repo. This will add a github button in the UI's header.
+# github = ""
+
+# Specify a CSS file that can be used to customize the user interface.
+# The CSS file can be served from the public directory or via an external link.
+# custom_css = "/public/test.css"
+
+# Specify a Javascript file that can be used to customize the user interface.
+# The Javascript file can be served from the public directory.
+# custom_js = "/public/test.js"
+
+# Specify a custom font url.
+# custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap"
+
+# Specify a custom meta image url.
+# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png"
+
+# Specify a custom build directory for the frontend.
+# This can be used to customize the frontend code.
+# Be careful: If this is a relative path, it should not start with a slash.
+# custom_build = "./public/build"
+
+[UI.theme]
+    default = "dark"
+    #layout = "wide"
+    #font_family = "Inter, sans-serif"
+# Override default MUI light theme. (Check theme.ts)
+[UI.theme.light]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.light.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+    [UI.theme.light.text]
+        #primary = "#212121"
+        #secondary = "#616161"
+
+# Override default MUI dark theme. (Check theme.ts)
+[UI.theme.dark]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.dark.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+    [UI.theme.dark.text]
+        #primary = "#EEEEEE"
+        #secondary = "#BDBDBD"
+
+[meta]
+generated_by = "1.1.402"

apps/chainlit_base/chainlit.md

@@ -0,0 +1,14 @@
+# Welcome to Chainlit! 🚀🤖
+
+Hi there, Developer! 👋 We're excited to have you on board. Chainlit is a powerful tool designed to help you prototype, debug and share applications built on top of LLMs.
+
+## Useful Links 🔗
+
+- **Documentation:** Get started with our comprehensive [Chainlit Documentation](https://docs.chainlit.io) 📚
+- **Discord Community:** Join our friendly [Chainlit Discord](https://discord.gg/k73SQ3FyUh) to ask questions, share your projects, and connect with other developers! 💬
+
+We can't wait to see what you create with Chainlit! Happy coding! 💻😊
+
+## Welcome screen
+
+To modify the welcome screen, edit the `chainlit.md` file at the root of your project. If you do not want a welcome screen, just leave this file empty.

{code → apps/chainlit_base}/chainlit_base.py

@@ -22,7 +22,7 @@ AGENT = "Agent"
 YOU = "User"
 ERROR = "Error"
 
-with open("modules/config/config.yml", "r") as f:
+with open("config/config.yml", "r") as f:
     config = yaml.safe_load(f)
 
 
@@ -51,7 +51,7 @@ class Chatbot:
         """
         Load the configuration from a YAML file.
         """
-        with open("modules/config/config.yml", "r") as f:
+        with open("config/config.yml", "r") as f:
             return yaml.safe_load(f)
 
     @no_type_check

apps/chainlit_base/config/config.yml

@@ -0,0 +1,60 @@
+log_dir: 'storage/logs' # str
+log_chunk_dir: 'storage/logs/chunks' # str
+device: 'cpu' # str [cuda, cpu]
+
+vectorstore:
+  load_from_HF: True # bool
+  reparse_files: True # bool
+  data_path: 'storage/data' # str
+  url_file_path: 'storage/data/urls.txt' # str
+  expand_urls: True # bool
+  db_option : 'RAGatouille' # str [FAISS, Chroma, RAGatouille, RAPTOR]
+  db_path : 'vectorstores' # str
+  model : 'sentence-transformers/all-MiniLM-L6-v2' # str [sentence-transformers/all-MiniLM-L6-v2, text-embedding-ada-002']
+  search_top_k : 3 # int
+  score_threshold : 0.2 # float
+
+  faiss_params: # Not used as of now
+    index_path: 'vectorstores/faiss.index' # str
+    index_type: 'Flat' # str [Flat, HNSW, IVF]
+    index_dimension: 384 # int
+    index_nlist: 100 # int
+    index_nprobe: 10 # int
+
+  colbert_params:
+    index_name: "new_idx" # str
+
+llm_params: 
+  llm_arch: 'langchain' # [langchain]
+  use_history: True # bool
+  generate_follow_up: False # bool
+  memory_window: 3 # int
+  llm_style: 'Normal' # str [Normal, ELI5]
+  llm_loader: 'gpt-4o-mini' # str [local_llm, gpt-3.5-turbo-1106, gpt-4, gpt-4o-mini]
+  openai_params:
+    temperature: 0.7 # float
+  local_llm_params:
+    temperature: 0.7 # float
+    repo_id: 'TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF' # HuggingFace repo id
+    filename: 'tinyllama-1.1b-chat-v1.0.Q5_0.gguf' # Specific name of gguf file in the repo
+    model_path: 'storage/models/tinyllama-1.1b-chat-v1.0.Q5_0.gguf' # Path to the model file
+  stream: False # bool
+  pdf_reader: 'gpt' # str [llama, pymupdf, gpt]
+
+chat_logging:
+  log_chat: True # bool
+  platform: 'literalai'
+  callbacks: True # bool
+
+splitter_options:
+  use_splitter: True # bool
+  split_by_token : True # bool
+  remove_leftover_delimiters: True # bool
+  remove_chunks: False # bool
+  chunking_mode: 'semantic' # str [fixed, semantic]
+  chunk_size : 300 # int
+  chunk_overlap : 30 # int
+  chunk_separators : ["\n\n", "\n", " ", ""] # list of strings
+  front_chunks_to_remove : null # int or None
+  last_chunks_to_remove : null # int or None
+  delimiters_to_remove : ['\t', '\n', '   ', '  '] # list of strings

apps/chainlit_base/config/project_config.yml

@@ -0,0 +1,7 @@
+retriever:
+  retriever_hf_paths:
+    RAGatouille: "XThomasBU/Colbert_Index"
+
+metadata:
+  metadata_links: ["https://dl4ds.github.io/sp2024/lectures/", "https://dl4ds.github.io/sp2024/schedule/"]
+  slide_base_link: "https://dl4ds.github.io"

code/chainlit.md

@@ -1,5 +0,0 @@
-# Welcome to DL4DS Tutor! 🚀🤖
-
-Hi there, this is an LLM chatbot designed to help answer questions on the course content.
-
-### --- Please wait while the Tutor loads... ---

{code/modules → modules}/chat/helpers.py

@@ -1,4 +1,4 @@
-from modules.config.prompts import prompts
+from modules.config.prompts import prompts  # TODO: MOVE THIS TO APP SPECIFIC DIRECTORY
 import chainlit as cl
 
 

{code/modules → modules}/dataloader/helpers.py

@@ -2,7 +2,7 @@ import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urlparse
 import tempfile
-from modules.config.constants import TIMEOUT
+from modules.config.constants import TIMEOUT  # TODO: MOVE THIS TO APP SPECIFIC DIRECTORY
 
 
 def get_urls_from_file(file_path: str):