Upload 20 files

.dockerignore

@@ -0,0 +1,91 @@
+# Git
+.git
+.gitignore
+.gitattributes
+
+
+# CI
+.codeclimate.yml
+.travis.yml
+.taskcluster.yml
+
+# Docker
+docker-compose.yml
+Dockerfile
+.docker
+.dockerignore
+
+# Byte-compiled / optimized / DLL files
+**/__pycache__/
+**/*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Virtual environment
+.env
+.venv/
+venv/
+
+# PyCharm
+.idea
+
+# Python mode for VIM
+.ropeproject
+**/.ropeproject
+
+# Vim swap files
+**/*.swp
+
+# VS Code
+.vscode/
+
+start.sh

.gitignore

@@ -0,0 +1,171 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc

.streamlit/config.toml

@@ -0,0 +1,7 @@
+[server]
+enableStaticServing = true
+maxUploadSize = 30
+
+[client]
+showErrorDetails = true
+

Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.12-slim-bookworm
+
+WORKDIR /app
+
+# Update package index and install necessary tools, including curl
+RUN apt-get update && apt-get install -y curl
+
+# Download the model
+RUN curl -Lo qwen2.5-coder-3b-instruct-q4_k_m.gguf https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/resolve/main/qwen2.5-coder-3b-instruct-q4_k_m.gguf?download=true
+
+# Install build tools required for llama-cpp-python
+RUN apt-get update && apt-get install -y build-essential
+
+# Copy requirements and install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Redundant command to invalidate cache
+RUN echo "Intentionally invalidating cache"
+
+# Copy the application code
+COPY . .
+
+# Expose the port that Streamlit will run on
+EXPOSE 8501
+
+# Define the entrypoint to run the Streamlit application
+ENTRYPOINT ["streamlit", "run", "🤖SQL_Agent.py", "--server.port=8501", "--server.address=0.0.0.0"]

pages/2 📊Manual SQL Executer.py

@@ -0,0 +1,129 @@
+import streamlit as st
+from utils.sql_utils import load_data, load_defaultdb_schema_text
+from utils.handle_sql_commands import execute_sql_duckdb
+
+st.markdown(
+    """
+<style>
+    /* Base styles for both themes */
+    .stPageLink {
+        background-image: linear-gradient(to right, #007BFF, #6610F2); /* Gradient background */
+        color: white !important; /* Ensure text is readable on the gradient */
+        padding: 12px 20px !important; /* Slightly larger padding */
+        border-radius: 8px !important; /* More rounded corners */
+        border: none !important; /* Remove default border */
+        text-decoration: none !important;
+        font-weight: 500 !important; /* Slightly lighter font weight */
+        transition: transform 0.2s ease-in-out, box-shadow 0.2s ease-in-out; /* Smooth transitions */
+        box-shadow: 0 2px 5px rgba(0, 0, 0, 0.15); /* Subtle shadow for depth */
+        display: inline-flex;
+        align-items: center;
+        justify-content: center;
+    }
+
+    .stPageLink:hover {
+        transform: scale(1.03); /* Slight scale up on hover */
+        box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); /* Increased shadow on hover */
+    }
+
+    .stPageLink span { /* Style the label text */
+        margin-left: 5px; /* Space between icon and text */
+    }
+
+    /* Dark theme adjustments (optional, if needed for better contrast) */
+    /* Consider using Streamlit's theme variables if possible for a more robust solution */
+    /* For simplicity, this example uses fixed colors that should work reasonably well */
+    /* [data-theme="dark"] .stPageLink {
+    }
+
+    [data-theme="dark"] .stPageLink:hover {
+    } */
+</style>
+""",
+    unsafe_allow_html=True,
+)
+
+default_dfs = load_data()
+
+if "uploaded_dataframes" not in st.session_state:
+    st.session_state.uploaded_dataframes = {}
+
+uploaded_dataframes = st.session_state.uploaded_dataframes
+
+
+with st.popover("Click here to see Database Schema", use_container_width=True):
+    uploaded_df_schema = st.session_state.get("uploaded_df_schema", False)
+
+    choice = st.segmented_control(
+        "Choose",
+        ["Default DB", "Uploaded Files"],
+        label_visibility="collapsed",
+        disabled=uploaded_df_schema == False,
+        default="Default DB" if uploaded_df_schema == False else "Uploaded Files",
+    )
+
+    if uploaded_df_schema is False:
+        st.markdown("""> If you want to use your own files(csv/xls), click here - """)
+        st.page_link(
+            page="pages/3 📂File Upload for SQL.py",
+            label="Upload your own CSV or Excel files",
+            icon="📜",
+        )
+        schema = load_defaultdb_schema_text()
+        st.markdown(schema, unsafe_allow_html=True)
+    elif choice == "Default DB":
+        schema = load_defaultdb_schema_text()
+        st.markdown(schema, unsafe_allow_html=True)
+    else:
+        pretty_schema, markdown = st.tabs(["Schema", "Copy Schema in Markdown"])
+        with pretty_schema:
+            st.info(
+                "You can copy this schema, and give it to any state of the art LLM models like (Gemini /ChatGPT /Claude etc) to cross check your answers.\n You can run the queries directly here, by using ***Manual Query Executer*** in the sidebar and download your results 😊",
+                icon="ℹ️",
+            )
+            st.markdown(uploaded_df_schema, unsafe_allow_html=True)
+        with markdown:
+            st.info(
+                "You can copy this schema, and give it to any state of the art LLM models like (Gemini /ChatGPT /Claude etc) to cross check your answers.\n You can run the queries directly here, by using ***Manual Query Executer*** in the sidebar and download your results 😊",
+                icon="ℹ️",
+            )
+            st.markdown(f"```\n{uploaded_df_schema}\n```")
+
+
+data_source = None
+if uploaded_dataframes is None or len(uploaded_dataframes) == 0:
+    data_source = "Default Database"
+    col1, col2 = st.columns([4, 1])
+    with col1:
+        st.caption(
+            "Use this dialog to execute SQL commands on the Default Database. To use your own Excel/CSV files"
+        )
+    with col2:
+        st.page_link(page="pages/3 📂File Upload for SQL.py", label="Click Here")
+else:
+    data_source = st.radio("Select Data Source", ["Default Database", "Uploaded Files"])
+
+if data_source == "Default Database":
+    dataframes = default_dfs
+    if uploaded_dataframes is not None and len(uploaded_dataframes) > 0:
+        st.caption(
+            "Use this dialog to execute SQL commands on the Default Database. To use your own Excel/CSV files"
+        )
+else:
+    dataframes = uploaded_dataframes
+    st.caption("Use this dialog to execute SQL commands on Your Uploaded Files")
+
+sql_command = st.text_area("Enter your SQL command here:")
+if st.button("Execute"):
+    df = execute_sql_duckdb(sql_command, dataframes)
+    if df is not None:
+        st.dataframe(df)
+        st.info(f"Rows x Columns: {df.shape[0]} x {df.shape[1]}")
+        st.subheader("Data Description:")
+        st.markdown(df.describe().T.to_markdown())
+        st.subheader("Data Types:")
+        st.write(df.dtypes)
+    else:
+        st.error(
+            "An error occurred while executing the SQL command. Please cross check your command as well as tables & columns names."
+        )

pages/3 📂File Upload for SQL.py

@@ -0,0 +1,90 @@
+import streamlit as st
+import pandas as pd
+from utils.sql_utils import create_schema
+
+MAX_FILES = 5
+
+if "uploaded_dataframes" not in st.session_state:
+    st.session_state.uploaded_dataframes = {}
+
+st.header("Upload your CSV or Excel files")
+st.caption("Maximum 5 files can be uploaded.")
+
+num_uploaded_files = len(st.session_state.uploaded_dataframes)
+
+disabled = num_uploaded_files >= MAX_FILES
+
+uploaded_files = st.file_uploader(
+    "Choose files",
+    type=["csv", "xlsx"],
+    accept_multiple_files=True,
+    disabled=disabled,
+)
+
+if uploaded_files and not disabled:
+    uploaded_count = 0
+    for uploaded_file in uploaded_files:
+        if len(st.session_state.uploaded_dataframes) < MAX_FILES:
+            df = None
+            try:
+                if uploaded_file.name.endswith(".csv"):
+                    df = pd.read_csv(uploaded_file)
+                elif uploaded_file.name.endswith(".xlsx"):
+                    df = pd.read_excel(uploaded_file)
+
+                if uploaded_file.name in st.session_state.uploaded_dataframes:
+                    st.toast(
+                        f"File {uploaded_file.name} already uploaded. Skipping...",
+                        icon="⚠️",
+                    )
+                else:
+                    key = (
+                        uploaded_file.name.lower()
+                        .strip()
+                        .replace(" ", "_")
+                        .replace(".csv", "")
+                        .replace(".xlsx", "")
+                    )
+                    st.session_state.uploaded_dataframes[key] = df
+                    uploaded_count += 1
+                print(f"Uploaded file: {str(uploaded_file.name)}")
+            except Exception as e:
+                st.error(f"Error reading file {uploaded_file.name}: {e}")
+        else:
+            st.warning(
+                f"Maximum number of files({MAX_FILES}) reached. Cannot upload more files."
+            )
+            break
+    if uploaded_count > 0:
+        st.success(
+            f"{uploaded_count} File(s) uploaded successfully. Total {len(st.session_state.uploaded_dataframes)} File(s)"
+        )
+
+
+if len(st.session_state.uploaded_dataframes) >= MAX_FILES:
+    st.warning(
+        f"Maximum number of files({MAX_FILES}) reached. Cannot upload more files."
+    )
+
+
+dataframes = st.session_state.uploaded_dataframes
+
+if dataframes:
+    st.header("Schema of Uploaded Files📚")
+    schema = create_schema(dataframes)
+    if "uploaded_df_schema" not in st.session_state:
+        st.session_state.uploaded_df_schema = schema
+    pretty_schema, markdown = st.tabs(["Schema", "Copy Schema in Markdown"])
+    with pretty_schema:
+        st.info(
+            "You can copy this schema, and give it to any state of the art LLM models like (Gemini /ChatGPT /Claude etc) to cross check your answers.\n You can run the queries directly here, by using ***Manual Query Executer*** in the sidebar and download your results 😊",
+            icon="ℹ️",
+        )
+        st.markdown(schema, unsafe_allow_html=True)
+    with markdown:
+        st.info(
+            "You can copy this schema, and give it to any state of the art LLM models like (Gemini /ChatGPT /Claude etc) to cross check your answers.\n You can run the queries directly here, by using ***Manual Query Executer*** in the sidebar and download your results 😊",
+            icon="ℹ️",
+        )
+        st.markdown(f"```\n{schema}\n```")
+

pages/4 👨‍🎓Connect with me.py

@@ -0,0 +1,276 @@
+import streamlit as st
+
+st.markdown(
+    """
+    <style>
+        body {
+            background-color: #1e1e1e;
+            color: #f0f0f0;
+            font-family: sans-serif;
+        }
+        h1, h2, h3, h4, h5, h6 {
+            color: #f0f0f0;
+        }
+        a {
+            color: #89b4fa;
+            text-decoration: none;
+        }
+        a:hover {
+            text-decoration: underline;
+        }
+        .social-link-button {
+            display: inline-block;
+            padding: 8px 16px;
+            margin: 4px;
+            background-color: #333;
+            color: #fff;
+            border-radius: 5px;
+            text-decoration: none;
+        }
+        .social-link-button:hover {
+            background-color: #555;
+        }
+        .project-card {
+            background-color: #282828;
+            border: 1px solid #444;
+        }
+        .skills, .experience, .projects, .volunteering, .achievements, .certifications, .education {
+            background-color: #282828;
+            border: 1px solid #444;
+        }
+        .status.deployed {
+            color: #a3be8c;
+        }
+        .status.ongoing {
+            color: #f9bb6d;
+        }
+        footer {
+            background-color: #333;
+            color: #f0f0f0;
+        }
+    </style>
+    """,
+    unsafe_allow_html=True,
+)
+
+st.html(
+    """<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Debopam Chowdhury (Param)</title>
+</head>
+<body>
+    <header style="text-align: center; padding: 2rem 0; border-bottom: 1px solid #444;">
+        <img src="https://tinyurl.com/ysa6yekw" alt="Debopam Chowdhury (Param)" class="profile-image" style="width: 150px; height: 150px; border-radius: 50%; object-fit: cover; margin-bottom: 1rem;">
+        <h1>Debopam Chowdhury <span class="nickname" style="font-weight: normal; color: #999;">(Param)</span></h1>
+        <p class="tagline" style="color: #ccc; margin-bottom: 1rem;">Creator of <a href="http://aigymbuddy.in" target="_blank" style="color: #89b4fa;">AiGymBuddy.in</a> | Machine Learning | Deep Learning | Flutter | Math | MLops | TensorFlow | FastAPI</p>
+        <p class="pronouns" style="font-size: 0.9rem; color: #999; margin-bottom: 1rem;">(He/Him)</p>
+        <div class="social-links">
+            <a href="https://www.linkedin.com/in/debopam-chowdhury-param-600619229/" target="_blank" class="social-link-button">LinkedIn</a>
+            <a href="https://www.youtube.com/@DCparam/featured" target="_blank" class="social-link-button">YouTube</a>
+            <a href="https://github.com/DebopamParam" target="_blank" class="social-link-button">GitHub</a>
+            <a href="https://www.instagram.com/debopam_param.ai/" target="_blank" class="social-link-button">Instagram</a>
+        </div>
+    </header>
+
+    <section class="about-me" style="padding: 2rem; margin-bottom: 1.5rem; border-radius: 8px;">
+        <h2>About Me</h2>
+        <p style="line-height: 1.7;">Passionate about Machine Learning, Computer Science and Mathematics. Has a strong grip over Machine & Deep Learning Fundamentals, Computer Networks, OS and DSA. Solved 100+ problems on leetcode. Have a keen interest in learning Mathematics, Deep Learning and Statistics. I like to understand things in a deeper way.</p>
+    </section>
+
+    <section class="skills" style="padding: 2rem; margin-bottom: 1.5rem; border-radius: 8px;">
+        <h2>Skills</h2>
+        <div class="skills-grid" style="display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1.5rem;">
+            <div class="skill-category">
+                <h3>Technical</h3>
+                <ul style="padding-left: 0; list-style: none;">
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><strong>Programming:</strong> Python, Java, Dart, VS Code, Git, GitHub, Jupyter Notebooks, CI/CD</li>
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><strong>Machine Learning & Deep Learning:</strong> TensorFlow, PyTorch, Scikit-learn, Keras, Supervised Learning, Unsupervised Learning, Neural Networks, Sequence Modeling, Convolution, Attention Mechanisms, Transformer, GPT, BERT, Hyperparameter Optimization</li>
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><strong>Data Handling:</strong> Pandas, NumPy, Data Manipulation, Data Preparation, SQL, Pyspark, NoSQL</li>
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><strong>Cloud & DevOps:</strong> Docker, AWS, Cloud-AI, FastAPI</li>
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><strong>Mathematics:</strong> Linear Algebra, Probability, Statistics, Boosting Methods</li>
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><strong>Frameworks & Tools:</strong> Flutter, Firebase, Deep Learning Frameworks, Model Training & Optimization, Version Control, LLM FineTuning</li>
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><strong>Generative AI:</strong> Vector Embeddings, Indexing, Chunking, RAG pipelines, LlamaIndex, LangChain, Colpali, Byaldi, Chroma DB</li>
+                </ul>
+            </div>
+        </div>
+    </section>
+
+    <section class="experience" style="padding: 2rem; margin-bottom: 1.5rem; border-radius: 8px;">
+        <h2>Experience</h2>
+        <div class="experience-item" style="margin-bottom: 1.5rem;">
+            <h3>GENERATIVE AI ENGINEER (Contract - Remote)</h3>
+            <p class="company" style="font-weight: bold; color: #bbb; margin-bottom: 0.25rem;">Private Client | Sydney, Australia</p>
+            <p class="duration" style="font-size: 0.9rem; color: #999; margin-bottom: 0.75rem;">October - November 2024</p>
+            <ul style="margin-top: 0.5rem; padding-left: 20px;">
+                <li style="margin-bottom: 0.5rem; font-size: 0.95rem;">Developed secure, on-premise solutions for private Complex data (Complex PDF with Images and Charts) Q&A and knowledge retrieval.</li>
+                <li style="margin-bottom: 0.5rem; font-size: 0.95rem;">Built data ingestion pipeline (100-500 documents daily) with vision embeddings and task scheduler updates.</li>
+                <li style="margin-bottom: 0.5rem; font-size: 0.95rem;">Built multimodal RAG pipelines (Byaldi, Colqwen2, Pixtral 12B) optimized for diverse document types (70% accuracy improvement).</li>
+                <li style="margin-bottom: 0.5rem; font-size: 0.95rem;">Containerized the application with Docker for deployment flexibility.</li>
+                <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><strong>Open Source Contribution - Byaldi - 575✩:</strong> while working on It</li>
+                <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><strong>Technologies:</strong> NLP, Vision Embeddings, Local Multimodal RAG, LangChain, Pixtral 12B, Col-Qwen2, Byaldi</li>
+            </ul>
+        </div>
+    </section>
+
+    <section class="projects" style="padding: 2rem; margin-bottom: 1.5rem; border-radius: 8px;">
+        <h2>Projects</h2>
+        <div class="project-list" style="display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1.5rem;">
+            <div class="project-card" style="border: 1px solid #444; padding: 1.5rem; border-radius: 8px;">
+                <h3 style="margin-top: 0; margin-bottom: 0.75rem;">LLM-finetuning and SQL Agent with Auto_Execution with DuckDB, Schema Retriever from CSVs, manual SQL executer.</h3>
+                <p class="status ongoing" style="font-size: 0.85rem; font-weight: bold; margin-top: 0.5rem; color: #f9bb6d;">Current WebAPP which you are using</p>
+                <p class="status deployed" style="font-size: 0.85rem; font-weight: bold; margin-top: 0.5rem; color: #a3be8c;">Deployed <i class="fas fa-check-circle"></i></p>
+            </div>
+
+            <div class="project-card" style="border: 1px solid #444; padding: 1.5rem; border-radius: 8px;">
+                <h3 style="margin-top: 0; margin-bottom: 0.75rem;">Deep Learning Based Recommendation System</h3>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;">Initially, I aimed to build a recommendation system from scratch using TensorFlow Recommenders (TFRS) on the MovieLens 1M dataset. This involved creating user and movie embeddings with a candidate generation and ranking model.</p>
+
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;">However, this TFRS approach proved too resource-intensive and time-consuming for effective training and testing. Crucially, the initial results weren't satisfactory for deployment.</p>
+
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;">Therefore, for the deployed web application, I switched to pre-trained models (BGE embeddings and re-ranking). This offered:</p>
+                <ul style="padding-left: 20px;">
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><b>Better Performance:</b> More relevant recommendations.</li>
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;"><b>Reduced Resources/Time:</b> Faster training and deployment.</li>
+                </ul>
+
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;">While the TFRS code is also included in the web app, the pre-trained model approach was chosen for its superior results and efficiency in a deployment setting. A future improvement could be fine-tuning the pre-trained models for even better performance.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Technologies used:</strong> TensorFlow Recommenders, Scann, Vector DB, Distributed GPU Training, Langchain, Streamlit, BAAI BGE Models</p>
+                <div class="project-links">
+                    <a href="https://debopam-movie-recommendation-system.streamlit.app/" target="_blank" style="display: inline-block; margin-right: 1rem; font-size: 0.9rem;">Try it out APP Live</a>
+                </div>
+                <p class="status deployed" style="font-size: 0.85rem; font-weight: bold; margin-top: 0.5rem; color: #a3be8c;">Deployed <i class="fas fa-check-circle"></i></p>
+            </div>
+
+            <div class="project-card" style="border: 1px solid #444; padding: 1.5rem; border-radius: 8px;">
+                <h3 style="margin-top: 0; margin-bottom: 0.75rem;">IBM EMPLOYEE ATTRITION PREDICTOR (End to End with Deployment to AWS, FastAPI with Proxy Server)</h3>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Objective:</strong> Predicted employee attrition with 85% AUC to improve employee retention and business performance.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Model Development:</strong> Hyperparameter optimized Multi-Layer Perceptron, XGBoost, Logistic Regression with Inference as well as Training Pipeline</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Backend:</strong> Developed a FastAPI backend for real-time predictions, using Pydantic for schema validation for incoming and outgoing requests</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Deployment:</strong> Containerized with Docker, deployed on AWS EC2, managed via AWS ECR.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>CI/CD:</strong> Set up an automated CI/CD pipeline using GitHub Actions for seamless updates.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Web Application:</strong> Built a user-friendly interface using Flutter Web for real-time interaction.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Security:</strong> Handled HTTPS requests using Caddy as a reverse proxy server</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Technologies used:</strong> TensorFlow, AWS, Docker, FastAPI, CI/CD Pipeline, Multi-Layer Perceptron, Neural Network, XGBoost, Logistic Regression, Hyperparameter Tuned Models, GitHub Actions, Pydantic, Flutter Web, Reverse-Proxy-Server: Caddy</p>
+                <div class="project-links">
+                    <a href="https://www.linkedin.com/posts/debopam-chowdhury-param-600619229_machinelearning-deeplearning-aws-activity-7244476917884608512-DfbD/?utm_source=share&utm_medium=member_desktop" target="_blank" style="display: inline-block; margin-right: 1rem; font-size: 0.9rem;">Explanation & Live Demo</a>
+                    <a href="http://www.debopamchowdhury.works" target="_blank" style="display: inline-block; margin-right: 1rem; font-size: 0.9rem;">Click to try out Live</a>
+                </div>
+                <p class="status deployed" style="font-size: 0.85rem; font-weight: bold; margin-top: 0.5rem; color: #a3be8c;">Deployed <i class="fas fa-check-circle"></i></p>
+            </div>
+
+            <div class="project-card" style="border: 1px solid #444; padding: 1.5rem; border-radius: 8px;">
+                <h3 style="margin-top: 0; margin-bottom: 0.75rem;">AI GYM BUDDY (Langchain | Flutter | Riverpod | Gemini)</h3>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;">Personalized AI-Driven Workouts with Smart Equipment Detection and Progress Tracking</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Features:</strong> Al Instrument Detection (Camera or Gallery), Exercises based on Available Equipments, Time, Preffered Muscle Groups & Custom requests, Dynamic Video Tutorial Finder for each exercise, Super personalized Al generated routine, Workout History Tracker, Easy SignUp/Login with Google Oauth</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Technologies used:</strong> Dart, flutter, firebase, gemini 1.5 flash, riverpod, langchain, fastapi, google oauth</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Licenses:</strong> This code of this app/website is written from scratch and I hold all the rights over distribution</p>
+                <div class="project-links">
+                    <a href="https://www.youtube.com/shorts/0ZR0IWiZJQE" target="_blank" style="display: inline-block; margin-right: 1rem; font-size: 0.9rem;">1 Minute Video Demo</a>
+                    <a href="https://play.google.com/store/apps/details?id=com.aigymbuddy.me&hl=en" target="_blank" style="display: inline-block; margin-right: 1rem; font-size: 0.9rem;">
+                        <img src="https://static-00.iconduck.com/assets.00/google-play-icon-2048x2048-487quz63.png" alt="Google Play Store" class="playstore-icon" style="width: 16px; height: 16px; vertical-align: middle; margin-right: 0.25rem;"> Google Play Store (Android)
+                    </a>
+                    <a href="http://www.aigymbuddy.in" target="_blank" style="display: inline-block; margin-right: 1rem; font-size: 0.9rem;">Prototype for WEB www.aigymbuddy.in</a>
+                </div>
+                <p class="status deployed" style="font-size: 0.85rem; font-weight: bold; margin-top: 0.5rem; color: #a3be8c;">Deployed <i class="fas fa-check-circle"></i></p>
+            </div>
+
+            <div class="project-card" style="border: 1px solid #444; padding: 1.5rem; border-radius: 8px;">
+                <h3 style="margin-top: 0; margin-bottom: 0.75rem;">Non-Sequential Breast Cancer Classification System</h3>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Multi-Modal Cancer Detection:</strong> Developed a novel multi-output deep learning model for breast cancer detection, predicting cancer presence, invasiveness, and difficult-negative case status. The model incorporates both mammogram images and tabular clinical data, leveraging a non-sequential architecture to process distinct data modalities.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Fine-Tuned Image Feature Extraction:</strong> Utilized a pre-trained EfficientNetV2B3 model for image feature extraction, fine-tuning layers from block 6 onwards to enhance its applicability to the specific task, thus improving the quality of learned representations and potentially making the model more robust and accurate.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Distributed Training:</strong> Accelerated model training through distributed training using TensorFlow's MirroredStrategy on 2xT4 GPUs for 9 hours on Kaggle, demonstrating proficiency in optimizing model training with limited computational resources.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Technologies used:</strong> TensorFlow, Transfer Learning, EfficientNetV2, Fused MB-CNN</p>
+                <div class="project-links">
+                    <a href="https://debopamparam-bcd-inference-vvyb1v.streamlit.app/" target="_blank" style="display: inline-block; margin-right: 1rem; font-size: 0.9rem;">Live Webapp + Architecture + Training Code + Evaluation Metrics</a>
+                </div>
+                <p class="status deployed" style="font-size: 0.85rem; font-weight: bold; margin-top: 0.5rem; color: #a3be8c;">Deployed <i class="fas fa-check-circle"></i></p>
+            </div>
+
+            <div class="project-card" style="border: 1px solid #444; padding: 1.5rem; border-radius: 8px;">
+                <h3 style="margin-top: 0; margin-bottom: 0.75rem;">Image Entity Extraction with Qwen2 VL: Large-Scale Inference</h3>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Problem Statement:</strong> E-commerce and healthcare industries struggle to efficiently extract product details (weight, volume, dimensions) from images at scale.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Action:</strong> Developed a large-scale image-to-text inference pipeline using Qwen2 VL: 2B, incorporating image preprocessing, Regex, and parallel processing. Processed 84,000 of 131,000 test images.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Result:</strong> Successfully extracted product values from a significant portion of the dataset. Our team of four ranked 172nd out of ~75,000 in the Amazon ML Challenge with Fl-Score=0.47, demonstrating the solution's potential for automated product information extraction.</p>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Technologies used:</strong> Qwen2 VL, Python, Regex, Parallel Processing</p>
+                <div class="project-links">
+                    <a href="https://colab.research.google.com/drive/1V5F1XMlYNHzv-hA9xmIJ-Jx5vuD0fKAR?usp=sharing" target="_blank" style="display: inline-block; margin-right: 1rem; font-size: 0.9rem;">Click Here to see the code</a>
+                </div>
+            </div>
+
+            <div class="project-card" style="border: 1px solid #444; padding: 1.5rem; border-radius: 8px;">
+                <h3 style="margin-top: 0; margin-bottom: 0.75rem;">LLM based ATS System using VertexAI Embedding</h3>
+                <p style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>Technologies used:</strong> Langchain, VertexAI Embedding, StreamLit, PostGresVector</p>
+                <p class="status ongoing" style="font-size: 0.85rem; font-weight: bold; margin-top: 0.5rem; color: #f9bb6d;">Ongoing</p>
+            </div>
+        </div>
+    </section>
+
+    <section class="volunteering" style="padding: 2rem; margin-bottom: 1.5rem; border-radius: 8px;">
+        <h2>Volunteering</h2>
+        <div class="volunteering-item" style="margin-bottom: 1rem;">
+            <h3>Git/GitHub Instructor (Volunteer)</h3>
+            <p style="color: #ccc;">Carried out sessions to teach juniors the fundamentals of Git and GitHub, covering version control, collaboration, and best practices.</p>
+        </div>
+        <div class="volunteering-item" style="margin-bottom: 1rem;">
+            <h3>Event Coordinator</h3>
+            <p style="color: #ccc;">Acharya Technical Club - Steigen</p>
+        </div>
+    </section>
+
+    <section class="achievements" style="padding: 2rem; margin-bottom: 1.5rem; border-radius: 8px;">
+        <h2>Achievements</h2>
+        <ul style="padding-left: 20px;">
+            <li style="margin-bottom: 0.75rem; font-size: 0.95rem;">Secured rank 172 out of 75000 participants in AMAZON ML Challenge Hackathon 2024.</li>
+            <li style="margin-bottom: 0.75rem; font-size: 0.95rem;">2nd place out of 60 in the TechnioD Hackathon.</li>
+            <li style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong><a href="https://github.com/AnswerDotAI/byaldi/pull/50" target="_blank" style="color: #89b4fa;">Open - Source Contribution - Byaldi - 575☆</a></strong>
+                <ul style="padding-left: 20px;">
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;">Fix langchain integration not present in pypi tar & whl-- pyproject.toml</li>
+                </ul>
+            </li>
+            <li style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>NON TECHNICAL :</strong>
+                <ul style="padding-left: 20px;">
+                    <li style="margin-bottom: 0.5rem; font-size: 0.95rem;">Performed in IIT BOMBAY'S Mood Indigo Bengaluru Event - Finalist.</li>
+                </ul>
+            </li>
+        </ul>
+    </section>
+
+    <section class="certifications" style="padding: 2rem; margin-bottom: 1.5rem; border-radius: 8px;">
+        <h2>Certifications</h2>
+        <ul style="padding-left: 20px;">
+            <li style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>COURSERA:</strong> Advanced Learning Algorithms | <a href="https://coursera.org/share/d540318c8cb4a7e802d8c4964a471d34" target="_blank" style="color: #89b4fa;">View Certificate</a></li>
+            <li style="margin-bottom: 0.75rem; font-size: 0.95rem;"><strong>COURSERA:</strong> Supervised Machine Learning: Regression and Classification | <a href="https://coursera.org/share/d540318c8cb4a7e802d8c4964a471d34" target="_blank" style="color: #89b4fa;">View Certificate</a></li>
+        </ul>
+    </section>
+
+    <section class="education" style="padding: 2rem; margin-bottom: 1.5rem; border-radius: 8px;">
+        <h2>Education</h2>
+        <div class="education-item" style="margin-bottom: 0.75rem; font-size: 0.95rem;">
+            <h3>BE in Information Science</h3>
+            <p class="institution" style="color: #ccc;">Acharya Institute of Technology, Bangalore</p>
+            <p class="duration" style="font-size: 0.9rem; color: #999;">2021-2025</p>
+            <p>CGPA-8.12</p>
+        </div>
+        <div class="education-item" style="margin-bottom: 0.75rem; font-size: 0.95rem;">
+            <h3>Higher Secondary Education</h3>
+            <p class="institution" style="color: #ccc;">Kalyani Public School, Barasat, Kolkata</p>
+            <p class="duration" style="font-size: 0.9rem; color: #999;">2021</p>
+            <p>77% (Auto Pass Covid Batch)</p>
+        </div>
+        <div class="education-item" style="margin-bottom: 0.75rem; font-size: 0.95rem;">
+            <h3>Secondary Education</h3>
+            <p class="institution" style="color: #ccc;">Sacred Heart Day High School, Kolkata</p>
+            <p class="duration" style="font-size: 0.9rem; color: #999;">2019</p>
+            <p>90%</p>
+        </div>
+    </section>
+
+    <footer style="text-align: center; padding: 1rem 0; font-size: 0.9rem;">
+        <p>© 2024 Debopam Chowdhury (Param)</p>
+    </footer>
+</body>
+</html>"""
+)

static/database_scema.txt

@@ -0,0 +1,103 @@
+## The Tables in the Default Database are: 
+    `customers`, `order_items`, `orders`, `payments`, and `products`.
+
+### **customers**
+|       | customer_id   |   customer_zip_code_prefix | customer_city   | customer_state   |
+|------:|:--------------|---------------------------:|:----------------|:-----------------|
+| 21921 | 0tgYlOTGgpO6  |                      79230 | russas          | CE               |
+|  9748 | jGhRQF3CIew4  |                      81460 | joao monlevade  | MG               |
+| 22679 | 1UutQTIhBvcP  |                      94480 | pelotas         | RS               |
+
+Rows: 38279, Columns: 4
+
+```sql
+CREATE TABLE customers (
+    customer_id VARCHAR(255) PRIMARY KEY,
+    customer_zip_code_prefix INT,
+    customer_city VARCHAR(255),
+    customer_state VARCHAR(2)
+);
+```
+
+### **order_items**
+|       | order_id     | product_id   | seller_id    |   price |   shipping_charges |
+|------:|:-------------|:-------------|:-------------|--------:|-------------------:|
+| 19729 | PDEzZdebLSn3 | aBpYjaBcwz6e | bzfcwRPnZzVO |   55.83 |              27.8  |
+|  6001 | R7bIPjjYqlHP | ZM2JJXV5m9hl | Ivbw25fb5t2Z |  100    |              42.05 |
+|   282 | Biqo21nETaMO | XqmdGKRbTetH | P2nCHWuo0HC0 |  113.49 |              91.32 |
+
+Rows: 38279, Columns: 5
+
+```sql
+CREATE TABLE order_items (
+    order_id VARCHAR(255),
+    product_id VARCHAR(255),
+    seller_id VARCHAR(255),
+    price DECIMAL(10, 2),
+    shipping_charges DECIMAL(10, 2),
+    PRIMARY KEY (order_id, product_id),
+    FOREIGN KEY (order_id) REFERENCES orders(order_id),
+    FOREIGN KEY (product_id) REFERENCES products(product_id)
+);
+```
+
+### **orders**
+
+|       | order_id     | customer_id   | order_purchase_timestamp   | order_approved_at   |
+|------:|:-------------|:--------------|:---------------------------|:--------------------|
+|  7294 | PMqwQc01iDTJ | c9ueC6k6V5WS  | 2018-06-19 21:23:48        | 2018-06-20 08:38:30 |
+| 13800 | P4l8R2Qat5n7 | ovKkGaXi5TmN  | 2018-01-05 08:26:03        | 2018-01-05 08:47:20 |
+| 17679 | NxIseZjAQCdC | o9qzmUQVJOxA  | 2018-01-28 23:46:53        | 2018-01-28 23:58:31 |
+
+Rows: 38279, Columns: 4
+
+```sql
+CREATE TABLE orders (
+    order_id VARCHAR(255) PRIMARY KEY,
+    customer_id VARCHAR(255),
+    order_purchase_timestamp TIMESTAMP,
+    order_approved_at TIMESTAMP,
+    FOREIGN KEY (customer_id) REFERENCES customers(customer_id)
+);
+```
+
+### **payments**
+|       | order_id     |   payment_sequential | payment_type   |   payment_installments |   payment_value |
+|------:|:-------------|---------------------:|:---------------|-----------------------:|----------------:|
+| 35526 | cQXl0pQtiMad |                    1 | wallet         |                      1 |          172.58 |
+| 35799 | olImD2k316Gz |                    1 | credit_card    |                      3 |           16.78 |
+| 13278 | G9MJYXXtPZSz |                    1 | credit_card    |                     10 |          221.86 |
+
+Rows: 38279, Columns: 5
+
+```sql
+CREATE TABLE payments (
+    order_id VARCHAR(255),
+    payment_sequential INT,
+    payment_type VARCHAR(50),
+    payment_installments INT,
+    payment_value DECIMAL(10, 2),
+    PRIMARY KEY (order_id, payment_sequential),
+    FOREIGN KEY (order_id) REFERENCES orders(order_id)
+);
+```
+
+### **products**
+|       | product_id   | product_category_name   |   product_weight_g |   product_length_cm |   product_height_cm |   product_width_cm |
+|------:|:-------------|:------------------------|-------------------:|--------------------:|--------------------:|-------------------:|
+| 18191 | hpiXwRzTkhkL | bed_bath_table          |               1150 |                  40 |                   9 |                 50 |
+|  2202 | iPoRkE7dkmlc | toys                    |              15800 |                  38 |                  62 |                 57 |
+| 27442 | hrjNaMt3Wyo5 | toys                    |               1850 |                  37 |                  22 |                 40 |
+
+Rows: 38279, Columns: 6
+
+```sql
+CREATE TABLE products (
+    product_id VARCHAR(255) PRIMARY KEY,
+    product_category_name VARCHAR(255),
+    product_weight_g INT,
+    product_length_cm INT,
+    product_height_cm INT,
+    product_width_cm INT
+);
+```

static/default_questions.txt

@@ -0,0 +1,173 @@
+These questions are generated by ChatGpt 4o. Copy and paste the questions in the ChatBox - run it - get the results - compare the results yourself.
+> You can also upload your own files, to get your schemas. You can then use these schemas to cross check your Answer by ChatGpt 4os with any bigger LLM models. You can cross-check directly with our - Manual SQL Executer😊.
+- Ask Questions
+- Run Queries: automatic + manual
+- Download Results
+
+### Easy Questions  
+
+1.  
+   **Question:**  
+   ```  
+   Retrieve all customer IDs and their corresponding cities from the `customers` table.  
+   ```  
+   **Fine-Tuned Model Results:**  
+   ✅ **Pass**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT customer_id, customer_city FROM customers;  
+   ```  
+
+---  
+
+2.  
+   **Question:**  
+   ```  
+   List all products along with their category names from the `products` table.  
+   ```  
+   **Fine-Tuned Model Results:**  
+   ✅ **Pass**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT product_id, product_category_name FROM products;  
+   ```  
+
+---  
+
+3.  
+   **Question:**  
+   ```  
+   Fetch the order IDs and their purchase timestamps from the `orders` table.  
+   ```  
+   **Fine-Tuned Model Results:**  
+   ✅ **Pass**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT order_id, order_purchase_timestamp FROM orders;  
+   ```  
+
+---  
+
+4.  
+   **Question:**  
+   ```  
+   Display the distinct payment types available in the `payments` table.  
+   ```  
+   **Fine-Tuned Model Results:**  
+   ✅ **Pass**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT DISTINCT payment_type FROM payments;  
+   ```  
+
+---  
+
+5.  
+   **Question:**  
+   ```  
+   Find the total number of rows in the `customers` table.  
+   ```  
+   **Fine-Tuned Model Results:**  
+   ✅ **Pass**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT COUNT(*) AS total_customers FROM customers;  
+   ```  
+
+---  
+
+### Medium Questions  
+
+1.  
+   **Question:**  
+   ```  
+   Retrieve the total payment value for each order from the `payments` table, grouped by `order_id`.  
+   ```  
+   **Fine-Tuned Model Results:**  
+   ✅ **Pass**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT order_id, SUM(payment_value) AS total_payment  
+   FROM payments  
+   GROUP BY order_id;  
+   ```  
+
+---  
+
+2.  
+   **Question:**  
+   ```  
+   Find all orders where the total shipping charges (sum of `shipping_charges`) exceed 100.  
+   ```  
+   **Fine-Tuned Model Results:**  
+   ❌ **Fail**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT order_id  
+   FROM order_items  
+   GROUP BY order_id  
+   HAVING SUM(shipping_charges) > 100;  
+   ```  
+   **Issue:** Missing validation for null or non-existent data.  
+
+---  
+
+3.  
+   **Question:**  
+   ```  
+   List the names of cities and the number of customers in each city, sorted in descending order of the number of customers.  
+   ```  
+   **Fine-Tuned Model Results:**  
+   ✅ **Pass**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT customer_city, COUNT(*) AS customer_count  
+   FROM customers  
+   GROUP BY customer_city  
+   ORDER BY customer_count DESC;  
+   ```  
+
+---  
+
+### Hard Questions  
+
+1.  
+   **Question:**  
+   ```  
+   Write a Query to find the total revenue (sum of `price` + `shipping_charges`) generated for each product category in the `order_items` table, joined with the `products` table.  
+   ```   
+   **Fine-Tuned Model Results:**  
+   ✅ **Pass**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT  
+       p.product_category_name,  
+       SUM(o.price + o.shipping_charges) AS total_revenue  
+   FROM order_items o  
+   JOIN products p ON o.product_id = p.product_id  
+   GROUP BY p.product_category_name  
+   ORDER BY total_revenue DESC;  
+   ```  
+
+---  
+
+2.  
+   **Question:**  
+   ```  
+   Write a Query to identify the top 5 products with the highest total sales value ( sum of `price` ) across all orders.  
+   ```  
+   **Fine-Tuned Model Results:**  
+   ❌ **Fail**  
+   **Answer by ChatGpt 4o:**  
+   ```sql  
+   SELECT  
+       product_id,  
+       SUM(price) AS total_sales  
+   FROM order_items  
+   GROUP BY product_id  
+   ORDER BY total_sales DESC  
+   LIMIT 5;  
+   ```  
+   **Issue:** Misalignment with finer-grained filters or lack of handling for tied ranks.  
+
+--- 

utils/handle_sql_commands.py

@@ -0,0 +1,18 @@
+# handle_sql_commands.py
+import duckdb
+import pandas as pd
+import streamlit as st
+
+
+def execute_sql_duckdb(sql_command: str, dataframes: dict) -> pd.DataFrame or None:
+    try:
+        con = duckdb.connect(database=":memory:", read_only=False)
+        for df_name, df in dataframes.items():
+            con.register(df_name, df)
+        result_df = con.execute(sql_command).fetchdf()
+        con.close()
+        return result_df
+    except duckdb.Error as e:
+        st.error(f"DuckDB Error: {e}")
+        return None
+

utils/llm_logic.py

@@ -0,0 +1,230 @@
+# llm_logic.py
+# from langchain_ollama import ChatOllama
+from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
+import streamlit as st
+import multiprocessing
+from langchain_community.chat_models import ChatLlamaCpp
+
+local_model = "qwen2.5-coder-3b-instruct-q4_k_m.gguf"
+
+stop = [
+    "<|image_pad|>",
+    "<|endoftext|>",
+    "<|quad_end|>",
+    "<|object_ref_end|>",
+    "<|object_ref_start|>",
+    "<|file_sep|>",
+    "<|repo_name|>",
+    "<|PAD_TOKEN|>",
+    "<|quad_start|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|im_start|>",
+    "</tool_call>",
+    "<|video_pad|>",
+    "<tool_call>",
+    "<|im_end|>",
+    "<|vision_",
+    "<|fim_",
+]
+
+
+def get_llm():
+    cache_llm = ChatLlamaCpp(
+        temperature=0.0,
+        model_path=local_model,
+        n_ctx=10000,
+        n_gpu_layers=0,
+        n_batch=1024,
+        max_tokens=500,
+        n_threads=multiprocessing.cpu_count() - 1,
+        top_p=0.97,
+        verbose=False,
+        stop=stop,
+    )
+    return cache_llm
+
+
+llm = get_llm()
+
+
+db_schema = """### **customers**
+|       | customer_id   |   customer_zip_code_prefix | customer_city   | customer_state   |
+|------:|:--------------|---------------------------:|:----------------|:-----------------|
+| 21921 | 0tgYlOTGgpO6  |                      79230 | russas          | CE               |
+|  9748 | jGhRQF3CIew4  |                      81460 | joao monlevade  | MG               |
+| 22679 | 1UutQTIhBvcP  |                      94480 | pelotas         | RS               |
+
+Rows: 38279, Columns: 4
+
+---
+
+### **order_items**
+|       | order_id     | product_id   | seller_id    |   price |   shipping_charges |
+|------:|:-------------|:-------------|:-------------|--------:|-------------------:|
+| 19729 | PDEzZdebLSn3 | aBpYjaBcwz6e | bzfcwRPnZzVO |   55.83 |              27.8  |
+|  6001 | R7bIPjjYqlHP | ZM2JJXV5m9hl | Ivbw25fb5t2Z |  100    |              42.05 |
+|   282 | Biqo21nETaMO | XqmdGKRbTetH | P2nCHWuo0HC0 |  113.49 |              91.32 |
+
+Rows: 38279, Columns: 5
+
+---
+
+### **orders**
+
+|       | order_id     | customer_id   | order_purchase_timestamp   | order_approved_at   |
+|------:|:-------------|:--------------|:---------------------------|:--------------------|
+|  7294 | PMqwQc01iDTJ | c9ueC6k6V5WS  | 2018-06-19 21:23:48        | 2018-06-20 08:38:30 |
+| 13800 | P4l8R2Qat5n7 | ovKkGaXi5TmN  | 2018-01-05 08:26:03        | 2018-01-05 08:47:20 |
+| 17679 | NxIseZjAQCdC | o9qzmUQVJOxA  | 2018-01-28 23:46:53        | 2018-01-28 23:58:31 |
+
+Rows: 38279, Columns: 4
+
+---
+
+### **payments**
+|       | order_id     |   payment_sequential | payment_type   |   payment_installments |   payment_value |
+|------:|:-------------|---------------------:|:---------------|-----------------------:|----------------:|
+| 35526 | cQXl0pQtiMad |                    1 | wallet         |                      1 |          172.58 |
+| 35799 | olImD2k316Gz |                    1 | credit_card    |                      3 |           16.78 |
+| 13278 | G9MJYXXtPZSz |                    1 | credit_card    |                     10 |          221.86 |
+
+Rows: 38279, Columns: 5
+
+---
+
+### **products**
+|       | product_id   | product_category_name   |   product_weight_g |   product_length_cm |   product_height_cm |   product_width_cm |
+|------:|:-------------|:------------------------|-------------------:|--------------------:|--------------------:|-------------------:|
+| 18191 | hpiXwRzTkhkL | bed_bath_table          |               1150 |                  40 |                   9 |                 50 |
+|  2202 | iPoRkE7dkmlc | toys                    |              15800 |                  38 |                  62 |                 57 |
+| 27442 | hrjNaMt3Wyo5 | toys                    |               1850 |                  37 |                  22 |                 40 |
+
+Rows: 38279, Columns: 6
+"""
+
+# Improved SQL generation prompt
+sql_system_prompt = """You are a highly skilled natural language to SQL translator. Your goal is to generate accurate SQL queries based on the provided database schema. You must only return the SQL query and no other text or explanations.
+
+DATABASE SCHEMA:
+{db_schema}
+"""
+sql_chat_template = """
+
+Translate the following natural language question into an accurate SQL query. Return only the SQL query.
+
+QUESTION: {question}
+
+### assistant:
+
+"""
+
+# Improved prompt for classifying the question
+classification_system_prompt = """You are an expert at classifying user questions as requiring a SQL query or being generic based on the provided database schema. Your response should be ONLY 'SQL' or 'GENERIC'.
+
+A question requires a SQL query if it asks for specific data that can be retrieved from the tables in the schema. A question is generic if it asks for explanations, definitions, or information not directly retrievable through a SQL query on the given schema.
+
+Consider the following database schema:
+{db_schema}
+
+Here are some examples:
+
+Question: What are the names of all customers?
+Response: SQL
+
+Question: Tell me about the sales table.
+Response: GENERIC
+
+Question: How much did product 'Product A' sell for?
+Response: SQL
+
+Question: What is a primary key?
+Response: GENERIC
+"""
+classification_chat_template = """
+
+Determine if the following question requires a SQL query based on the database schema. Respond with 'SQL' or 'GENERIC'.
+
+QUESTION: {question}
+
+### assistant:
+"""
+
+
+def classify_question(question: str, use_default_schema: bool = True):
+    classification_system_prompt_local = classification_system_prompt  # Initialize here
+    if use_default_schema:
+        classification_system_prompt_local = classification_system_prompt_local.format(
+            db_schema=db_schema
+        )
+    else:
+        uploaded_schema = st.session_state.uploaded_df_schema
+        classification_system_prompt_local = classification_system_prompt_local.format(
+            db_schema=uploaded_schema
+        )
+    classification_messages = [
+        SystemMessage(content=classification_system_prompt_local),
+        HumanMessage(content=classification_chat_template.format(question=question)),
+    ]
+    response = llm.invoke(classification_messages)
+    return response.content.strip().upper()
+
+
+def generate_llm_response(prompt: str, use_default_schema: bool = True):
+    question_type = classify_question(prompt, use_default_schema)
+    chosen_schema = None
+    if use_default_schema:
+        chosen_schema = db_schema
+        sql_system_prompt_local = sql_system_prompt.format(db_schema=chosen_schema)
+    else:
+        uploaded_schema = st.session_state.uploaded_df_schema
+        chosen_schema = uploaded_schema
+        sql_system_prompt_local = sql_system_prompt.format(db_schema=chosen_schema)
+
+    # Retrieve the chat history from the session state
+    chat_history = st.session_state.get("chat_history", [])
+
+    if "SQL" in question_type:
+        print("SQL question detected")
+        st.toast("Detected Task: SQL Query Generation", icon="🚨")
+        formatted_prompt = sql_chat_template.format(question=prompt)
+
+        # Create the messages list, including the system prompt and the chat history
+        messages_for_llm = [SystemMessage(content=sql_system_prompt_local)]
+        for message in chat_history:
+            if isinstance(message, HumanMessage):
+                messages_for_llm.append(HumanMessage(content=message.content))
+            elif isinstance(message, AIMessage):
+                # Only include the assistant's text response, not the additional kwargs
+                messages_for_llm.append(AIMessage(content=message.content))
+        messages_for_llm.append(HumanMessage(content=formatted_prompt))
+
+        full_response = ""
+        for chunk in llm.stream(messages_for_llm):
+            full_response += chunk.content
+            yield f"<sql>\n```sql\n{full_response.strip()}\n```\n</sql>"
+    elif "GENERIC" in question_type:
+        print("Generic question detected")
+        st.toast("Detected Task: Generic QA", icon="🚨")
+        generic_prompt = f"Answer the following question related to SQL or coding:\n\nQUESTION: {prompt}\n\n### assistant:"
+
+        # Create the messages list, including the system prompt and the chat history
+        messages_for_generic = [
+            SystemMessage(
+                content=f"You are a helpful assistant finetuned from Qwen2.5-coder:3B-Instruct for answering questions about SQL.\nYou have a database with the Database Schema:\n{chosen_schema}.\n"
+            )
+        ]
+        for message in chat_history:
+            if isinstance(message, HumanMessage):
+                messages_for_generic.append(HumanMessage(content=message.content))
+            elif isinstance(message, AIMessage):
+                # Only include the assistant's text response, not the additional kwargs
+                messages_for_generic.append(AIMessage(content=message.content))
+        messages_for_generic.append(HumanMessage(content=generic_prompt))
+
+        generic_response = ""
+        for chunk in llm.stream(messages_for_generic):
+            generic_response += chunk.content
+            yield generic_response
+    else:
+        yield "I am sorry, I am small language model fine-tuned specifically to answer questions that can be solved using SQL. I won't be able to answer this question."

utils/sql_utils.py

@@ -0,0 +1,64 @@
+import re
+import streamlit as st
+import pandas as pd
+
+def extract_sql_command(text):
+    """
+    Extracts the SQL command enclosed within ```sql ``` delimiters from a given string.
+
+    Args:
+        text: The input string containing the SQL command.
+
+    Returns:
+        The extracted SQL command as a string, or None if no SQL command is found.
+    """
+    pattern = r"```sql\s*([\s\S]*?)\s*```"
+    match = re.search(pattern, text)
+    if match:
+        return match.group(1).strip()
+    else:
+        return None
+
+def create_schema(dataframes):
+    schema = ""
+    for df_name, df in dataframes.items():
+        schema += f"### {df_name}\n"
+        schema += df.head(3).to_markdown(index=False)
+        schema += "\n\nRows: " + str(df.shape[0]) + ", Columns: " + str(df.shape[1]) + "\n\n---\n\n"
+    return schema
+
+
+@st.cache_resource
+def load_defaultdb_schema_text():
+    with open("static/database_scema.txt", "r", encoding="utf-8") as file:
+        return file.read()
+
+
+@st.cache_resource
+def load_defaultdb_queries():
+    with open("static/default_questions.txt", "r", encoding="utf-8") as file:
+        return file.read()
+
+
+@st.cache_data
+def convert_df(df):
+    # IMPORTANT: Cache the conversion to prevent computation on every rerun
+    return df.to_csv().encode("utf-8")
+
+
+# Load CSV files into pandas default_dfs
+@st.cache_resource
+def load_data():
+    # text-to-sql-streamlit\static\df_Customers.csv
+    df_customers = pd.read_csv("static/df_Customers.csv")
+    df_order_items = pd.read_csv("static/df_OrderItems.csv")
+    df_orders = pd.read_csv("static/df_Orders.csv")
+    df_payments = pd.read_csv("static/df_Payments.csv")
+    df_products = pd.read_csv("static/df_Products.csv")
+    return {
+        "customers": df_customers,
+        "order_items": df_order_items,
+        "orders": df_orders,
+        "payments": df_payments,
+        "products": df_products,
+    }

🤖SQL_Agent.py

@@ -0,0 +1,310 @@
+import re
+import streamlit as st
+from langchain_core.messages import HumanMessage, AIMessage
+from utils.llm_logic import generate_llm_response
+from utils.sql_utils import (
+    extract_sql_command,
+    load_defaultdb_schema_text,
+    load_defaultdb_queries,
+    load_data,
+)
+from utils.handle_sql_commands import execute_sql_duckdb
+
+
+st.set_page_config(
+    page_title="Text-to-SQL Agent",
+    page_icon="🤖",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+default_db_questions = {
+    "easy": [
+        "Retrieve all customer IDs and their corresponding cities from the `customers` table.",
+        "List all products along with their category names from the `products` table.",
+        "Fetch the order IDs and their purchase timestamps from the `orders` table.",
+        "Display the distinct payment types available in the `payments` table.",
+        "Find the total number of rows in the `customers` table.",
+    ],
+    "medium": [
+        "Retrieve the total payment value for each order from the `payments` table, grouped by `order_id`.",
+        "Find all orders where the total shipping charges (sum of `shipping_charges`) exceed 100.",
+        "List the names of cities and the number of customers in each city, sorted in descending order of the number of customers.",
+    ],
+    "hard": [
+        "Write a query to find the total revenue (sum of `price` + `shipping_charges`) generated for each product category in the `order_items` table, joined with the `products` table.",
+        "Identify the top 5 products with the highest total sales value (sum of `price`) across all orders.",
+    ],
+}
+
+
+default_dfs = load_data()
+selected_df = default_dfs
+use_default_schema = True
+
+st.markdown(
+    """
+<style>
+    /* Base styles for both themes */
+    .stPageLink {
+        background-image: linear-gradient(to right, #007BFF, #6610F2); /* Gradient background */
+        color: white !important; /* Ensure text is readable on the gradient */
+        padding: 12px 20px !important; /* Slightly larger padding */
+        border-radius: 8px !important; /* More rounded corners */
+        border: none !important; /* Remove default border */
+        text-decoration: none !important;
+        font-weight: 500 !important; /* Slightly lighter font weight */
+        transition: transform 0.2s ease-in-out, box-shadow 0.2s ease-in-out; /* Smooth transitions */
+        box-shadow: 0 2px 5px rgba(0, 0, 0, 0.15); /* Subtle shadow for depth */
+        display: inline-flex;
+        align-items: center;
+        justify-content: center;
+    }
+
+    .stPageLink:hover {
+        transform: scale(1.03); /* Slight scale up on hover */
+        box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); /* Increased shadow on hover */
+    }
+
+    .stPageLink span { /* Style the label text */
+        margin-left: 5px; /* Space between icon and text */
+    }
+
+    /* Dark theme adjustments (optional, if needed for better contrast) */
+    /* Consider using Streamlit's theme variables if possible for a more robust solution */
+    /* For simplicity, this example uses fixed colors that should work reasonably well */
+    /* [data-theme="dark"] .stPageLink {
+    }
+
+    [data-theme="dark"] .stPageLink:hover {
+    } */
+</style>
+""",
+    unsafe_allow_html=True,
+)
+
+with st.popover("Click here to see Database Schema", use_container_width=True):
+    uploaded_df_schema = st.session_state.get("uploaded_df_schema", False)
+
+    choice = st.segmented_control(
+        "Choose",
+        ["Default DB", "Uploaded Files"],
+        label_visibility="collapsed",
+        disabled=uploaded_df_schema == False,
+        default="Default DB" if uploaded_df_schema == False else "Uploaded Files",
+    )
+
+    if uploaded_df_schema is False:
+        st.markdown(
+            """> You can also upload your own files, to get your schemas. You can then use those schemas to cross-check our answers with ChatGpt/Gemini/Claude (Preferred if the Question is very Complex). You can run the queries directly with our Manual SQL Executer😊.
+- Ask Questions
+- Run Queries: automatic + manual
+- Download Results """
+        )
+        st.page_link(
+            page="pages/3 📂File Upload for SQL.py",
+            label="Upload your own CSV or Excel files",
+            icon="📜",
+        )
+        schema = load_defaultdb_schema_text()
+        st.markdown(schema, unsafe_allow_html=True)
+    elif choice == "Default DB":
+        schema = load_defaultdb_schema_text()
+        st.markdown(schema, unsafe_allow_html=True)
+    else:
+        pretty_schema, markdown = st.tabs(["Schema", "Copy Schema in Markdown"])
+        with pretty_schema:
+            st.info(
+                "You can copy this schema, and give it to any state of the art LLM models like (Gemini /ChatGPT /Claude etc) to cross check your answers.\n You can run the queries directly here, by using ***Manual Query Executer*** in the sidebar and download your results 😊",
+                icon="ℹ️",
+            )
+            st.markdown(uploaded_df_schema, unsafe_allow_html=True)
+        with markdown:
+            st.info(
+                "You can copy this schema, and give it to any state of the art LLM models like (Gemini /ChatGPT /Claude etc) to cross check your answers.\n You can run the queries directly here, by using ***Manual Query Executer*** in the sidebar and download your results 😊",
+                icon="ℹ️",
+            )
+            st.markdown(f"```\n{uploaded_df_schema}\n```")
+
+
+col1, col2 = st.columns([2, 1], vertical_alignment="bottom")
+with col1:
+    st.header("Natural Language to SQL Query Agent🤖")
+
+with col2:
+    st.caption("> ***Execute on the Go!*** 🚀   In-Built DuckDB Execution Engine")
+
+st.caption(
+    "This is a Qwen2.5-Coder-3B model fine-tuned for SQL queries integrated with langchain for Agentic Workflow. To see the Fine-Tuning code - [click here](https://www.kaggle.com/code/debopamchowdhury/qwen-2-5coder-3b-instruct-finetuning)."
+)
+
+
+col1, col2 = st.columns([2, 1], vertical_alignment="bottom")
+with col1:
+    # Button to refresh the conversation
+    if st.button("Start New Conversation", type="primary"):
+        st.session_state.chat_history = []
+        st.session_state.conversation_turns = 0
+        st.rerun()
+with col2:
+    disabled_selection = True
+    if (
+        "uploaded_dataframes" in st.session_state
+    ) and st.session_state.uploaded_dataframes:
+        disabled_selection = False
+    options = ["default_db", "uploaded_files"]
+    selected = st.segmented_control(
+        "Choose",
+        options,
+        selection_mode="single",
+        disabled=disabled_selection,
+        label_visibility="collapsed",
+        default="default_db" if disabled_selection else "uploaded_files",
+    )
+    if not disabled_selection:
+        if selected == "uploaded_files":
+            selected_df = st.session_state.uploaded_dataframes
+            print(selected_df)
+            use_default_schema = False
+        else:
+            selected_df = default_dfs
+            print(selected_df)
+            use_default_schema = True
+if selected_df == default_dfs:
+    with st.popover("Default Database Queries 📚 - Trial"):
+        default_db_questions = load_defaultdb_queries()
+        st.markdown(default_db_questions)
+
+# Initialize chat history in session state
+if "chat_history" not in st.session_state:
+    st.session_state.chat_history = []
+
+# Initialize conversation turn counter
+if "conversation_turns" not in st.session_state:
+    st.session_state.conversation_turns = 0
+
+# Set the maximum number of conversation turns
+MAX_TURNS = 5
+
+# Display existing chat messages
+for message in st.session_state.chat_history:
+    with st.chat_message(message.type):
+        st.markdown(message.content)
+        if (
+            isinstance(message, AIMessage)
+            and "response_df" in message.additional_kwargs
+            and message.additional_kwargs["response_df"] is not None
+            and not message.additional_kwargs["response_df"].empty
+        ):
+            with st.expander("View SQL-Query Execution Result"):
+                df = message.additional_kwargs["response_df"]
+                # download_csv = convert_df(df)
+                # st.download_button(
+                #     label="Download data as CSV",
+                #     data=download_csv,
+                #     file_name="query_results.csv",
+                #     mime="text/csv",
+                # )
+                # renderer = StreamlitRenderer(
+                #     df,
+                #     spec_io_mode="rw",
+                #     default_tab="data",
+                #     appearance="dark",
+                #     kernel_computation=True,
+                # )
+                # renderer.explorer(default_tab="data")
+                st.dataframe(df)
+                st.info(f"Rows x Columns: {df.shape[0]} x {df.shape[1]}")
+                st.subheader("Data Description:")
+                st.markdown(df.describe().T.to_markdown())
+                st.subheader("Data Types:")
+                st.write(df.dtypes)
+
+# Get user input only if the conversation turn limit is not reached
+if st.session_state.conversation_turns < MAX_TURNS:
+    if prompt := st.chat_input("Ask me a SQL query question"):
+        # Add user message to chat history in session state
+        st.session_state.chat_history.append(HumanMessage(content=prompt))
+        # Display user message in chat
+        with st.chat_message("user"):
+            st.markdown(prompt)
+
+        duckdb_result = None
+        # Get assistant response with streaming
+        with st.chat_message("assistant"):
+            message_placeholder = st.empty()
+            full_response = ""
+            with st.spinner(
+                "I know it is taking a lot of time. To run the model I'm using `Free` small vCPUs provided by `HuggingFace Spaces` for deployment. Thank you so much for your patience😊"
+            ):
+                for response_so_far in generate_llm_response(
+                    prompt, use_default_schema
+                ):
+                    # Remove <sql> and </sql> tags for streaming display
+                    streaming_response = response_so_far.replace("<sql>", "").replace(
+                        "</sql>", ""
+                    )
+                    # Remove duplicate ```sql tags with or without space for streaming display
+                    streaming_response = re.sub(
+                        r"```sql\s*```sql", "```sql", streaming_response
+                    )
+                    message_placeholder.markdown(streaming_response + "▌")
+                    full_response = response_so_far
+
+            # Remove <sql> and </sql> tags from the full response
+            full_response = full_response.replace("<sql>", "").replace("</sql>", "")
+            # Remove duplicate ```sql tags with or without space from the full response
+            full_response = re.sub(r"```sql\s*```sql", "```sql", full_response)
+            # Remove trailing duplicate ``` tags from the full response
+            full_response = re.sub(r"[\s\n]*`+$", "```", full_response)
+            message_placeholder.markdown(full_response)
+            # st.text(extract_sql_command(full_response))
+
+            sql_command = extract_sql_command(full_response)
+            # dataframe_html = None
+            if sql_command:
+                # st.text("Extracted SQL Command:")
+                # st.code(sql_command, language="sql")
+                duckdb_result = execute_sql_duckdb(sql_command, selected_df)
+                if duckdb_result is not None:
+                    st.text("Query Execution Result:")
+                    with st.expander("View Result"):
+                        # st.dataframe(duckdb_result)
+                        st.dataframe(duckdb_result)
+                        st.info(
+                            f"Rows x Columns: {duckdb_result.shape[0]} x {duckdb_result.shape[1]}"
+                        )
+                        st.subheader("Data Description:")
+                        st.markdown(duckdb_result.describe().T.to_markdown())
+                        st.subheader("Data Types:")
+                        st.write(duckdb_result.dtypes)
+                        # renderer = StreamlitRenderer(
+                        #     duckdb_result,
+                        #     spec_io_mode="rw",
+                        #     default_tab="data",
+                        #     appearance="dark",
+                        #     kernel_computation=True,
+                        # )
+                        # renderer.explorer(default_tab="data")
+
+            else:
+                # st.warning("No SQL command found in the response.")
+                pass
+
+        # Add assistant response to chat history in session state
+        st.session_state.chat_history.append(
+            AIMessage(
+                content=full_response,
+                additional_kwargs={"response_df": duckdb_result},
+            )
+        )
+
+        # Increment the conversation turn counter
+        st.session_state.conversation_turns += 1
+else:
+    st.warning(
+        "Maximum number of questions reached. Please click 'Start New Conversation' to continue."
+    )
+    st.chat_input(
+        "Ask me a SQL query question", disabled=True
+    )  # Disable the input field