Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+MiniGPTv2.pdf filter=lfs diff=lfs merge=lfs -text
+examples_v2/cockdial.png filter=lfs diff=lfs merge=lfs -text
+examples_v2/float.png filter=lfs diff=lfs merge=lfs -text
+figs/demo.png filter=lfs diff=lfs merge=lfs -text
+figs/minigpt2_demo.png filter=lfs diff=lfs merge=lfs -text
+figs/online_demo.png filter=lfs diff=lfs merge=lfs -text
+figs/overview.png filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.gitignore

@@ -0,0 +1,184 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+wandb/
+jobs/logs/
+*.out
+*ipynb
+.history/
+*.json
+*.sh
+.ipynb_common
+logs/
+results/
+prompts/
+output/
+ckpt/
+divide_vqa.py
+jobs/
+
+*.slurm
+slurm*
+sbatch_generate*
+eval_data/
+dataset/Evaluation.md
+jupyter_notebook.slurm

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+https://discord.gg/2aNvvYVv.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

LICENSE.md

@@ -0,0 +1,14 @@
+BSD 3-Clause License
+
+Copyright 2023 Deyao Zhu
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LICENSE_Lavis.md

@@ -0,0 +1,14 @@
+BSD 3-Clause License
+
+Copyright (c) 2022 Salesforce, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MiniGPT4_Train.md

@@ -0,0 +1,41 @@
+## Training of MiniGPT-4
+
+The training of MiniGPT-4 contains two alignment stages.
+
+**1. First pretraining stage**
+
+In the first pretrained stage, the model is trained using image-text pairs from Laion and CC datasets
+to align the vision and language model. To download and prepare the datasets, please check 
+our [first stage dataset preparation instruction](dataset/README_1_STAGE.md). 
+After the first stage, the visual features are mapped and can be understood by the language
+model.
+To launch the first stage training, run the following command. In our experiments, we use 4 A100. 
+You can change the save path in the config file 
+[train_configs/minigpt4_stage1_pretrain.yaml](train_configs/minigpt4_stage1_pretrain.yaml)
+
+```bash
+torchrun --nproc-per-node NUM_GPU train.py --cfg-path train_configs/minigpt4_stage1_pretrain.yaml
+```
+
+A MiniGPT-4 checkpoint with only stage one training can be downloaded 
+[here (13B)](https://drive.google.com/file/d/1u9FRRBB3VovP1HxCAlpD9Lw4t4P6-Yq8/view?usp=share_link) or [here (7B)](https://drive.google.com/file/d/1HihQtCEXUyBM1i9DQbaK934wW3TZi-h5/view?usp=share_link).
+Compared to the model after stage two, this checkpoint generate incomplete and repeated sentences frequently.
+
+
+**2. Second finetuning stage**
+
+In the second stage, we use a small high quality image-text pair dataset created by ourselves
+and convert it to a conversation format to further align MiniGPT-4.
+To download and prepare our second stage dataset, please check our 
+[second stage dataset preparation instruction](dataset/README_2_STAGE.md).
+To launch the second stage alignment, 
+first specify the path to the checkpoint file trained in stage 1 in 
+[train_configs/minigpt4_stage1_pretrain.yaml](train_configs/minigpt4_stage2_finetune.yaml).
+You can also specify the output path there. 
+Then, run the following command. In our experiments, we use 1 A100.
+
+```bash
+torchrun --nproc-per-node NUM_GPU train.py --cfg-path train_configs/minigpt4_stage2_finetune.yaml
+```
+
+After the second stage alignment, MiniGPT-4 is able to talk about the image coherently and user-friendly. 

MiniGPTv2.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:429b0f5e3d70828fd691ef4ffb90c6efa094a8454bf03f8ec00b10fcd443f346
+size 4357853

MiniGPTv2_Train.md

@@ -0,0 +1,24 @@
+## Finetune of MiniGPT-4
+
+
+You firstly need to prepare the dataset. you can follow this step to prepare the dataset.
+our [dataset preparation](dataset/README_MINIGPTv2_FINETUNE.md). 
+
+In the train_configs/minigptv2_finetune.yaml, you need to set up the following paths:
+
+llama_model checkpoint path: "/path/to/llama_checkpoint"
+
+ckpt: "/path/to/pretrained_checkpoint"
+
+ckpt save path: "/path/to/save_checkpoint"
+
+For ckpt, you may load from our pretrained model checkpoints:
+| MiniGPT-v2 (after stage-2) | MiniGPT-v2 (after stage-3) | MiniGPT-v2 (online developing demo) | 
+|------------------------------|------------------------------|------------------------------|
+| [Download](https://drive.google.com/file/d/1Vi_E7ZtZXRAQcyz4f8E6LtLh2UXABCmu/view?usp=sharing) |[Download](https://drive.google.com/file/d/1HkoUUrjzFGn33cSiUkI-KcT-zysCynAz/view?usp=sharing) | [Download](https://drive.google.com/file/d/1aVbfW7nkCSYx99_vCRyP1sOlQiWVSnAl/view?usp=sharing) |
+
+
+```bash
+torchrun --nproc-per-node NUM_GPU train.py --cfg-path train_configs/minigptv2_finetune.yaml
+```
+

README.md

@@ -1,12 +1,212 @@
 ---
-title: MiniGPT 4
-emoji: 🏃
-colorFrom: blue
-colorTo: indigo
+title: MiniGPT-4
+app_file: demo_v2.py
 sdk: gradio
-sdk_version: 4.38.1
-app_file: app.py
-pinned: false
+sdk_version: 3.47.1
 ---
+# MiniGPT-V
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+<font size='5'>**MiniGPT-v2: Large Language Model as a Unified Interface for Vision-Language Multi-task Learning**</font>
+
+Jun Chen, Deyao Zhu, Xiaoqian Shen, Xiang Li, Zechun Liu, Pengchuan Zhang, Raghuraman Krishnamoorthi, Vikas Chandra, Yunyang Xiong☨, Mohamed Elhoseiny☨
+
+☨equal last author
+
+<a href='https://minigpt-v2.github.io'><img src='https://img.shields.io/badge/Project-Page-Green'></a> <a href='https://arxiv.org/abs/2310.09478.pdf'><img src='https://img.shields.io/badge/Paper-Arxiv-red'></a>  <a href='https://huggingface.co/spaces/Vision-CAIR/MiniGPT-v2'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue'> <a href='https://minigpt-v2.github.io'><img src='https://img.shields.io/badge/Gradio-Demo-blue'></a> [![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://www.youtube.com/watch?v=atFCwV2hSY4)
+
+
+<font size='5'> **MiniGPT-4: Enhancing Vision-language Understanding with Advanced Large Language Models**</font>
+
+Deyao Zhu*, Jun Chen*, Xiaoqian Shen, Xiang Li, Mohamed Elhoseiny
+
+*equal contribution
+
+<a href='https://minigpt-4.github.io'><img src='https://img.shields.io/badge/Project-Page-Green'></a>  <a href='https://arxiv.org/abs/2304.10592'><img src='https://img.shields.io/badge/Paper-Arxiv-red'></a> <a href='https://huggingface.co/spaces/Vision-CAIR/minigpt4'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue'></a> <a href='https://huggingface.co/Vision-CAIR/MiniGPT-4'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-blue'></a> [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1OK4kYsZphwt5DXchKkzMBjYF6jnkqh4R?usp=sharing) [![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://www.youtube.com/watch?v=__tftoxpBAw&feature=youtu.be)
+
+*King Abdullah University of Science and Technology*
+
+## 💡 Get help - [Q&A](https://github.com/Vision-CAIR/MiniGPT-4/discussions/categories/q-a) or [Discord 💬](https://discord.gg/5WdJkjbAeE)
+
+<font size='4'> **Example Community Efforts Built on Top of MiniGPT-4 ** </font> 
+  
+* <a href='https://github.com/waltonfuture/InstructionGPT-4?tab=readme-ov-file'><img src='https://img.shields.io/badge/Project-Page-Green'></a> **InstructionGPT-4**: A 200-Instruction Paradigm for Fine-Tuning MiniGPT-4 Lai Wei, Zihao Jiang, Weiran Huang, Lichao Sun, Arxiv, 2023
+
+* <a href='https://openaccess.thecvf.com/content/ICCV2023W/CLVL/papers/Aubakirova_PatFig_Generating_Short_and_Long_Captions_for_Patent_Figures_ICCVW_2023_paper.pdf'><img src='https://img.shields.io/badge/Project-Page-Green'></a> **PatFig**: Generating Short and Long Captions for Patent Figures.", Aubakirova, Dana, Kim Gerdes, and Lufei Liu, ICCVW, 2023 
+
+
+* <a href='https://github.com/JoshuaChou2018/SkinGPT-4'><img src='https://img.shields.io/badge/Project-Page-Green'></a> **SkinGPT-4**: An Interactive Dermatology Diagnostic System with Visual Large Language Model, Juexiao Zhou and Xiaonan He and Liyuan Sun and Jiannan Xu and Xiuying Chen and Yuetan Chu and Longxi Zhou and Xingyu Liao and Bin Zhang and Xin Gao,  Arxiv, 2023 
+
+
+* <a href='https://huggingface.co/Tyrannosaurus/ArtGPT-4'><img src='https://img.shields.io/badge/Project-Page-Green'></a> **ArtGPT-4**: Artistic Vision-Language Understanding with Adapter-enhanced MiniGPT-4.",  Yuan, Zhengqing, Huiwen Xue, Xinyi Wang, Yongming Liu, Zhuanzhe Zhao, and Kun Wang, Arxiv, 2023 
+
+
+</font>
+
+## News
+[Oct.31 2023] We release the evaluation code of our MiniGPT-v2.  
+
+[Oct.24 2023] We release the finetuning code of our MiniGPT-v2.
+
+[Oct.13 2023] Breaking! We release the first major update with our MiniGPT-v2
+
+[Aug.28 2023] We now provide a llama 2 version of MiniGPT-4
+
+## Online Demo
+
+Click the image to chat with MiniGPT-v2 around your images
+[![demo](figs/minigpt2_demo.png)](https://minigpt-v2.github.io/)
+
+Click the image to chat with MiniGPT-4 around your images
+[![demo](figs/online_demo.png)](https://minigpt-4.github.io)
+
+
+## MiniGPT-v2 Examples
+
+![MiniGPT-v2 demos](figs/demo.png)
+
+
+
+## MiniGPT-4 Examples
+  |   |   |
+:-------------------------:|:-------------------------:
+![find wild](figs/examples/wop_2.png) |  ![write story](figs/examples/ad_2.png)
+![solve problem](figs/examples/fix_1.png)  |  ![write Poem](figs/examples/rhyme_1.png)
+
+More examples can be found in the [project page](https://minigpt-4.github.io).
+
+
+
+## Getting Started
+### Installation
+
+**1. Prepare the code and the environment**
+
+Git clone our repository, creating a python environment and activate it via the following command
+
+```bash
+git clone https://github.com/Vision-CAIR/MiniGPT-4.git
+cd MiniGPT-4
+conda env create -f environment.yml
+conda activate minigptv
+```
+
+
+**2. Prepare the pretrained LLM weights**
+
+**MiniGPT-v2** is based on Llama2 Chat 7B. For **MiniGPT-4**, we have both Vicuna V0 and Llama 2 version.
+Download the corresponding LLM weights from the following huggingface space via clone the repository using git-lfs.
+
+|                            Llama 2 Chat 7B                             |                                           Vicuna V0 13B                                           |                                          Vicuna V0 7B                                          |
+:------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------:
+[Download](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/tree/main) | [Downlad](https://huggingface.co/Vision-CAIR/vicuna/tree/main) | [Download](https://huggingface.co/Vision-CAIR/vicuna-7b/tree/main) 
+
+
+Then, set the variable *llama_model* in the model config file to the LLM weight path.
+
+* For MiniGPT-v2, set the LLM path 
+[here](minigpt4/configs/models/minigpt_v2.yaml#L15) at Line 14.
+
+* For MiniGPT-4 (Llama2), set the LLM path 
+[here](minigpt4/configs/models/minigpt4_llama2.yaml#L15) at Line 15.
+
+* For MiniGPT-4 (Vicuna), set the LLM path 
+[here](minigpt4/configs/models/minigpt4_vicuna0.yaml#L18) at Line 18
+
+**3. Prepare the pretrained model checkpoints**
+
+Download the pretrained model checkpoints
+
+
+| MiniGPT-v2 (after stage-2) | MiniGPT-v2 (after stage-3) | MiniGPT-v2 (online developing demo)| 
+|------------------------------|------------------------------|------------------------------|
+| [Download](https://drive.google.com/file/d/1Vi_E7ZtZXRAQcyz4f8E6LtLh2UXABCmu/view?usp=sharing) |[Download](https://drive.google.com/file/d/1HkoUUrjzFGn33cSiUkI-KcT-zysCynAz/view?usp=sharing) | [Download](https://drive.google.com/file/d/1aVbfW7nkCSYx99_vCRyP1sOlQiWVSnAl/view?usp=sharing) |
+
+
+For **MiniGPT-v2**, set the path to the pretrained checkpoint in the evaluation config file 
+in [eval_configs/minigptv2_eval.yaml](eval_configs/minigptv2_eval.yaml#L10) at Line 8.
+
+
+
+| MiniGPT-4 (Vicuna 13B) | MiniGPT-4 (Vicuna 7B) | MiniGPT-4 (LLaMA-2 Chat 7B) |
+|----------------------------|---------------------------|---------------------------------|
+| [Download](https://drive.google.com/file/d/1a4zLvaiDBr-36pasffmgpvH5P7CKmpze/view?usp=share_link) | [Download](https://drive.google.com/file/d/1RY9jV0dyqLX-o38LrumkKRh6Jtaop58R/view?usp=sharing) | [Download](https://drive.google.com/file/d/11nAPjEok8eAGGEG1N2vXo3kBLCg0WgUk/view?usp=sharing) |
+
+For **MiniGPT-4**, set the path to the pretrained checkpoint in the evaluation config file 
+in [eval_configs/minigpt4_eval.yaml](eval_configs/minigpt4_eval.yaml#L10) at Line 8 for Vicuna version or [eval_configs/minigpt4_llama2_eval.yaml](eval_configs/minigpt4_llama2_eval.yaml#L10) for LLama2 version.   
+
+
+
+### Launching Demo Locally
+
+For MiniGPT-v2, run
+```
+python demo_v2.py --cfg-path eval_configs/minigptv2_eval.yaml  --gpu-id 0
+```
+
+For MiniGPT-4 (Vicuna version), run
+
+```
+python demo.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0
+```
+
+For MiniGPT-4 (Llama2 version), run
+
+```
+python demo.py --cfg-path eval_configs/minigpt4_llama2_eval.yaml  --gpu-id 0
+```
+
+
+To save GPU memory, LLMs loads as 8 bit by default, with a beam search width of 1. 
+This configuration requires about 23G GPU memory for 13B LLM and 11.5G GPU memory for 7B LLM. 
+For more powerful GPUs, you can run the model
+in 16 bit by setting `low_resource` to `False` in the relevant config file:
+
+* MiniGPT-v2: [minigptv2_eval.yaml](eval_configs/minigptv2_eval.yaml#6) 
+* MiniGPT-4 (Llama2): [minigpt4_llama2_eval.yaml](eval_configs/minigpt4_llama2_eval.yaml#6)
+* MiniGPT-4 (Vicuna): [minigpt4_eval.yaml](eval_configs/minigpt4_eval.yaml#6)
+
+Thanks [@WangRongsheng](https://github.com/WangRongsheng), you can also run MiniGPT-4 on [Colab](https://colab.research.google.com/drive/1OK4kYsZphwt5DXchKkzMBjYF6jnkqh4R?usp=sharing)
+
+
+### Training
+For training details of MiniGPT-4, check [here](MiniGPT4_Train.md).
+
+For finetuning details of MiniGPT-v2, check [here](MiniGPTv2_Train.md)
+
+
+### Evaluation
+For finetuning details of MiniGPT-v2, check [here](eval_scripts/EVAL_README.md)  
+
+
+## Acknowledgement
+
++ [BLIP2](https://huggingface.co/docs/transformers/main/model_doc/blip-2) The model architecture of MiniGPT-4 follows BLIP-2. Don't forget to check this great open-source work if you don't know it before!
++ [Lavis](https://github.com/salesforce/LAVIS) This repository is built upon Lavis!
++ [Vicuna](https://github.com/lm-sys/FastChat) The fantastic language ability of Vicuna with only 13B parameters is just amazing. And it is open-source!
++ [LLaMA](https://github.com/facebookresearch/llama) The strong open-sourced LLaMA 2 language model.
+
+
+If you're using MiniGPT-4/MiniGPT-v2 in your research or applications, please cite using this BibTeX:
+```bibtex
+
+
+@article{chen2023minigptv2,
+      title={MiniGPT-v2: large language model as a unified interface for vision-language multi-task learning}, 
+      author={Chen, Jun and Zhu, Deyao and Shen, Xiaoqian and Li, Xiang and Liu, Zechu and Zhang, Pengchuan and Krishnamoorthi, Raghuraman and Chandra, Vikas and Xiong, Yunyang and Elhoseiny, Mohamed},
+      year={2023},
+      journal={arXiv preprint arXiv:2310.09478},
+}
+
+@article{zhu2023minigpt,
+  title={MiniGPT-4: Enhancing Vision-Language Understanding with Advanced Large Language Models},
+  author={Zhu, Deyao and Chen, Jun and Shen, Xiaoqian and Li, Xiang and Elhoseiny, Mohamed},
+  journal={arXiv preprint arXiv:2304.10592},
+  year={2023}
+}
+```
+
+
+## License
+This repository is under [BSD 3-Clause License](LICENSE.md).
+Many codes are based on [Lavis](https://github.com/salesforce/LAVIS) with 
+BSD 3-Clause License [here](LICENSE_Lavis.md).

SECURITY.md

@@ -0,0 +1,21 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 5.1.x   | :white_check_mark: |
+| 5.0.x   | :x:                |
+| 4.0.x   | :white_check_mark: |
+| < 4.0   | :x:                |
+
+## Reporting a Vulnerability
+
+Use this section to tell people how to report a vulnerability.
+
+Tell them where to go, how often they can expect to get an update on a
+reported vulnerability, what to expect if the vulnerability is accepted or
+declined, etc.

checkpoint_stage2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d23539a5d0f2e02539dde09de21c89bcb054bb989d0a2800b280abd9d7f57a0
+size 679804205

dataset/README_1_STAGE.md

@@ -0,0 +1,96 @@
+## Download the filtered Conceptual Captions, SBU, LAION datasets
+
+### Pre-training datasets download:
+We use the filtered synthetic captions prepared by BLIP. For more details about the dataset, please refer to [BLIP](https://github.com/salesforce/BLIP).
+
+It requires ~2.3T to store LAION and CC3M+CC12M+SBU datasets
+
+Image source | Filtered synthetic caption by ViT-L
+--- | :---:
+CC3M+CC12M+SBU | <a href="https://storage.googleapis.com/sfr-vision-language-research/BLIP/datasets/ccs_synthetic_filtered_large.json">Download</a>
+LAION115M |  <a href="https://storage.googleapis.com/sfr-vision-language-research/BLIP/datasets/laion_synthetic_filtered_large.json">Download</a>
+
+This will download two json files 
+```
+ccs_synthetic_filtered_large.json
+laion_synthetic_filtered_large.json
+```
+
+## prepare the data step-by-step
+
+
+### setup the dataset folder and move the annotation file to the data storage folder
+```
+export MINIGPT4_DATASET=/YOUR/PATH/FOR/LARGE/DATASET/
+mkdir ${MINIGPT4_DATASET}/cc_sbu
+mkdir ${MINIGPT4_DATASET}/laion
+mv ccs_synthetic_filtered_large.json ${MINIGPT4_DATASET}/cc_sbu
+mv laion_synthetic_filtered_large.json ${MINIGPT4_DATASET}/laion
+```
+
+### Convert the scripts to data storate folder
+```
+cp convert_cc_sbu.py ${MINIGPT4_DATASET}/cc_sbu
+cp download_cc_sbu.sh ${MINIGPT4_DATASET}/cc_sbu
+cp convert_laion.py ${MINIGPT4_DATASET}/laion
+cp download_laion.sh ${MINIGPT4_DATASET}/laion
+```
+
+
+### Convert the laion and cc_sbu annotation file format to be img2dataset format
+```
+cd ${MINIGPT4_DATASET}/cc_sbu
+python convert_cc_sbu.py
+
+cd ${MINIGPT4_DATASET}/laion
+python convert_laion.py
+```
+
+### Download the datasets with img2dataset
+```
+cd ${MINIGPT4_DATASET}/cc_sbu
+sh download_cc_sbu.sh
+cd ${MINIGPT4_DATASET}/laion
+sh download_laion.sh
+```
+
+
+The final dataset structure
+
+```
+.
+├── ${MINIGPT4_DATASET}
+│   ├── cc_sbu
+│       ├── convert_cc_sbu.py
+│       ├── download_cc_sbu.sh
+│       ├── ccs_synthetic_filtered_large.json
+│       ├── ccs_synthetic_filtered_large.tsv
+│       └── cc_sbu_dataset
+│           ├── 00000.tar
+│           ├── 00000.parquet
+│           ...
+│   ├── laion
+│       ├── convert_laion.py
+│       ├── download_laion.sh
+│       ├── laion_synthetic_filtered_large.json
+│       ├── laion_synthetic_filtered_large.tsv
+│       └── laion_dataset
+│           ├── 00000.tar
+│           ├── 00000.parquet
+│           ...
+...   
+```
+
+
+## Set up the dataset configuration files
+
+Then, set up the LAION dataset loading path in 
+[here](../minigpt4/configs/datasets/laion/defaults.yaml#L5) at Line 5 as 
+${MINIGPT4_DATASET}/laion/laion_dataset/{00000..10488}.tar
+
+and the Conceptual Captoin and SBU datasets loading path in 
+[here](../minigpt4/configs/datasets/cc_sbu/defaults.yaml#L5) at Line 5 as 
+${MINIGPT4_DATASET}/cc_sbu/cc_sbu_dataset/{00000..01255}.tar
+
+
+

dataset/README_2_STAGE.md

@@ -0,0 +1,19 @@
+## Second Stage Data Preparation
+
+Our second stage dataset can be downloaded from 
+[here](https://drive.google.com/file/d/1nJXhoEcy3KTExr17I7BXqY5Y9Lx_-n-9/view?usp=share_link) 
+After extraction, you will get a data follder with the following structure:
+
+```
+cc_sbu_align
+├── filter_cap.json
+└── image
+    ├── 2.jpg
+    ├── 3.jpg
+    ...   
+```
+
+Put the folder to any path you want.
+Then, set up the dataset path in the dataset config file 
+[here](../minigpt4/configs/datasets/cc_sbu/align.yaml#L5) at Line 5.
+

dataset/README_MINIGPTv2_FINETUNE.md

@@ -0,0 +1,285 @@
+## Download the dataset for finetuning the MiniGPT-v2
+
+
+Download the dataset
+
+Image source | Download path
+--- | :---:
+COCO 2014 images | <a href="http://images.cocodataset.org/zips/train2014.zip">images</a> &nbsp;&nbsp;  <a href="https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_train.json"> captions</a>
+COCO VQA | <a href="https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_train.json">vqa train</a> &nbsp;&nbsp;  <a href="https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_val.json"> vqa val</a>
+Visual Genome |  <a href="https://cs.stanford.edu/people/rak248/VG_100K_2/images.zip">images part1</a> &nbsp;&nbsp; <a href="https://cs.stanford.edu/people/rak248/VG_100K_2/images2.zip">images part2</a> &nbsp;&nbsp; <a href="https://homes.cs.washington.edu/~ranjay/visualgenome/data/dataset/image_data.json.zip"> image meta data </a>
+TextCaps | <a href="https://cs.stanford.edu/people/rak248/VG_100K_2/images.zip">images</a>  &nbsp;&nbsp; <a href="https://dl.fbaipublicfiles.com/textvqa/data/textcaps/TextCaps_0.1_train.json"> annotations</a> 
+RefCOCO | <a href="https://bvisionweb1.cs.unc.edu/licheng/referit/data/refcoco.zip"> annotations </a>
+RefCOCO+ | <a href="https://bvisionweb1.cs.unc.edu/licheng/referit/data/refcoco+.zip"> annotations </a>
+RefCOCOg | <a href="https://bvisionweb1.cs.unc.edu/licheng/referit/data/refcocog.zip"> annotations </a>
+OKVQA | <a href="https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/okvqa/okvqa_train.json"> annotations </a>
+AOK-VQA | <a href="https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/aokvqa_v1p0_train.json"> annotations </a>
+OCR-VQA | <a href="https://drive.google.com/drive/folders/1_GYPY5UkUy7HIcR0zq3ZCFgeZN7BAfm_?usp=sharing"> annotations </a>
+GQA | <a href="https://downloads.cs.stanford.edu/nlp/data/gqa/images.zip">images</a>  &nbsp;&nbsp; <a href="https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/gqa/train_balanced_questions.json"> annotations </a>
+Filtered flickr-30k |  <a href="https://drive.google.com/drive/folders/19c_ggBI77AvdtYlPbuI0ZpnPz73T5teX?usp=sharing"> annotations </a>
+Multi-task conversation |  <a href="https://drive.google.com/file/d/11HHqB2c29hbSk-WLxdta-nG8UCUrcCN1/view?usp=sharing"> annotations </a> 
+Filtered unnatural instruction |  <a href="https://drive.google.com/file/d/1lXNnBcb5WU-sc8Fe2T2N8J0NRw4sBLev/view?usp=sharing"> annotations </a>
+LLaVA | <a href="https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/complex_reasoning_77k.json"> Compelex reasoning </a> &nbsp;&nbsp;<a href="https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/detail_23k.json"> Detailed description </a> &nbsp;&nbsp; <a href="https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/conversation_58k.json"> Conversation </a> 
+
+
+
+### COCO captions
+Download the COCO 2014 images and captions
+
+coco 2014 images path
+
+```
+${MINIGPTv2_DATASET}
+├── coco
+│   ├── images
+...
+```
+
+
+coco caption annotation path
+
+```
+${MINIGPTv2_DATASET}
+├── coco_captions
+│   └── annotations
+│       ├── coco_karpathy_train.json
+...
+```
+
+Set **image_path** to the COCO 2014 image folder.
+Similarly, set **ann_path** to the coco_karpathy_train.json path
+- [minigpt4/configs/datasets/coco/caption.yaml](../minigpt4/configs/datasets/coco/caption.yaml)
+
+### COCO VQA
+Download the vqa v2 train and validation json files
+
+```
+├── ${MINIGPTv2_DATASET}
+│   ├── vqav2
+│       ├── vqa_train.json
+|       ├── vqa_val.json
+```
+
+Set **image_path** to the COCO 2014 image folder.
+Similarly, set **ann_path** to the vqa_train.json and vqa_val.json path
+- [minigpt4/configs/datasets/coco/defaults_vqa.yaml](../minigpt4/configs/datasets/coco/defaults_vqa.yaml)
+
+
+### Visual genome
+Download visiual genome images and annotation files
+
+```
+${MINIGPTv2_DATASET}
+├── visual_genome
+│   ├── VG_100K
+│   ├── VG_100K_2
+│   └── region_descriptions.json
+│   └── image_data.json
+...
+```
+
+Set **image_path** to visual_genome folder.
+Similarly, set **ann_path** to the visual_genome folder.
+
+- [minigpt4/configs/datasets/vg/ref.yaml](../minigpt4/configs/datasets/vg/ref.yaml)
+
+
+### TextCaps
+Download the TextCaps images and annotation files
+
+```
+├── ${MINIGPTv2_DATASET}
+│   ├── textcaps
+│       ├── train_images
+│       ├── TextCaps_0.1_train.json
+```
+
+Set **image_path** to TextCaps train_images folder.
+Similarly, set **ann_path** to the TextCaps_0.1_train.json path
+
+- [minigpt4/configs/datasets/textcaps/caption.yaml](../minigpt4/configs/datasets/textcaps/caption.yaml)
+
+### RefCOCO, RefCOCO+, RefCOCOg
+Download the RefCOCO, RefCOCO+, RefCOCOg annotation files
+
+```
+
+${MINIGPTv2_DATASET}
+├── refcoco_annotations
+│   ├── refcoco
+│   │   ├── instances.json
+│   │   ├��─ refs(google).p
+│   │   └── refs(unc).p
+│   ├── refcoco+
+│   │   ├── instances.json
+│   │   └── refs(unc).p
+│   └── refcocog
+│       ├── instances.json
+│       ├── refs(google).p
+│       └─── refs(und).p
+...
+```
+
+
+Set **image_path** to the COCO 2014 image folder.
+Similarly, set **ann_path** in all the following configs to the above folder *refcoco_annotations* that contains refcoco, refcoco+, and refcocog.
+
+- [minigpt4/configs/datasets/coco_bbox/refcoco.yaml](../minigpt4/configs/datasets/coco_bbox/refcoco.yaml)
+- [minigpt4/configs/datasets/coco_bbox/refcocog.yaml](../minigpt4/configs/datasets/coco_bbox/refcocog.yaml) 
+- [minigpt4/configs/datasets/coco_bbox/refcocop.yaml](../minigpt4/configs/datasets/coco_bbox/refcocop.yaml)
+- [minigpt4/configs/datasets/coco_bbox/invrefcoco.yaml](../minigpt4/configs/datasets/coco_bbox/invrefcoco.yaml)
+- [minigpt4/configs/datasets/coco_bbox/invrefcocog.yaml](../minigpt4/configs/datasets/coco_bbox/invrefcocog.yaml) 
+- [minigpt4/configs/datasets/coco_bbox/invrefcocop.yaml](../minigpt4/configs/datasets/coco_bbox/invrefcocop.yaml)
+
+
+
+
+### OKVQA
+
+
+```
+Location_you_like
+├── ${MINIGPTv2_DATASET}
+│   ├── okvqa
+│       ├── okvqa_train.json
+```
+
+Set **image_path** to the COCO 2014 image folder.
+Similarly, set **ann_path** to the location of the OKVQA dataset
+- [minigpt4/configs/datasets/okvqa/defaults.yaml](../minigpt4/configs/datasets/okvqa/defaults.yaml)
+
+
+### COCO-VQA
+
+- [OK-VQA Input Questions](https://okvqa.allenai.org/static/data/OpenEnded_mscoco_train2014_questions.json.zip)
+- [OK-VQA Annotations](https://okvqa.allenai.org/static/data/mscoco_train2014_annotations.json.zip)
+
+
+### AOK-VQA
+Download the AOK-VQA annotation dataset
+
+```
+export AOKVQA_DIR=YOUR_DATASET_PATH
+mkdir -p ${AOKVQA_DIR}
+curl -fsSL https://prior-datasets.s3.us-east-2.amazonaws.com/aokvqa/aokvqa_v1p0.tar.gz | tar xvz -C ${AOKVQA_DIR}
+```
+
+```
+Location_you_like
+├── ${MINIGPTv2_DATASET}
+│   ├── aokvqa
+│       ├── aokvqa_v1p0_train.json
+```
+
+
+Set **image_path** to the COCO 2014 image folder.
+Similarly, set **ann_path** to the location of the AOKVQA dataset
+- [minigpt4/configs/datasets/aokvqa/defaults.yaml](../minigpt4/configs/datasets/aokvqa/defaults.yaml)
+
+
+
+### OCR-VQA
+Download the OCR-VQA annotation files
+download the images with loadDataset.py script
+
+```
+Location_you_like
+├── ${MINIGPTv2_DATASET}
+│   ├── ocrvqa
+│       ├── images
+│       ├── dataset.json
+```
+
+Set **image_path** as the ocrvqa/images folder.
+Similarly, set **ann_path** to the dataset.json
+- [minigpt4/configs/datasets/ocrvqa/ocrvqa.yaml](../minigpt4/configs/datasets/ocrvqa/ocrvqa.yaml)
+
+### GQA
+Download the GQA annotation files and images
+
+```
+Location_you_like
+├── ${MINIGPTv2_DATASET}
+│   ├── gqa
+│       ├── images
+│       ├── train_balanced_questions.json
+```
+
+Set **image_path** as the gqa/images folder.
+Similarly, set **ann_path** to the train_balanced_questions.json
+- [minigpt4/configs/datasets/gqa/balanced_val.yaml](../minigpt4/configs/datasets/gqa/balanced_val.yaml)
+
+
+
+### filtered Flickr-30k
+Download filtered Flickr-30k images (fill this [form](https://forms.illinois.edu/sec/229675) on official website or from [kaggle](https://www.kaggle.com/datasets/hsankesara/flickr-image-dataset/download?datasetVersionNumber=1)) and annotation files
+
+```
+${MINIGPTv2_DATASET}
+├── filtered_flickr
+│   ├── images
+│   ├── captiontobbox.json
+│   ├── groundedcaption.json
+│   └── phrasetobbox.json
+...
+```
+
+Set **image_path** as the flickr-30k images foler.
+Similarly, set **ann_path** to the groundedcaption.json, captiontobbox.json and phrasetobbox.json for the 
+grounded image caption, caption to bbox, and phrase to bbox datasets.
+
+- [minigpt4/configs/datasets/flickr/default.yaml](../minigpt4/configs/datasets/flickr/default.yaml)
+- [minigpt4/configs/datasets/flickr/caption_to_phrase.yaml](../minigpt4/configs/datasets/flickr/caption_to_phrase.yaml)
+- [minigpt4/configs/datasets/flickr/object_to_phrase.yaml](../minigpt4/configs/datasets/flickr/object_to_phrase.yaml)
+
+
+### Multi-task conversation
+Download the multi-task converstation dataset
+
+```
+Location_you_like
+${MINIGPTv2_DATASET}
+├── multitask_conversation
+│   └── multitask_conversation.json
+...
+```
+
+Set **image_path** as the COCO 2014 images folder.
+Similarly, set **ann_path** to the multitask_conversation.json file path
+
+- [minigpt4/configs/datasets/multitask_conversation/default.yaml](../minigpt4/configs/datasets/multitask_conversation/default.yaml)
+
+### Unnatural instruction
+Download the filtered unnatural instruction annotation files (we remove the very long sentences from the original unnatural instruction dataset)
+
+```
+Location_you_like
+├── ${MINIGPTv2_DATASET}
+│   ├── unnatural_instructions
+│       ├── filtered_unnatural_instruction.json
+```
+
+There is no image path.
+Similarly, set **ann_path** to the filtered_unnatural_instruction.json file path
+
+- [minigpt4/configs/datasets/nlp/unnatural_instruction.yaml](../minigpt4/configs/datasets/nlp/unnatural_instruction.yaml)
+
+### LLaVA
+
+```
+Location_you_like
+├── ${MINIGPTv2_DATASET}
+│   ├── llava
+│       ├── conversation_58k.json
+│       ├── detail_23k.json
+│       ├── complex_reasoning_77k.json
+```
+
+Set **image_path** to the COCO 2014 image folder.
+Similarly, set **ann_path** to the location of the previous downloaded conversation_58k.json, 
+detail_23k.json, and complex_reasoning_77k.json in conversation.yaml, detail.yaml, and reason.yaml, respectively.
+
+
+- [minigpt4/configs/datasets/llava/conversation.yaml](../minigpt4/configs/datasets/llava/conversation.yaml)
+- [minigpt4/configs/datasets/llava/detail.yaml](../minigpt4/configs/datasets/llava/detail.yaml) 
+- [minigpt4/configs/datasets/llava/reason.yaml](../minigpt4/configs/datasets/llava/reason.yaml)

dataset/convert_cc_sbu.py

@@ -0,0 +1,20 @@
+import json
+import csv
+
+# specify input and output file paths
+input_file = 'ccs_synthetic_filtered_large.json'
+output_file = 'ccs_synthetic_filtered_large.tsv'
+
+# load JSON data from input file
+with open(input_file, 'r') as f:
+    data = json.load(f)
+
+# extract header and data from JSON
+header = data[0].keys()
+rows = [x.values() for x in data]
+
+# write data to TSV file
+with open(output_file, 'w') as f:
+    writer = csv.writer(f, delimiter='\t')
+    writer.writerow(header)
+    writer.writerows(rows)

dataset/convert_laion.py

@@ -0,0 +1,20 @@
+import json
+import csv
+
+# specify input and output file paths
+input_file = 'laion_synthetic_filtered_large.json'
+output_file = 'laion_synthetic_filtered_large.tsv'
+
+# load JSON data from input file
+with open(input_file, 'r') as f:
+    data = json.load(f)
+
+# extract header and data from JSON
+header = data[0].keys()
+rows = [x.values() for x in data]
+
+# write data to TSV file
+with open(output_file, 'w') as f:
+    writer = csv.writer(f, delimiter='\t')
+    writer.writerow(header)
+    writer.writerows(rows)

dataset/download_cc_sbu.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+img2dataset --url_list ccs_synthetic_filtered_large.tsv --input_format "tsv"\
+         --url_col "url" --caption_col "caption" --output_format webdataset\
+           --output_folder cc_sbu_dataset --processes_count 16 --thread_count 128 --image_size 224 \
+             --enable_wandb True

dataset/download_laion.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+img2dataset --url_list laion_synthetic_filtered_large.tsv --input_format "tsv"\
+         --url_col "url" --caption_col "caption" --output_format webdataset\
+           --output_folder laion_dataset --processes_count 16 --thread_count 128 --image_size 224 \
+             --enable_wandb True

demo.py

@@ -0,0 +1,171 @@
+import argparse
+import os
+import random
+
+import numpy as np
+import torch
+import torch.backends.cudnn as cudnn
+import gradio as gr
+
+from transformers import StoppingCriteriaList
+
+from minigpt4.common.config import Config
+from minigpt4.common.dist_utils import get_rank
+from minigpt4.common.registry import registry
+from minigpt4.conversation.conversation import Chat, CONV_VISION_Vicuna0, CONV_VISION_LLama2, StoppingCriteriaSub
+
+# imports modules for registration
+from minigpt4.datasets.builders import *
+from minigpt4.models import *
+from minigpt4.processors import *
+from minigpt4.runners import *
+from minigpt4.tasks import *
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Demo")
+    parser.add_argument("--cfg-path", required=True, help="path to configuration file.")
+    parser.add_argument("--gpu-id", type=int, default=0, help="specify the gpu to load the model.")
+    parser.add_argument(
+        "--options",
+        nargs="+",
+        help="override some settings in the used config, the key-value pair "
+        "in xxx=yyy format will be merged into config file (deprecate), "
+        "change to --cfg-options instead.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def setup_seeds(config):
+    seed = config.run_cfg.seed + get_rank()
+
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+
+    cudnn.benchmark = False
+    cudnn.deterministic = True
+
+
+# ========================================
+#             Model Initialization
+# ========================================
+
+conv_dict = {'pretrain_vicuna0': CONV_VISION_Vicuna0,
+             'pretrain_llama2': CONV_VISION_LLama2}
+
+print('Initializing Chat')
+args = parse_args()
+cfg = Config(args)
+
+model_config = cfg.model_cfg
+model_config.device_8bit = args.gpu_id
+model_cls = registry.get_model_class(model_config.arch)
+model = model_cls.from_config(model_config).to('cuda:{}'.format(args.gpu_id))
+
+CONV_VISION = conv_dict[model_config.model_type]
+
+vis_processor_cfg = cfg.datasets_cfg.cc_sbu_align.vis_processor.train
+vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg)
+
+stop_words_ids = [[835], [2277, 29937]]
+stop_words_ids = [torch.tensor(ids).to(device='cuda:{}'.format(args.gpu_id)) for ids in stop_words_ids]
+stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops=stop_words_ids)])
+
+chat = Chat(model, vis_processor, device='cuda:{}'.format(args.gpu_id), stopping_criteria=stopping_criteria)
+print('Initialization Finished')
+
+
+# ========================================
+#             Gradio Setting
+# ========================================
+
+
+def gradio_reset(chat_state, img_list):
+    if chat_state is not None:
+        chat_state.messages = []
+    if img_list is not None:
+        img_list = []
+    return None, gr.update(value=None, interactive=True), gr.update(placeholder='Please upload your image first', interactive=False),gr.update(value="Upload & Start Chat", interactive=True), chat_state, img_list
+
+
+def upload_img(gr_img, text_input, chat_state):
+    if gr_img is None:
+        return None, None, gr.update(interactive=True), chat_state, None
+    chat_state = CONV_VISION.copy()
+    img_list = []
+    llm_message = chat.upload_img(gr_img, chat_state, img_list)
+    chat.encode_img(img_list)
+    return gr.update(interactive=False), gr.update(interactive=True, placeholder='Type and press Enter'), gr.update(value="Start Chatting", interactive=False), chat_state, img_list
+
+
+def gradio_ask(user_message, chatbot, chat_state):
+    if len(user_message) == 0:
+        return gr.update(interactive=True, placeholder='Input should not be empty!'), chatbot, chat_state
+    chat.ask(user_message, chat_state)
+    chatbot = chatbot + [[user_message, None]]
+    return '', chatbot, chat_state
+
+
+def gradio_answer(chatbot, chat_state, img_list, num_beams, temperature):
+    llm_message = chat.answer(conv=chat_state,
+                              img_list=img_list,
+                              num_beams=num_beams,
+                              temperature=temperature,
+                              max_new_tokens=300,
+                              max_length=2000)[0]
+    chatbot[-1][1] = llm_message
+    return chatbot, chat_state, img_list
+
+
+title = """<h1 align="center">Demo of MiniGPT-4</h1>"""
+description = """<h3>This is the demo of MiniGPT-4. Upload your images and start chatting!</h3>"""
+article = """<p><a href='https://minigpt-4.github.io'><img src='https://img.shields.io/badge/Project-Page-Green'></a></p><p><a href='https://github.com/Vision-CAIR/MiniGPT-4'><img src='https://img.shields.io/badge/Github-Code-blue'></a></p><p><a href='https://raw.githubusercontent.com/Vision-CAIR/MiniGPT-4/main/MiniGPT_4.pdf'><img src='https://img.shields.io/badge/Paper-PDF-red'></a></p>
+"""
+
+#TODO show examples below
+
+with gr.Blocks() as demo:
+    gr.Markdown(title)
+    gr.Markdown(description)
+    gr.Markdown(article)
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            image = gr.Image(type="pil")
+            upload_button = gr.Button(value="Upload & Start Chat", interactive=True, variant="primary")
+            clear = gr.Button("Restart")
+            
+            num_beams = gr.Slider(
+                minimum=1,
+                maximum=10,
+                value=1,
+                step=1,
+                interactive=True,
+                label="beam search numbers)",
+            )
+            
+            temperature = gr.Slider(
+                minimum=0.1,
+                maximum=2.0,
+                value=1.0,
+                step=0.1,
+                interactive=True,
+                label="Temperature",
+            )
+
+        with gr.Column(scale=2):
+            chat_state = gr.State()
+            img_list = gr.State()
+            chatbot = gr.Chatbot(label='MiniGPT-4')
+            text_input = gr.Textbox(label='User', placeholder='Please upload your image first', interactive=False)
+    
+    upload_button.click(upload_img, [image, text_input, chat_state], [image, text_input, upload_button, chat_state, img_list])
+    
+    text_input.submit(gradio_ask, [text_input, chatbot, chat_state], [text_input, chatbot, chat_state]).then(
+        gradio_answer, [chatbot, chat_state, img_list, num_beams, temperature], [chatbot, chat_state, img_list]
+    )
+    clear.click(gradio_reset, [chat_state, img_list], [chatbot, image, text_input, upload_button, chat_state, img_list], queue=False)
+
+demo.launch(share=True, enable_queue=True)

demo_v2.py

@@ -0,0 +1,651 @@
+import argparse
+import os
+import random
+from collections import defaultdict
+
+import cv2
+import re
+
+import numpy as np
+from PIL import Image
+import torch
+import html
+import gradio as gr
+from torch.nn import DataParallel
+import torchvision.transforms as T
+import torch.backends.cudnn as cudnn
+
+from minigpt4.common.config import Config
+
+from minigpt4.common.registry import registry
+from minigpt4.conversation.conversation import Conversation, SeparatorStyle, Chat
+
+# imports modules for registration
+from minigpt4.datasets.builders import *
+from minigpt4.models import *
+from minigpt4.processors import *
+from minigpt4.runners import *
+from minigpt4.tasks import *
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Demo")
+    parser.add_argument("--cfg-path", default='eval_configs/minigptv2_eval.yaml',
+                        help="path to configuration file.")
+    parser.add_argument("--gpu-id", type=int, default=0, help="specify the gpu to load the model.")
+    parser.add_argument(
+        "--options",
+        nargs="+",
+        help="override some settings in the used config, the key-value pair "
+             "in xxx=yyy format will be merged into config file (deprecate), "
+             "change to --cfg-options instead.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+random.seed(42)
+np.random.seed(42)
+torch.manual_seed(42)
+
+cudnn.benchmark = False
+cudnn.deterministic = True
+
+print('Initializing Chat')
+args = parse_args()
+cfg = Config(args)
+
+device = 'cuda:{}'.format(args.gpu_id)
+
+model_config = cfg.model_cfg
+model_config.device_8bit = args.gpu_id
+model_cls = registry.get_model_class(model_config.arch)
+model = model_cls.from_config(model_config).to(device)
+bounding_box_size = 100
+
+vis_processor_cfg = cfg.datasets_cfg.cc_sbu_align.vis_processor.train
+vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg)
+
+# model = DataParallel(model)
+model = model.eval()
+
+CONV_VISION = Conversation(
+    system="",
+    roles=(r"<s>[INST] ", r" [/INST]"),
+    messages=[],
+    offset=2,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="",
+)
+
+
+def extract_substrings(string):
+    # first check if there is no-finished bracket
+    index = string.rfind('}')
+    if index != -1:
+        string = string[:index + 1]
+
+    pattern = r'<p>(.*?)\}(?!<)'
+    matches = re.findall(pattern, string)
+    substrings = [match for match in matches]
+
+    return substrings
+
+
+def is_overlapping(rect1, rect2):
+    x1, y1, x2, y2 = rect1
+    x3, y3, x4, y4 = rect2
+    return not (x2 < x3 or x1 > x4 or y2 < y3 or y1 > y4)
+
+
+def computeIoU(bbox1, bbox2):
+    x1, y1, x2, y2 = bbox1
+    x3, y3, x4, y4 = bbox2
+    intersection_x1 = max(x1, x3)
+    intersection_y1 = max(y1, y3)
+    intersection_x2 = min(x2, x4)
+    intersection_y2 = min(y2, y4)
+    intersection_area = max(0, intersection_x2 - intersection_x1 + 1) * max(0, intersection_y2 - intersection_y1 + 1)
+    bbox1_area = (x2 - x1 + 1) * (y2 - y1 + 1)
+    bbox2_area = (x4 - x3 + 1) * (y4 - y3 + 1)
+    union_area = bbox1_area + bbox2_area - intersection_area
+    iou = intersection_area / union_area
+    return iou
+
+
+def save_tmp_img(visual_img):
+    file_name = "".join([str(random.randint(0, 9)) for _ in range(5)]) + ".jpg"
+    file_path = "/tmp/gradio" + file_name
+    visual_img.save(file_path)
+    return file_path
+
+
+def mask2bbox(mask):
+    if mask is None:
+        return ''
+    mask = mask.resize([100, 100], resample=Image.NEAREST)
+    mask = np.array(mask)[:, :, 0]
+
+    rows = np.any(mask, axis=1)
+    cols = np.any(mask, axis=0)
+
+    if rows.sum():
+        # Get the top, bottom, left, and right boundaries
+        rmin, rmax = np.where(rows)[0][[0, -1]]
+        cmin, cmax = np.where(cols)[0][[0, -1]]
+        bbox = '{{<{}><{}><{}><{}>}}'.format(cmin, rmin, cmax, rmax)
+    else:
+        bbox = ''
+
+    return bbox
+
+
+def escape_markdown(text):
+    # List of Markdown special characters that need to be escaped
+    md_chars = ['<', '>']
+
+    # Escape each special character
+    for char in md_chars:
+        text = text.replace(char, '\\' + char)
+
+    return text
+
+
+def reverse_escape(text):
+    md_chars = ['\\<', '\\>']
+
+    for char in md_chars:
+        text = text.replace(char, char[1:])
+
+    return text
+
+
+colors = [
+    (255, 0, 0),
+    (0, 255, 0),
+    (0, 0, 255),
+    (210, 210, 0),
+    (255, 0, 255),
+    (0, 255, 255),
+    (114, 128, 250),
+    (0, 165, 255),
+    (0, 128, 0),
+    (144, 238, 144),
+    (238, 238, 175),
+    (255, 191, 0),
+    (0, 128, 0),
+    (226, 43, 138),
+    (255, 0, 255),
+    (0, 215, 255),
+]
+
+color_map = {
+    f"{color_id}": f"#{hex(color[2])[2:].zfill(2)}{hex(color[1])[2:].zfill(2)}{hex(color[0])[2:].zfill(2)}" for
+    color_id, color in enumerate(colors)
+}
+
+used_colors = colors
+
+
+def visualize_all_bbox_together(image, generation):
+    if image is None:
+        return None, ''
+
+    generation = html.unescape(generation)
+
+    image_width, image_height = image.size
+    image = image.resize([500, int(500 / image_width * image_height)])
+    image_width, image_height = image.size
+
+    string_list = extract_substrings(generation)
+    if string_list:  # it is grounding or detection
+        mode = 'all'
+        entities = defaultdict(list)
+        i = 0
+        j = 0
+        for string in string_list:
+            try:
+                obj, string = string.split('</p>')
+            except ValueError:
+                print('wrong string: ', string)
+                continue
+            bbox_list = string.split('<delim>')
+            flag = False
+            for bbox_string in bbox_list:
+                integers = re.findall(r'-?\d+', bbox_string)
+                if len(integers) == 4:
+                    x0, y0, x1, y1 = int(integers[0]), int(integers[1]), int(integers[2]), int(integers[3])
+                    left = x0 / bounding_box_size * image_width
+                    bottom = y0 / bounding_box_size * image_height
+                    right = x1 / bounding_box_size * image_width
+                    top = y1 / bounding_box_size * image_height
+
+                    entities[obj].append([left, bottom, right, top])
+
+                    j += 1
+                    flag = True
+            if flag:
+                i += 1
+    else:
+        integers = re.findall(r'-?\d+', generation)
+
+        if len(integers) == 4:  # it is refer
+            mode = 'single'
+
+            entities = list()
+            x0, y0, x1, y1 = int(integers[0]), int(integers[1]), int(integers[2]), int(integers[3])
+            left = x0 / bounding_box_size * image_width
+            bottom = y0 / bounding_box_size * image_height
+            right = x1 / bounding_box_size * image_width
+            top = y1 / bounding_box_size * image_height
+            entities.append([left, bottom, right, top])
+        else:
+            # don't detect any valid bbox to visualize
+            return None, ''
+
+    if len(entities) == 0:
+        return None, ''
+
+    if isinstance(image, Image.Image):
+        image_h = image.height
+        image_w = image.width
+        image = np.array(image)
+
+    elif isinstance(image, str):
+        if os.path.exists(image):
+            pil_img = Image.open(image).convert("RGB")
+            image = np.array(pil_img)[:, :, [2, 1, 0]]
+            image_h = pil_img.height
+            image_w = pil_img.width
+        else:
+            raise ValueError(f"invaild image path, {image}")
+    elif isinstance(image, torch.Tensor):
+
+        image_tensor = image.cpu()
+        reverse_norm_mean = torch.tensor([0.48145466, 0.4578275, 0.40821073])[:, None, None]
+        reverse_norm_std = torch.tensor([0.26862954, 0.26130258, 0.27577711])[:, None, None]
+        image_tensor = image_tensor * reverse_norm_std + reverse_norm_mean
+        pil_img = T.ToPILImage()(image_tensor)
+        image_h = pil_img.height
+        image_w = pil_img.width
+        image = np.array(pil_img)[:, :, [2, 1, 0]]
+    else:
+        raise ValueError(f"invaild image format, {type(image)} for {image}")
+
+    indices = list(range(len(entities)))
+
+    new_image = image.copy()
+
+    previous_bboxes = []
+    # size of text
+    text_size = 0.5
+    # thickness of text
+    text_line = 1  # int(max(1 * min(image_h, image_w) / 512, 1))
+    box_line = 2
+    (c_width, text_height), _ = cv2.getTextSize("F", cv2.FONT_HERSHEY_COMPLEX, text_size, text_line)
+    base_height = int(text_height * 0.675)
+    text_offset_original = text_height - base_height
+    text_spaces = 2
+
+    # num_bboxes = sum(len(x[-1]) for x in entities)
+    used_colors = colors  # random.sample(colors, k=num_bboxes)
+
+    color_id = -1
+    for entity_idx, entity_name in enumerate(entities):
+        if mode == 'single' or mode == 'identify':
+            bboxes = entity_name
+            bboxes = [bboxes]
+        else:
+            bboxes = entities[entity_name]
+        color_id += 1
+        for bbox_id, (x1_norm, y1_norm, x2_norm, y2_norm) in enumerate(bboxes):
+            skip_flag = False
+            orig_x1, orig_y1, orig_x2, orig_y2 = int(x1_norm), int(y1_norm), int(x2_norm), int(y2_norm)
+
+            color = used_colors[entity_idx % len(used_colors)]  # tuple(np.random.randint(0, 255, size=3).tolist())
+            new_image = cv2.rectangle(new_image, (orig_x1, orig_y1), (orig_x2, orig_y2), color, box_line)
+
+            if mode == 'all':
+                l_o, r_o = box_line // 2 + box_line % 2, box_line // 2 + box_line % 2 + 1
+
+                x1 = orig_x1 - l_o
+                y1 = orig_y1 - l_o
+
+                if y1 < text_height + text_offset_original + 2 * text_spaces:
+                    y1 = orig_y1 + r_o + text_height + text_offset_original + 2 * text_spaces
+                    x1 = orig_x1 + r_o
+
+                # add text background
+                (text_width, text_height), _ = cv2.getTextSize(f"  {entity_name}", cv2.FONT_HERSHEY_COMPLEX, text_size,
+                                                               text_line)
+                text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2 = x1, y1 - (
+                            text_height + text_offset_original + 2 * text_spaces), x1 + text_width, y1
+
+                for prev_bbox in previous_bboxes:
+                    if computeIoU((text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2), prev_bbox['bbox']) > 0.95 and \
+                            prev_bbox['phrase'] == entity_name:
+                        skip_flag = True
+                        break
+                    while is_overlapping((text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2), prev_bbox['bbox']):
+                        text_bg_y1 += (text_height + text_offset_original + 2 * text_spaces)
+                        text_bg_y2 += (text_height + text_offset_original + 2 * text_spaces)
+                        y1 += (text_height + text_offset_original + 2 * text_spaces)
+
+                        if text_bg_y2 >= image_h:
+                            text_bg_y1 = max(0, image_h - (text_height + text_offset_original + 2 * text_spaces))
+                            text_bg_y2 = image_h
+                            y1 = image_h
+                            break
+                if not skip_flag:
+                    alpha = 0.5
+                    for i in range(text_bg_y1, text_bg_y2):
+                        for j in range(text_bg_x1, text_bg_x2):
+                            if i < image_h and j < image_w:
+                                if j < text_bg_x1 + 1.35 * c_width:
+                                    # original color
+                                    bg_color = color
+                                else:
+                                    # white
+                                    bg_color = [255, 255, 255]
+                                new_image[i, j] = (alpha * new_image[i, j] + (1 - alpha) * np.array(bg_color)).astype(
+                                    np.uint8)
+
+                    cv2.putText(
+                        new_image, f"  {entity_name}", (x1, y1 - text_offset_original - 1 * text_spaces),
+                        cv2.FONT_HERSHEY_COMPLEX, text_size, (0, 0, 0), text_line, cv2.LINE_AA
+                    )
+
+                    previous_bboxes.append(
+                        {'bbox': (text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2), 'phrase': entity_name})
+
+    if mode == 'all':
+        def color_iterator(colors):
+            while True:
+                for color in colors:
+                    yield color
+
+        color_gen = color_iterator(colors)
+
+        # Add colors to phrases and remove <p></p>
+        def colored_phrases(match):
+            phrase = match.group(1)
+            color = next(color_gen)
+            return f'<span style="color:rgb{color}">{phrase}</span>'
+
+        generation = re.sub(r'{<\d+><\d+><\d+><\d+>}|<delim>', '', generation)
+        generation_colored = re.sub(r'<p>(.*?)</p>', colored_phrases, generation)
+    else:
+        generation_colored = ''
+
+    pil_image = Image.fromarray(new_image)
+    return pil_image, generation_colored
+
+
+def gradio_reset(chat_state, img_list):
+    if chat_state is not None:
+        chat_state.messages = []
+    if img_list is not None:
+        img_list = []
+    return None, gr.update(value=None, interactive=True), gr.update(placeholder='Upload your image and chat',
+                                                                    interactive=True), chat_state, img_list
+
+
+def image_upload_trigger(upload_flag, replace_flag, img_list):
+    # set the upload flag to true when receive a new image.
+    # if there is an old image (and old conversation), set the replace flag to true to reset the conv later.
+    upload_flag = 1
+    if img_list:
+        replace_flag = 1
+    return upload_flag, replace_flag
+
+
+def example_trigger(text_input, image, upload_flag, replace_flag, img_list):
+    # set the upload flag to true when receive a new image.
+    # if there is an old image (and old conversation), set the replace flag to true to reset the conv later.
+    upload_flag = 1
+    if img_list or replace_flag == 1:
+        replace_flag = 1
+
+    return upload_flag, replace_flag
+
+
+def gradio_ask(user_message, chatbot, chat_state, gr_img, img_list, upload_flag, replace_flag):
+    if len(user_message) == 0:
+        text_box_show = 'Input should not be empty!'
+    else:
+        text_box_show = ''
+
+    if isinstance(gr_img, dict):
+        gr_img, mask = gr_img['image'], gr_img['mask']
+    else:
+        mask = None
+
+    if '[identify]' in user_message:
+        # check if user provide bbox in the text input
+        integers = re.findall(r'-?\d+', user_message)
+        if len(integers) != 4:  # no bbox in text
+            bbox = mask2bbox(mask)
+            user_message = user_message + bbox
+
+    if chat_state is None:
+        chat_state = CONV_VISION.copy()
+
+    if upload_flag:
+        if replace_flag:
+            chat_state = CONV_VISION.copy()  # new image, reset everything
+            replace_flag = 0
+            chatbot = []
+        img_list = []
+        llm_message = chat.upload_img(gr_img, chat_state, img_list)
+        upload_flag = 0
+
+    chat.ask(user_message, chat_state)
+
+    chatbot = chatbot + [[user_message, None]]
+
+    if '[identify]' in user_message:
+        visual_img, _ = visualize_all_bbox_together(gr_img, user_message)
+        if visual_img is not None:
+            file_path = save_tmp_img(visual_img)
+            chatbot = chatbot + [[(file_path,), None]]
+
+    return text_box_show, chatbot, chat_state, img_list, upload_flag, replace_flag
+
+
+def gradio_answer(chatbot, chat_state, img_list, temperature):
+    llm_message = chat.answer(conv=chat_state,
+                              img_list=img_list,
+                              temperature=temperature,
+                              max_new_tokens=500,
+                              max_length=2000)[0]
+    chatbot[-1][1] = llm_message
+    return chatbot, chat_state
+
+
+def gradio_stream_answer(chatbot, chat_state, img_list, temperature):
+    if len(img_list) > 0:
+        if not isinstance(img_list[0], torch.Tensor):
+            chat.encode_img(img_list)
+    streamer = chat.stream_answer(conv=chat_state,
+                                  img_list=img_list,
+                                  temperature=temperature,
+                                  max_new_tokens=500,
+                                  max_length=2000)
+    output = ''
+    for new_output in streamer:
+        escapped = escape_markdown(new_output)
+        output += escapped
+        chatbot[-1][1] = output
+        yield chatbot, chat_state
+    chat_state.messages[-1][1] = '</s>'
+    return chatbot, chat_state
+
+
+def gradio_visualize(chatbot, gr_img):
+    if isinstance(gr_img, dict):
+        gr_img, mask = gr_img['image'], gr_img['mask']
+
+    unescaped = reverse_escape(chatbot[-1][1])
+    visual_img, generation_color = visualize_all_bbox_together(gr_img, unescaped)
+    if visual_img is not None:
+        if len(generation_color):
+            chatbot[-1][1] = generation_color
+        file_path = save_tmp_img(visual_img)
+        chatbot = chatbot + [[None, (file_path,)]]
+
+    return chatbot
+
+
+def gradio_taskselect(idx):
+    prompt_list = [
+        '',
+        '[grounding] describe this image in detail',
+        '[refer] ',
+        '[detection] ',
+        '[identify] what is this ',
+        '[vqa] '
+    ]
+    instruct_list = [
+        '**Hint:** Type in whatever you want',
+        '**Hint:** Send the command to generate a grounded image description',
+        '**Hint:** Type in a phrase about an object in the image and send the command',
+        '**Hint:** Type in a caption or phrase, and see object locations in the image',
+        '**Hint:** Draw a bounding box on the uploaded image then send the command. Click the "clear" botton on the top right of the image before redraw',
+        '**Hint:** Send a question to get a short answer',
+    ]
+    return prompt_list[idx], instruct_list[idx]
+
+
+
+
+chat = Chat(model, vis_processor, device=device)
+
+title = """<h1 align="center">MiniGPT-v2 Demo</h1>"""
+description = 'Welcome to Our MiniGPT-v2 Chatbot Demo!'
+# article = """<p><a href='https://minigpt-v2.github.io'><img src='https://img.shields.io/badge/Project-Page-Green'></a></p><p><a href='https://github.com/Vision-CAIR/MiniGPT-4/blob/main/MiniGPTv2.pdf'><img src='https://img.shields.io/badge/Paper-PDF-red'></a></p><p><a href='https://github.com/Vision-CAIR/MiniGPT-4'><img src='https://img.shields.io/badge/GitHub-Repo-blue'></a></p><p><a href='https://www.youtube.com/watch?v=atFCwV2hSY4'><img src='https://img.shields.io/badge/YouTube-Video-red'></a></p>"""
+article = """<p><a href='https://minigpt-v2.github.io'><img src='https://img.shields.io/badge/Project-Page-Green'></a></p>"""
+
+introduction = '''
+For Abilities Involving Visual Grounding:
+1. Grounding: CLICK **Send** to generate a grounded image description.
+2. Refer: Input a referring object and CLICK **Send**.
+3. Detection: Write a caption or phrase, and CLICK **Send**.
+4. Identify: Draw the bounding box on the uploaded image window and CLICK **Send** to generate the bounding box. (CLICK "clear" button before re-drawing next time).
+5. VQA: Input a visual question and CLICK **Send**.
+6. No Tag: Input whatever you want and CLICK **Send** without any tagging
+
+You can also simply chat in free form!
+'''
+
+text_input = gr.Textbox(placeholder='Upload your image and chat', interactive=True, show_label=False, container=False,
+                        scale=8)
+with gr.Blocks() as demo:
+    gr.Markdown(title)
+    # gr.Markdown(description)
+    gr.Markdown(article)
+
+    with gr.Row():
+        with gr.Column(scale=0.5):
+            image = gr.Image(type="pil", tool='sketch', brush_radius=20)
+
+            temperature = gr.Slider(
+                minimum=0.1,
+                maximum=1.5,
+                value=0.6,
+                step=0.1,
+                interactive=True,
+                label="Temperature",
+            )
+
+            clear = gr.Button("Restart")
+
+            gr.Markdown(introduction)
+
+        with gr.Column():
+            chat_state = gr.State(value=None)
+            img_list = gr.State(value=[])
+            chatbot = gr.Chatbot(label='MiniGPT-v2')
+
+            dataset = gr.Dataset(
+                components=[gr.Textbox(visible=False)],
+                samples=[['No Tag'], ['Grounding'], ['Refer'], ['Detection'], ['Identify'], ['VQA']],
+                type="index",
+                label='Task Shortcuts',
+            )
+            task_inst = gr.Markdown('**Hint:** Upload your image and chat')
+            with gr.Row():
+                text_input.render()
+                send = gr.Button("Send", variant='primary', size='sm', scale=1)
+
+    upload_flag = gr.State(value=0)
+    replace_flag = gr.State(value=0)
+    image.upload(image_upload_trigger, [upload_flag, replace_flag, img_list], [upload_flag, replace_flag])
+
+    with gr.Row():
+        with gr.Column():
+            gr.Examples(examples=[
+                ["examples_v2/office.jpg", "[grounding] describe this image in detail", upload_flag, replace_flag,
+                 img_list],
+                ["examples_v2/sofa.jpg", "[detection] sofas", upload_flag, replace_flag, img_list],
+                ["examples_v2/2000x1372_wmkn_0012149409555.jpg", "[refer] the world cup", upload_flag, replace_flag,
+                 img_list],
+                ["examples_v2/KFC-20-for-20-Nuggets.jpg", "[identify] what is this {<4><50><30><65>}", upload_flag,
+                 replace_flag, img_list],
+            ], inputs=[image, text_input, upload_flag, replace_flag, img_list], fn=example_trigger,
+                outputs=[upload_flag, replace_flag])
+        with gr.Column():
+            gr.Examples(examples=[
+                ["examples_v2/glip_test.jpg", "[vqa] where should I hide in this room when playing hide and seek",
+                 upload_flag, replace_flag, img_list],
+                ["examples_v2/float.png", "Please write a poem about the image", upload_flag, replace_flag, img_list],
+                ["examples_v2/thief.png", "Is the weapon fateful", upload_flag, replace_flag, img_list],
+                ["examples_v2/cockdial.png", "What might happen in this image in the next second", upload_flag,
+                 replace_flag, img_list],
+            ], inputs=[image, text_input, upload_flag, replace_flag, img_list], fn=example_trigger,
+                outputs=[upload_flag, replace_flag])
+
+    dataset.click(
+        gradio_taskselect,
+        inputs=[dataset],
+        outputs=[text_input, task_inst],
+        show_progress="hidden",
+        postprocess=False,
+        queue=False,
+    )
+
+    text_input.submit(
+        gradio_ask,
+        [text_input, chatbot, chat_state, image, img_list, upload_flag, replace_flag],
+        [text_input, chatbot, chat_state, img_list, upload_flag, replace_flag], queue=False
+    ).success(
+        gradio_stream_answer,
+        [chatbot, chat_state, img_list, temperature],
+        [chatbot, chat_state]
+    ).success(
+        gradio_visualize,
+        [chatbot, image],
+        [chatbot],
+        queue=False,
+    )
+
+    send.click(
+        gradio_ask,
+        [text_input, chatbot, chat_state, image, img_list, upload_flag, replace_flag],
+        [text_input, chatbot, chat_state, img_list, upload_flag, replace_flag], queue=False
+    ).success(
+        gradio_stream_answer,
+        [chatbot, chat_state, img_list, temperature],
+        [chatbot, chat_state]
+    ).success(
+        gradio_visualize,
+        [chatbot, image],
+        [chatbot],
+        queue=False,
+    )
+
+    clear.click(gradio_reset, [chat_state, img_list], [chatbot, image, text_input, chat_state, img_list], queue=False)
+
+demo.launch(share=True, enable_queue=True)
+
+
+

environment.yml

@@ -0,0 +1,35 @@
+name: minigptv
+channels:
+  - pytorch
+  - defaults
+  - anaconda
+dependencies:
+  - python=3.9
+  - cudatoolkit
+  - pip
+  - pip:
+    - torch==2.0.0
+    - torchaudio
+    - torchvision
+    - huggingface-hub==0.18.0
+    - matplotlib==3.7.0
+    - psutil==5.9.4
+    - iopath
+    - pyyaml==6.0
+    - regex==2022.10.31
+    - tokenizers==0.13.2
+    - tqdm==4.64.1
+    - transformers==4.30.0
+    - timm==0.6.13
+    - webdataset==0.2.48
+    - omegaconf==2.3.0
+    - opencv-python==4.7.0.72
+    - decord==0.6.0
+    - peft==0.2.0
+    - sentence-transformers
+    - gradio==3.47.1
+    - accelerate==0.20.3
+    - bitsandbytes==0.37.0
+    - scikit-image
+    - visual-genome
+    - wandb

eval_configs/minigpt4_eval.yaml

@@ -0,0 +1,22 @@
+model:
+  arch: minigpt4
+  model_type: pretrain_vicuna0
+  max_txt_len: 160
+  end_sym: "###"
+  low_resource: True
+  prompt_template: '###Human: {} ###Assistant: '
+  ckpt: 'please set this value to the path of pretrained checkpoint'
+
+
+datasets:
+  cc_sbu_align:
+    vis_processor:
+      train:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      train:
+        name: "blip_caption"
+
+run:
+  task: image_text_pretrain

eval_configs/minigpt4_llama2_eval.yaml

@@ -0,0 +1,22 @@
+model:
+  arch: minigpt4
+  model_type: pretrain_llama2
+  max_txt_len: 160
+  end_sym: "</s>"
+  low_resource: True
+  prompt_template: '[INST] {} [/INST] '
+  ckpt: 'please set this value to the path of pretrained checkpoint'
+
+
+datasets:
+  cc_sbu_align:
+    vis_processor:
+      train:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      train:
+        name: "blip_caption"
+
+run:
+  task: image_text_pretrain

eval_configs/minigptv2_benchmark_evaluation.yaml

@@ -0,0 +1,79 @@
+model:
+  arch: minigpt_v2
+  model_type: pretrain
+  max_txt_len: 500
+  end_sym: "</s>"
+  low_resource: False
+  prompt_template: '[INST] {} [/INST]'
+  llama_model: ""
+  ckpt: ""
+  lora_r: 64
+  lora_alpha: 16
+
+
+datasets:
+  cc_sbu_align:
+    vis_processor:
+      train:
+        name: "blip2_image_eval"
+        image_size: 448
+    text_processor:
+      train:
+        name: "blip_caption"
+
+evaluation_datasets:
+  refcoco:
+    eval_file_path: /path/to/eval/annotation/path  
+    img_path: /path/to/eval/image/path      
+    max_new_tokens: 20
+    batch_size: 10
+  refcocog:
+    eval_file_path: /path/to/eval/annotation/path  
+    img_path: /path/to/eval/image/path    
+    max_new_tokens: 20
+    batch_size: 10
+  refcoco+:
+    eval_file_path: /path/to/eval/annotation/path  
+    img_path: /path/to/eval/image/path    
+    max_new_tokens: 20
+    batch_size: 10
+  gqa:
+    eval_file_path: /path/to/eval/annotation/path  
+    img_path: /path/to/eval/image/path    
+    max_new_tokens: 20
+    batch_size: 10
+  okvqa:
+    eval_file_path: /path/to/eval/annotation/path  
+    img_path: /path/to/eval/image/path     
+    max_new_tokens: 20
+    batch_size: 10
+  vizwiz:
+    eval_file_path: /path/to/eval/annotation/path  
+    img_path: /path/to/eval/image/path    
+    max_new_tokens: 20
+    batch_size: 10
+  iconvqa:
+    eval_file_path: /path/to/eval/annotation/path  
+    img_path: /path/to/eval/image/path    
+    max_new_tokens: 20
+    batch_size: 10
+  vsr:
+    eval_file_path: cambridgeltl/vsr_zeroshot 
+    img_path: /path/to/eval/image/path    
+    max_new_tokens: 20
+    batch_size: 10
+  hm:
+    eval_file_path: /path/to/eval/annotation/path  
+    img_path: /path/to/eval/image/path 
+    max_new_tokens: 20
+    batch_size: 100
+
+run:
+  task: image_text_pretrain
+  name: minigptv2_evaluation
+  save_path: /path/to/save/folder_path
+
+  
+
+  
+

eval_configs/minigptv2_eval.yaml

@@ -0,0 +1,24 @@
+model:
+  arch: minigpt_v2
+  model_type: pretrain
+  max_txt_len: 500
+  end_sym: "</s>"
+  low_resource: True
+  prompt_template: '[INST] {} [/INST]'
+  ckpt: "/data3/chengzhi/MiniGPT-4/checkpoint_stage2.pth"
+  lora_r: 64
+  lora_alpha: 16
+
+
+datasets:
+  cc_sbu_align:
+    vis_processor:
+      train:
+        name: "blip2_image_eval"
+        image_size: 448
+    text_processor:
+      train:
+        name: "blip_caption"
+
+run:
+  task: image_text_pretrain

eval_scripts/EVAL_README.md

@@ -0,0 +1,104 @@
+## Evaluation Instruction for MiniGPT-v2
+
+### Data preparation
+Images download
+Image source | Download path
+--- | :---:
+OKVQA| <a href="https://drive.google.com/drive/folders/1jxIgAhtaLu_YqnZEl8Ym11f7LhX3nptN?usp=sharing">annotations</a> &nbsp;&nbsp;  <a href="http://images.cocodataset.org/zips/train2017.zip"> images</a>
+gqa | <a href="https://drive.google.com/drive/folders/1-dF-cgFwstutS4qq2D9CFQTDS0UTmIft?usp=drive_link">annotations</a> &nbsp;&nbsp;  <a href="https://downloads.cs.stanford.edu/nlp/data/gqa/images.zip">images</a> 
+hateful meme |  <a href="https://github.com/faizanahemad/facebook-hateful-memes">images and annotations</a> 
+iconqa |  <a href="https://iconqa.github.io/#download">images and annotation</a>
+vizwiz |  <a href="https://vizwiz.org/tasks-and-datasets/vqa/">images and annotation</a>
+RefCOCO | <a href="https://bvisionweb1.cs.unc.edu/licheng/referit/data/refcoco.zip"> annotations </a>
+RefCOCO+ | <a href="https://bvisionweb1.cs.unc.edu/licheng/referit/data/refcoco+.zip"> annotations </a>
+RefCOCOg | <a href="https://bvisionweb1.cs.unc.edu/licheng/referit/data/refcocog.zip"> annotations </a>
+
+### Evaluation dataset structure
+
+```
+${MINIGPTv2_EVALUATION_DATASET}
+├── gqa
+│   └── test_balanced_questions.json
+│   ├── testdev_balanced_questions.json
+│   ├── gqa_images
+├── hateful_meme
+│   └── hm_images
+│   ├── dev.jsonl
+├── iconvqa
+│   └── iconvqa_images
+│   ├── choose_text_val.json
+├── vizwiz
+│   └── vizwiz_images
+│   ├── val.json
+├── vsr
+│   └── vsr_images
+├── okvqa
+│   ├── okvqa_test_split.json
+│   ├── mscoco_val2014_annotations_clean.json
+│   ├── OpenEnded_mscoco_val2014_questions_clean.json
+├── refcoco
+│   └── instances.json
+│   ├── refs(google).p
+│   ├── refs(unc).p
+├── refcoco+
+│   └── instances.json
+│   ├── refs(unc).p
+├── refercocog
+│   └── instances.json
+│   ├── refs(google).p
+│   ├── refs(und).p
+...
+```
+
+
+### environment setup
+
+```
+export PYTHONPATH=$PYTHONPATH:/path/to/directory/of/MiniGPT-4
+```
+
+### config file setup
+
+Set **llama_model** to the path of LLaMA model.  
+Set **ckpt** to the path of our pretrained model.  
+Set **eval_file_path** to the path of the annotation files for each evaluation data.  
+Set **img_path** to the img_path for each evaluation dataset.  
+Set **save_path** to the save_path for each evaluation dataset.    
+
+in [eval_configs/minigptv2_benchmark_evaluation.yaml](../eval_configs/minigptv2_benchmark_evaluation.yaml) 
+
+
+
+
+### start evalauting RefCOCO, RefCOCO+, RefCOCOg
+port=port_number  
+cfg_path=/path/to/eval_configs/minigptv2_benchmark_evaluation.yaml  
+
+dataset names:  
+| refcoco | refcoco+ | refcocog |
+| ------- | -------- | -------- |
+
+```
+torchrun --master-port ${port} --nproc_per_node 1 eval_ref.py \
+ --cfg-path ${cfg_path} --dataset refcoco,refcoco+,refcocog --resample
+```
+
+
+### start evaluating visual question answering
+
+port=port_number  
+cfg_path=/path/to/eval_configs/minigptv2_benchmark_evaluation.yaml 
+
+dataset names:  
+| okvqa | vizwiz | iconvqa | gqa | vsr | hm |
+| ------- | -------- | -------- |-------- | -------- | -------- |
+
+
+```
+torchrun --master-port ${port} --nproc_per_node 1 eval_vqa.py \
+ --cfg-path ${cfg_path} --dataset okvqa,vizwiz,iconvqa,gqa,vsr,hm
+```
+
+
+
+

eval_scripts/eval_ref.py

@@ -0,0 +1,128 @@
+import os
+import re
+import json
+import argparse
+from collections import defaultdict
+import random
+import numpy as np
+from PIL import Image
+from tqdm import tqdm
+import torch
+from torch.utils.data import DataLoader
+from minigpt4.common.config import Config
+from minigpt4.common.eval_utils import prepare_texts, init_model, eval_parser, computeIoU
+from minigpt4.conversation.conversation import CONV_VISION_minigptv2
+
+from minigpt4.datasets.datasets.coco_caption import RefCOCOEvalData
+
+def list_of_str(arg):
+    return list(map(str, arg.split(',')))
+
+parser = eval_parser()
+parser.add_argument("--dataset", type=list_of_str, default='refcoco', help="dataset to evaluate")
+parser.add_argument("--res", type=float, default=100.0, help="resolution used in refcoco")
+parser.add_argument("--resample", action='store_true', help="resolution used in refcoco")
+args = parser.parse_args()
+
+cfg = Config(args)
+
+eval_dict = {'refcoco': ['val','testA','testB'], 
+            'refcoco+': ['val','testA','testB'],
+            'refcocog': ['val','test']}
+
+
+model, vis_processor = init_model(args)
+model.eval()
+CONV_VISION = CONV_VISION_minigptv2
+conv_temp = CONV_VISION.copy()
+conv_temp.system = ""
+
+# 
+model.eval()
+save_path = cfg.run_cfg.save_path
+
+
+
+for dataset in args.dataset:
+    for split in eval_dict[dataset]:
+
+        eval_file_path = cfg.evaluation_datasets_cfg[dataset]["eval_file_path"]
+        img_path = cfg.evaluation_datasets_cfg[dataset]["img_path"]
+        batch_size = cfg.evaluation_datasets_cfg[dataset]["batch_size"]
+        max_new_tokens = cfg.evaluation_datasets_cfg[dataset]["max_new_tokens"]
+
+        with open(os.path.join(eval_file_path,f"{dataset}/{dataset}_{split}.json"), 'r') as f:
+            refcoco = json.load(f)
+
+        data = RefCOCOEvalData(refcoco, vis_processor, img_path)
+        eval_dataloader = DataLoader(data, batch_size=batch_size, shuffle=False)
+        minigpt4_predict = defaultdict(list)
+        resamples = []
+
+        for images, questions, img_ids in tqdm(eval_dataloader):
+            texts = prepare_texts(questions, conv_temp)  # warp the texts with conversation template
+            answers = model.generate(images, texts, max_new_tokens=max_new_tokens, do_sample=False)
+            for answer, img_id, question in zip(answers, img_ids, questions):
+                answer = answer.replace("<unk>","").replace(" ","").strip()
+                pattern = r'\{<\d{1,3}><\d{1,3}><\d{1,3}><\d{1,3}>\}'
+                if re.match(pattern, answer):
+                    minigpt4_predict[img_id].append(answer)
+                else:
+                    resamples.append({'img_id': img_id, 'sents': [question.replace('[refer] give me the location of','').strip()]})
+        if args.resample:
+            for i in range(20):
+                data = RefCOCOEvalData(resamples, vis_processor, img_path)
+                resamples = []
+                eval_dataloader = DataLoader(data, batch_size=batch_size, shuffle=False)
+                for images, questions, img_ids in tqdm(eval_dataloader):
+                    texts = prepare_texts(questions, conv_temp)  # warp the texts with conversation template
+                    answers = model.generate(images, texts, max_new_tokens=max_new_tokens, do_sample=False)
+                    for answer, img_id, question in zip(answers, img_ids, questions):
+                        answer = answer.replace("<unk>","").replace(" ","").strip()
+                        pattern = r'\{<\d{1,3}><\d{1,3}><\d{1,3}><\d{1,3}>\}'
+                        if re.match(pattern, answer) or i == 4:
+                            minigpt4_predict[img_id].append(answer)
+                        else:
+                            resamples.append({'img_id': img_id, 'sents': [question.replace('[refer] give me the location of','').strip()]})
+                            
+                if len(resamples) == 0:
+                    break
+        
+        file_save_path = os.path.join(save_path,f"{args.dataset}_{split}.json")
+        with open(file_save_path,'w') as f:
+            json.dump(minigpt4_predict, f)
+
+        count=0
+        total=len(refcoco)
+        res=args.res
+        refcoco_dict = defaultdict()
+        for item in refcoco:
+            refcoco_dict[item['img_id']] = item
+        for img_id in refcoco_dict:
+            item = refcoco_dict[img_id]
+            bbox = item['bbox']
+            outputs = minigpt4_predict[img_id]
+            for output in outputs:
+                try:
+                    integers = re.findall(r'\d+', output)
+                    pred_bbox = [int(num) for num in integers]
+                    height = item['height']
+                    width = item['width']
+                    pred_bbox[0] = pred_bbox[0] / res * width
+                    pred_bbox[1] = pred_bbox[1] / res * height
+                    pred_bbox[2] = pred_bbox[2] / res * width
+                    pred_bbox[3] = pred_bbox[3] / res * height
+
+                    gt_bbox = [0,0,0,0]
+                    gt_bbox[0] = bbox[0]
+                    gt_bbox[1] = bbox[1]
+                    gt_bbox[2] = bbox[0] + bbox[2]
+                    gt_bbox[3] = bbox[1] + bbox[3]
+
+                    iou_score = computeIoU(pred_bbox, gt_bbox)
+                    if iou_score > 0.5:
+                        count+=1
+                except:
+                    continue
+        
+        print(f'{dataset} {split}:', count / total * 100, flush=True)

eval_scripts/eval_vqa.py

@@ -0,0 +1,252 @@
+import os
+import re
+import json
+import argparse
+from collections import defaultdict
+
+import numpy as np
+from PIL import Image
+from tqdm import tqdm
+import torch
+from torch.utils.data import DataLoader
+from datasets import load_dataset
+
+
+from minigpt4.datasets.datasets.vqa_datasets import OKVQAEvalData,VizWizEvalData,IconQAEvalData,GQAEvalData,VSREvalData,HMEvalData
+from minigpt4.common.vqa_tools.VQA.PythonHelperTools.vqaTools.vqa import VQA
+from minigpt4.common.vqa_tools.VQA.PythonEvaluationTools.vqaEvaluation.vqaEval import VQAEval
+
+from minigpt4.common.eval_utils import prepare_texts, init_model, eval_parser
+from minigpt4.conversation.conversation import CONV_VISION_minigptv2
+from minigpt4.common.config import Config
+
+
+def list_of_str(arg):
+    return list(map(str, arg.split(',')))
+
+parser = eval_parser()
+parser.add_argument("--dataset", type=list_of_str, default='refcoco', help="dataset to evaluate")
+args = parser.parse_args()
+cfg = Config(args)
+
+
+
+model, vis_processor = init_model(args)
+conv_temp = CONV_VISION_minigptv2.copy()
+conv_temp.system = ""
+model.eval()
+save_path = cfg.run_cfg.save_path
+
+
+if 'okvqa' in args.dataset:
+
+    eval_file_path = cfg.evaluation_datasets_cfg["okvqa"]["eval_file_path"]
+    img_path = cfg.evaluation_datasets_cfg["okvqa"]["img_path"]
+    batch_size = cfg.evaluation_datasets_cfg["okvqa"]["batch_size"]
+    max_new_tokens = cfg.evaluation_datasets_cfg["okvqa"]["max_new_tokens"]
+    
+
+    evaluation_annntation_path = os.path.join(eval_file_path, "okvqa_test_split.json")
+    with open(evaluation_annntation_path) as f:
+        ok_vqa_test_split = json.load(f)
+
+    data = OKVQAEvalData(ok_vqa_test_split, vis_processor, img_path)
+    eval_dataloader = DataLoader(data, batch_size=batch_size, shuffle=False)
+    minigpt4_predict = []
+
+    for images, questions, question_ids, img_ids in eval_dataloader:
+        texts = prepare_texts(questions, conv_temp)  # warp the texts with conversation template
+        answers = model.generate(images, texts, max_new_tokens=max_new_tokens, do_sample=False)
+
+        for answer, question_id, question, img_id in zip(answers, question_ids, questions, img_ids):
+            result = dict()
+            answer = answer.lower().replace('<unk>','').strip()
+            result['answer'] = answer
+            result['question_id'] = int(question_id)
+            minigpt4_predict.append(result)
+
+    file_save_path= os.path.join(save_path,"okvqa.json")
+    with open(file_save_path,'w') as f:
+        json.dump(minigpt4_predict, f)
+
+    annFile = os.path.join(eval_file_path,"mscoco_val2014_annotations_clean.json")
+    quesFile = os.path.join(eval_file_path,"OpenEnded_mscoco_val2014_questions_clean.json" )
+
+    vqa = VQA(annFile, quesFile)
+    vqaRes = vqa.loadRes(file_save_path, quesFile)
+
+    vqaEval = VQAEval(vqa, vqaRes, n=2)
+    vqaEval.evaluate()
+    print ("Overall OKVQA Accuracy is: %.02f\n" %(vqaEval.accuracy['overall']), flush=True)
+
+if 'vizwiz' in args.dataset:
+
+    eval_file_path = cfg.evaluation_datasets_cfg["vizwiz"]["eval_file_path"]
+    img_path = cfg.evaluation_datasets_cfg["vizwiz"]["img_path"]
+    batch_size = cfg.evaluation_datasets_cfg["vizwiz"]["batch_size"]
+    max_new_tokens = cfg.evaluation_datasets_cfg["vizwiz"]["max_new_tokens"]
+
+    vizwiz = json.load(open(eval_file_path, 'r'))
+
+    data = VizWizEvalData(vizwiz, vis_processor, img_path)
+    eval_dataloader = DataLoader(data, batch_size=batch_size, shuffle=False)
+    minigpt4_predict = []
+    total_acc = []
+    for images, texts, gt_answers in tqdm(eval_dataloader):
+        texts = prepare_texts(texts, conv_temp)  # warp the texts with conversation template
+        with torch.no_grad():
+            answers = model.generate(images, texts, max_new_tokens=max_new_tokens, do_sample=False,repetition_penalty=1.0)
+
+        for answer, gt_answer in zip(answers, gt_answers):
+            result = dict()
+            result['answer'] = answer.replace('<unk>','').strip()
+            minigpt4_predict.append(result)
+            count=0
+            gt_answer = gt_answer.split('_')
+            for gt in gt_answer:
+                if gt.lower() == answer.lower():
+                    count += 1
+            acc = min(count/3.0, 1.0)
+            total_acc.append(acc)
+        
+    file_save_path = os.path.join(save_path, "vizwiz.json")
+    with open(file_save_path,'w') as f:
+        json.dump(minigpt4_predict, f)
+    print('vizwiz Acc: ', np.average(total_acc)* 100.0, flush=True)
+
+
+if 'iconvqa' in args.dataset:
+
+    eval_file_path = cfg.evaluation_datasets_cfg["iconvqa"]["eval_file_path"]
+    img_path = cfg.evaluation_datasets_cfg["iconvqa"]["img_path"]
+    batch_size = cfg.evaluation_datasets_cfg["iconvqa"]["batch_size"]
+    max_new_tokens = cfg.evaluation_datasets_cfg["iconvqa"]["max_new_tokens"]
+
+    iconqa_text_val = json.load(open(eval_file_path,"r"))
+
+    data = IconQAEvalData(iconqa_text_val, vis_processor, img_path)
+    eval_dataloader = DataLoader(data, batch_size=batch_size, shuffle=False)
+
+    count = 0
+    for images, texts, candidates, answers in tqdm(eval_dataloader):
+        candidates = [candidate.split('_') for candidate in candidates]
+        num_cand = [len(candidate) for candidate in candidates]
+        for candidate in candidates:
+            candidate.extend(['none'] * (max(num_cand) - len(candidate)))
+        candidates = [list(x) for x in zip(*candidates)]
+        instructions = ["<s>[INST] <Img><ImageHere></Img> {} [/INST]".format(text) for text in texts]
+        answer_ranks = model.multi_select(images, instructions, candidates, num_cand=num_cand)
+        for idx, answer in enumerate(answers):
+            if answer_ranks[idx][0] == answer:
+                count += 1
+
+    print('iconqa Acc: ', count / len(iconqa_text_val) * 100.0, flush=True)
+
+
+if 'gqa' in args.dataset:
+
+    eval_file_path = cfg.evaluation_datasets_cfg["gqa"]["eval_file_path"]
+    img_path = cfg.evaluation_datasets_cfg["gqa"]["img_path"]
+    batch_size = cfg.evaluation_datasets_cfg["gqa"]["batch_size"]
+    max_new_tokens = cfg.evaluation_datasets_cfg["gqa"]["max_new_tokens"]
+
+    gqa = json.load(open(eval_file_path))
+    data = GQAEvalData(gqa, vis_processor, img_path)
+    eval_dataloader = DataLoader(data, batch_size=batch_size, shuffle=False)
+    count=0
+    total=0
+    minigpt4_predict = []
+    for images, texts, labels in tqdm(eval_dataloader):
+        texts = prepare_texts(texts, conv_temp)  # warp the texts with conversation template
+        answers = model.generate(images, texts, max_new_tokens=max_new_tokens, do_sample=False)
+
+        for answer, label in zip(answers, labels):
+            result = dict()
+            result['pred'] = answer.lower().replace('<unk>','').strip()
+            result['gt'] = label
+            minigpt4_predict.append(result)
+            if answer.lower() == label:
+                count+=1
+            total+=1
+    print('gqa val:', count / total * 100, flush=True)
+
+    file_save_path = os.path.join(save_path, "gqa.json")
+    with open(file_save_path,'w') as f:
+        json.dump(minigpt4_predict, f)
+
+if 'vsr' in args.dataset:
+
+    img_path = cfg.evaluation_datasets_cfg["vsr"]["img_path"]
+    batch_size = cfg.evaluation_datasets_cfg["vsr"]["batch_size"]
+    max_new_tokens = cfg.evaluation_datasets_cfg["vsr"]["max_new_tokens"]
+
+    annotation = load_dataset("cambridgeltl/vsr_zeroshot", split='test')
+    data = VSREvalData(annotation, vis_processor, img_path)
+    eval_dataloader = DataLoader(data, batch_size=batch_size, shuffle=False)
+    count=0
+    total=0
+
+    minigpt4_predict = []
+
+    for images, texts, labels in tqdm(eval_dataloader):
+        texts = prepare_texts(texts, conv_temp)  # warp the texts with conversation template
+        answers = model.generate(images, texts, max_new_tokens=max_new_tokens, do_sample=False)
+
+        for answer, label in zip(answers, labels):
+            result = dict()
+            result['pred'] = answer.replace('<unk>','').strip()
+            result['gt'] = label
+            minigpt4_predict.append(result)
+            if answer.lower() ==  label.lower():
+                count+=1
+            total+=1
+    print('vsr test:', count / total * 100, flush=True)
+    file_save_path = os.path.join(save_path,"vsr.json")
+    with open(file_save_path,'w') as f:
+        json.dump(minigpt4_predict, f)
+
+if 'hm' in args.dataset:
+
+    eval_file_path = cfg.evaluation_datasets_cfg["hm"]["eval_file_path"]
+    img_path = cfg.evaluation_datasets_cfg["hm"]["img_path"]
+    batch_size = cfg.evaluation_datasets_cfg["hm"]["batch_size"]
+    max_new_tokens = cfg.evaluation_datasets_cfg["hm"]["max_new_tokens"]
+
+    annotation = []
+    with open(eval_file_path, 'r') as jsonl_file:
+        for line in jsonl_file:
+            json_obj = json.loads(line)
+            annotation.append(json_obj)
+
+    data = HMEvalData(annotation, vis_processor, img_path)
+    eval_dataloader = DataLoader(data, batch_size=batch_size, shuffle=False)
+    count=0
+    total=0
+
+    minigpt4_predict = []
+
+    for images, texts, labels in tqdm(eval_dataloader):
+        texts = prepare_texts(texts, conv_temp)  # warp the texts with conversation template
+        
+        answers = model.generate(images, texts, max_new_tokens=max_new_tokens, do_sample=False)
+
+        for answer, label in zip(answers, labels):
+            result = dict()
+            if answer.lower().strip() =="yes":
+                answer=1
+            elif answer.lower().strip()=="no":
+                answer=0
+            else:
+                print("non-matching answer",answer)
+
+            result['pred'] = answer
+            result['gt'] = int(label)
+            minigpt4_predict.append(result)
+            if answer == label:
+                count+=1
+            total+=1
+
+    print('hm val:', count / total * 100, flush=True)
+    file_save_path = os.path.join(save_path, "hm.json")
+    with open(file_save_path,'w') as f:
+        json.dump(minigpt4_predict, f)