init

app.py

@@ -2,34 +2,123 @@ from transformers import Blip2Processor, Blip2ForConditionalGeneration
 from peft import LoraConfig, get_peft_model, PeftModel
 import torch
 import streamlit as st
+from PIL import Image
+from streamlit_chat import message
+from io import BytesIO, StringIO
+# device = "cuda" if torch.cuda.is_available() else "cpu"
+device = "cpu"
+@st.cache_resource
+def load_model():
+    config = LoraConfig(
+        r=16,
+        lora_alpha=32,
+        lora_dropout=0.05,
+        bias="none",
+    )
 
-device = "cuda" if torch.cuda.is_available() else "cpu"
+    model_name = "./blip2_fakenews_all"
+    #
+    processor = Blip2Processor.from_pretrained("Salesforce/blip2-flan-t5-xl")
+    # device_map = {"": 0}
+    device_map = "auto"
+    model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-flan-t5-xl",
+                                                          load_in_8bit=True,
+                                                          device_map=device_map)
+    model = PeftModel.from_pretrained(model, model_name)
+    model = get_peft_model(model, config)
+    return processor, model
 
-config = LoraConfig(
-    r=16,
-    lora_alpha=32,
-    lora_dropout=0.05,
-    bias="none",
-)
 
-# model_name = "/home/yejiang/blip2_fakedit/saved_model/blip2_fakenews_all"
-#
-# processor = Blip2Processor.from_pretrained("Salesforce/blip2-flan-t5-xl")
-# device_map = {"": 0}
-# model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-flan-t5-xl",
-#                                                       load_in_8bit=True,
-#                                                       device_map=device_map)
-# model = PeftModel.from_pretrained(model, model_name)
-# model = get_peft_model(model, config)
+st.title('Blip2 Fake News Debunker')
 
+if 'generated' not in st.session_state:
+    st.session_state['generated'] = []
+
+if 'past' not in st.session_state:
+    st.session_state['past'] = []
+
+if 'bot_prompt' not in st.session_state:
+    st.session_state.bot_prompt = []
+
+
+def get_text():
+    chat = st.text_input('Start to chat:', placeholder="Hello! Let's start to chat from here! ")
+    return chat
+
+def generate_output(image, prompt):
+    encoding = processor(images=image, text=prompt, max_length=512, truncation=True,
+                         padding="max_length", return_tensors="pt")
+    predictions = model.generate(input_ids=encoding['input_ids'].to(device),
+                                 pixel_values=encoding['pixel_values'].to(device, torch.float16),
+                                 max_length=20)
+    p = processor.batch_decode(predictions, skip_special_tokens=True)
+    out = " ".join(p)
+    return out
+
+if st.button('Start a new chat'):
+    st.cache_resource.clear()
+    st.cache_data.clear()
+    for key in st.session_state.keys():
+        del st.session_state[key]
+    st.experimental_rerun()
 
 col1, col2 = st.columns(2)
+show_file = st.empty()
+
 with col1:
-    chat = st.text_input("Chat",
-                     label_visibility=st.session_state.visibility,
-                     disabled=st.session_state.disabled,
-                     placeholder=st.session_state.placeholder)
+    st.markdown("Step 1: ")
+    uploaded_file = st.file_uploader("Upload a news image here: ", type=["png", "jpg"])
+
+    if not uploaded_file:
+        show_file.info("Please upload a file of type: " + ", ".join(["png", "jpg"]))
+    if isinstance(uploaded_file, BytesIO):
+            image = Image.open(uploaded_file)
+            st.image(image)
+
+
+with col2:
+    st.markdown("Step 2: ")
+    txt = st.text_area("Paste news content here: ")
+    st.markdown("Step 3: ")
+    user_input = get_text()
+    # if user_input:
+    #     st.write("You: ", user_input)
+
+processor, model = load_model()
+def main():
+    if uploaded_file and user_input:
+        prompt = "Qustions: What is this news about? " \
+                 "\nAnswer: " + txt  + \
+                 "\nQustions: " + user_input
+
+        if len(st.session_state.bot_prompt) == 0:
+            pr: list = prompt.split('\n')
+            pr = [p for p in pr if len(p)]  # remove empty string
+            st.session_state.bot_prompt = pr
+            print(f'init: {st.session_state.bot_prompt}')
+
+        if user_input:
+            st.session_state.bot_prompt.append(f'You: {user_input}')
+
+            # Convert a list of prompts to a string for the GPT bot.
+            input_prompt: str = '\n'.join(st.session_state.bot_prompt)
+            print(f'bot prompt input list:\n{st.session_state.bot_prompt}')
+            print(f'bot prompt input string:\n{input_prompt}')
+
+            output = generate_output(image, prompt=input_prompt)
+
+            st.session_state.past.append(user_input)
+            st.session_state.generated.append(output)
+
+            # Add bot response for next prompt.
+            st.session_state.bot_prompt.append(f'Answer: {output}')
+        with col2:
+            if st.session_state['generated']:
+                for i in range(len(st.session_state['generated']) - 1, -1, -1):
+                    message(st.session_state["generated"][i], key=str(i))
+                    message(st.session_state['past'][i], is_user=True, key=str(i) + '_user')
+
 
-    if chat:
-        st.write("You entered: ", chat)
 
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -1,312 +0,0 @@
-absl-py==1.2.0
-accelerate==0.17.0
-aiofiles==22.1.0
-aiohttp @ file:///home/conda/feedstock_root/build_artifacts/aiohttp_1649013154501/work
-aiosignal @ file:///home/conda/feedstock_root/build_artifacts/aiosignal_1667935791922/work
-aiostream==0.4.5
-altair==4.2.0
-antlr4-python3-runtime==4.9.3
-anyio==3.6.1
-arxiv==1.4.7
-asttokens==2.2.1
-async-timeout @ file:///home/conda/feedstock_root/build_artifacts/async-timeout_1640026696943/work
-attrs @ file:///home/conda/feedstock_root/build_artifacts/attrs_1671632566681/work
-backcall==0.2.0
-beautifulsoup4==4.7.1
-bitsandbytes==0.37.1
-blinker==1.5
-blis==0.7.9
-blobfile==2.0.2
-boto3==1.26.13
-botocore==1.29.13
-Bottleneck @ file:///opt/conda/conda-bld/bottleneck_1657175564434/work
-braceexpand==0.1.7
-brotlipy==0.7.0
-cached-property==1.5.2
-cachetools==5.2.0
-catalogue==2.0.8
-certifi==2022.12.7
-cffi @ file:///opt/conda/conda-bld/cffi_1642701102775/work
-cfgv==3.3.1
-chardet==3.0.4
-charset-normalizer @ file:///tmp/build/80754af9/charset-normalizer_1630003229654/work
-chex==0.1.6
-click==8.1.3
-clip @ git+https://github.com/openai/CLIP.git@d50d76daa670286dd6cacf3bcd80b5e4823fc8e1
-clip-client==0.6.2
-clip-server==0.6.2
-colorama @ file:///home/conda/feedstock_root/build_artifacts/colorama_1666700638685/work
-comet-ml==3.12.2
-commonmark==0.9.1
-confection==0.0.4
-configobj==5.0.8
-contexttimer==0.3.3
-contourpy @ file:///opt/conda/conda-bld/contourpy_1663827406301/work
-cryptography @ file:///tmp/build/80754af9/cryptography_1652101588893/work
-cssselect==1.0.3
-cycler @ file:///tmp/build/80754af9/cycler_1637851556182/work
-cymem==2.0.7
-dataclasses @ file:///home/conda/feedstock_root/build_artifacts/dataclasses_1628958434797/work
-datasets==2.8.0
-decorator==5.1.1
-decord==0.6.0
-deepspeed==0.8.3
-dill @ file:///home/conda/feedstock_root/build_artifacts/dill_1666603105584/work
-distlib==0.3.6
-dm-tree==0.1.8
-docarray==0.16.3
-docker==6.0.0
-docker-pycreds==0.4.0
-dulwich==0.21.3
-einops==0.6.0
-en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1-py3-none-any.whl
-entrypoints==0.4
-etils==1.0.0
-evaluate==0.4.0
-everett==3.1.0
-exceptiongroup==1.1.0
-executing==1.2.0
-fairscale==0.4.4
-fastapi==0.82.0
-feedfinder2==0.0.4
-feedparser==5.2.1
-ffmpy==0.3.0
-filelock @ file:///home/conda/feedstock_root/build_artifacts/filelock_1672354931606/work
-Flask==1.0.2
-Flask-Cors==3.0.7
-flax==0.6.6
-fonttools==4.37.1
-frozenlist @ file:///croot/frozenlist_1670004507010/work
-fsspec @ file:///home/conda/feedstock_root/build_artifacts/fsspec_1668082755814/work
-ftfy==6.1.1
-gdown==4.5.1
-gitdb==4.0.10
-GitPython==3.1.30
-google-auth==2.9.1
-google-auth-oauthlib==0.4.6
-gradio==3.25.0
-gradio_client==0.1.3
-grpcio==1.47.0
-grpcio-health-checking==1.47.0
-grpcio-reflection==1.47.0
-h11==0.13.0
-hjson==3.1.0
-httpcore==0.17.0
-httptools==0.4.0
-httpx==0.24.0
-huggingface-hub==0.13.4
-identify==2.5.18
-idna @ file:///croot/idna_1666125576474/work
-imageio==2.22.4
-imbalanced-learn==0.10.1
-importlib-metadata @ file:///home/conda/feedstock_root/build_artifacts/importlib-metadata_1672612343532/work
-importlib-resources==5.12.0
-iniconfig==2.0.0
-iopath==0.1.10
-ipython==8.11.0
-itsdangerous==1.1.0
-jax==0.4.5
-jaxlib==0.4.4
-jcloud==0.0.35
-jedi==0.18.2
-jieba3k==0.35.1
-jina==3.8.3
-jina-hubble-sdk==0.15.5
-Jinja2==3.1.2
-jmespath==1.0.1
-joblib==1.2.0
-jsonschema==4.17.3
-kaggle==1.5.13
-kiwisolver==1.4.4
-langcodes==3.3.0
-langdetect==1.0.7
-latex2mathml==3.75.2
-linkify-it-py==2.0.0
-lxml==4.9.1
-lz4==4.0.2
-Markdown==3.4.1
-markdown-it-py==2.2.0
-MarkupSafe==2.1.1
-matplotlib @ file:///croot/matplotlib-suite_1670466153205/work
-matplotlib-inline==0.1.6
-mdit-py-plugins==0.3.3
-mdtex2html==1.2.0
-mdurl==0.1.2
-mkl-fft==1.3.1
-mkl-random @ file:///tmp/build/80754af9/mkl_random_1626186066731/work
-mkl-service==2.4.0
-msgpack==1.0.4
-multidict @ file:///croot/multidict_1665674239670/work
-multiprocess==0.70.13
-munkres==1.1.4
-murmurhash==1.0.9
-networkx==2.8.8
-newspaper3k==0.2.8
-ninja==1.11.1
-nltk==3.4
-nodeenv==1.7.0
-numexpr @ file:///croot/numexpr_1668713893690/work
-numpy @ file:///opt/conda/conda-bld/numpy_and_numpy_base_1652801679809/work
-nvidia-ml-py3==7.352.0
-oauthlib==3.0.1
-omegaconf==2.3.0
-open-clip-torch==1.3.0
-openai==0.27.0
-opencv-python-headless==4.5.5.64
-opendatasets==0.1.22
-openprompt==1.0.1
-opt-einsum==3.3.0
-optax==0.1.4
-orbax==0.1.3
-orjson==3.8.10
-packaging @ file:///tmp/build/80754af9/packaging_1637314298585/work
-pandas==1.5.2
-parso==0.8.3
-pathspec==0.10.1
-pathtools==0.1.2
-pathy==0.10.1
-peft @ git+https://github.com/huggingface/peft.git@4fd374e80d670781c0d82c96ce94d1215ff23306
-pexpect==4.8.0
-pickleshare==0.7.5
-Pillow==9.4.0
-platformdirs==3.1.0
-plotly==5.13.1
-pluggy==1.0.0
-portalocker==2.7.0
-pre-commit==3.1.1
-preshed==3.0.8
-prometheus-client==0.14.1
-promise==2.3
-prompt-toolkit==3.0.38
-protobuf==3.19.4
-psutil==5.9.4
-ptyprocess==0.7.0
-pure-eval==0.2.2
-py-cpuinfo==9.0.0
-pyarrow==8.0.0
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pybase64==1.2.3
-pycocoevalcap==1.2
-pycocotools==2.0.6
-pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
-pycryptodome==3.17
-pycryptodomex==3.17
-pydantic==1.10.2
-pydeck==0.8.0
-pydub==0.25.1
-Pygments==2.13.0
-Pympler==1.0.1
-PyMuPDF==1.21.1
-pyOpenSSL @ file:///opt/conda/conda-bld/pyopenssl_1643788558760/work
-pyparsing @ file:///opt/conda/conda-bld/pyparsing_1661452539315/work
-pyrsistent==0.19.3
-PySocks @ file:///tmp/build/80754af9/pysocks_1605305812635/work
-pytest==7.2.0
-python-dateutil @ file:///tmp/build/80754af9/python-dateutil_1626374649649/work
-python-docx==0.8.11
-python-dotenv==0.18.0
-python-magic==0.4.27
-python-markdown-math==0.8
-python-multipart==0.0.5
-python-slugify==8.0.1
-pytorch-pretrained-bert==0.6.2
-pytorch-transformers==1.0.0
-pytz @ file:///opt/conda/conda-bld/pytz_1654762638606/work
-pytz-deprecation-shim==0.1.0.post0
-PyWavelets==1.4.1
-pywebarchive==0.5.0
-PyYAML @ file:///home/conda/feedstock_root/build_artifacts/pyyaml_1648757097602/work
-regex==2022.7.25
-requests @ file:///opt/conda/conda-bld/requests_1657734628632/work
-requests-file==1.4.3
-requests-oauthlib==1.2.0
-requests-toolbelt==0.10.1
-responses==0.10.15
-rich==12.5.1
-rouge==1.0.0
-rouge-score==0.1.2
-rsa==4.9
-s3transfer==0.6.0
-salesforce-lavis==1.0.0
-scikit-image==0.19.3
-scikit-learn==1.1.2
-scipy==1.9.1
-seaborn @ file:///croot/seaborn_1669627814970/work
-semantic-version==2.10.0
-semver==2.13.0
-sentencepiece==0.1.96
-sentry-sdk==1.12.1
-seqeval==1.2.2
-setproctitle==1.3.2
-shortuuid==1.0.11
-simpletransformers==0.63.9
-singledispatch==3.4.0.3
-six @ file:///tmp/build/80754af9/six_1644875935023/work
-skorch==0.12.1
-smart-open==6.3.0
-smmap==5.0.0
-sniffio==1.3.0
-soupsieve==1.8
-spacy==3.4.4
-spacy-langdetect==0.1.2
-spacy-legacy==3.0.11
-spacy-loggers==1.0.4
-srsly==2.4.5
-stack-data==0.6.2
-starlette==0.19.1
-streamlit==1.16.0
-tabulate==0.9.0
-tenacity==8.2.2
-tensorboard==2.9.1
-tensorboard-data-server==0.6.1
-tensorboard-plugin-wit==1.8.1
-tensorboardX==2.5.1
-tensorstore==0.1.33
-text-unidecode==1.3
-thinc==8.1.6
-threadpoolctl==3.1.0
-tifffile==2022.10.10
-tiktoken==0.4.0
-timm==0.4.12
-tinysegmenter==0.3
-tldextract==2.2.1
-tokenizers==0.12.1
-toml==0.10.2
-tomli==2.0.1
-toolz==0.12.0
-torch==1.13.1
-torchaudio==0.13.1
-torchvision==0.14.1
-tornado==6.2
-tqdm @ file:///home/conda/feedstock_root/build_artifacts/tqdm_1662214488106/work
-traitlets==5.9.0
-transformers @ git+https://github.com/huggingface/transformers.git@a9bd5df16a46356463f2712dd8f6c109fa83d6f9
-twython==3.7.0
-typer==0.7.0
-typing_extensions==4.5.0
-tzdata==2022.7
-tzlocal==4.2
-uc-micro-py==1.0.1
-urllib3 @ file:///croot/urllib3_1670526988650/work
-uvicorn==0.18.3
-uvloop==0.16.0
-validators==0.20.0
-virtualenv==20.20.0
-wandb==0.13.7
-wasabi==0.10.1
-watchdog==2.2.1
-watchfiles==0.16.1
-waybackpy==3.0.6
-wcwidth==0.2.5
-webdataset==0.2.35
-websocket-client==1.4.1
-websockets==10.3
-Werkzeug==2.2.2
-wrapt==1.15.0
-wurlitzer==3.0.3
-xmlx==2.0.0
-xxhash==3.0.0
-yacs==0.1.8
-yarl @ file:///home/conda/feedstock_root/build_artifacts/yarl_1648966524636/work
-zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1669453021653/work

test.py

@@ -0,0 +1,35 @@
+from PIL import Image
+import requests
+from transformers import Blip2Processor, Blip2ForConditionalGeneration
+import torch
+from peft import LoraConfig, get_peft_model, PeftModel
+
+config = LoraConfig(
+    r=16,
+    lora_alpha=32,
+    lora_dropout=0.05,
+    bias="none",
+)
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model_name = "./blip2_fakenews_all"
+
+# device_map = {"": 0}
+device_map = "auto"
+processor = Blip2Processor.from_pretrained("blip2")
+model = Blip2ForConditionalGeneration.from_pretrained("blip2",
+                                                      load_in_8bit=True,
+                                                      device_map=device_map)
+model = PeftModel.from_pretrained(model, model_name)
+model = get_peft_model(model, config)
+
+
+url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+image = Image.open(requests.get(url, stream=True).raw)
+
+prompt = "Question: Is this real or fake? Answer: real. Question: Why? "
+inputs = processor(images=image, text=prompt, return_tensors="pt").to(device, torch.float16)
+
+generated_ids = model.generate(**inputs)
+generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
+print(generated_text)