Add files using upload-large-folder tool
Browse files- .gitattributes +7 -0
- README.md +501 -0
- added_tokens.json +3 -0
- config.json +36 -0
- generation_config.json +8 -0
- model-00001-of-00005.safetensors +3 -0
- model-00002-of-00005.safetensors +3 -0
- model-00003-of-00005.safetensors +3 -0
- model-00004-of-00005.safetensors +3 -0
- model-00005-of-00005.safetensors +3 -0
- model.safetensors.index.json +0 -0
- preprocessor_config.json +29 -0
- processor_config.json +4 -0
- special_tokens_map.json +33 -0
- tokenizer.json +3 -0
- tokenizer.model +3 -0
- tokenizer_config.json +0 -0
.gitattributes
CHANGED
@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
37 |
+
model-00001-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text
|
38 |
+
model-00002-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text
|
39 |
+
model-00003-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text
|
40 |
+
model-00004-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text
|
41 |
+
model-00005-of-00005.safetensors filter=lfs diff=lfs merge=lfs -text
|
42 |
+
tokenizer.model filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
@@ -0,0 +1,501 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
license: gemma
|
3 |
+
library_name: transformers
|
4 |
+
pipeline_tag: image-text-to-text
|
5 |
+
extra_gated_heading: Access Gemma on Hugging Face
|
6 |
+
extra_gated_prompt: >-
|
7 |
+
To access Gemma on Hugging Face, you’re required to review and agree to
|
8 |
+
Google’s usage license. To do this, please ensure you’re logged in to Hugging
|
9 |
+
Face and click below. Requests are processed immediately.
|
10 |
+
extra_gated_button_content: Acknowledge license
|
11 |
+
base_model: google/gemma-3-12b
|
12 |
+
---
|
13 |
+
|
14 |
+
# Gemma 3 model card
|
15 |
+
|
16 |
+
**Model Page**: [Gemma](https://ai.google.dev/gemma/docs/core)
|
17 |
+
|
18 |
+
**Resources and Technical Documentation**:
|
19 |
+
|
20 |
+
* [Gemma 3 Technical Report][g3-tech-report]
|
21 |
+
* [Responsible Generative AI Toolkit][rai-toolkit]
|
22 |
+
* [Gemma on Kaggle][kaggle-gemma]
|
23 |
+
* [Gemma on Vertex Model Garden][vertex-mg-gemma3]
|
24 |
+
|
25 |
+
**Terms of Use**: [Terms][terms]
|
26 |
+
|
27 |
+
**Authors**: Google DeepMind
|
28 |
+
|
29 |
+
## Model Information
|
30 |
+
|
31 |
+
Summary description and brief definition of inputs and outputs.
|
32 |
+
|
33 |
+
### Description
|
34 |
+
|
35 |
+
Gemma is a family of lightweight, state-of-the-art open models from Google,
|
36 |
+
built from the same research and technology used to create the Gemini models.
|
37 |
+
Gemma 3 models are multimodal, handling text and image input and generating text
|
38 |
+
output, with open weights for both pre-trained variants and instruction-tuned
|
39 |
+
variants. Gemma 3 has a large, 128K context window, multilingual support in over
|
40 |
+
140 languages, and is available in more sizes than previous versions. Gemma 3
|
41 |
+
models are well-suited for a variety of text generation and image understanding
|
42 |
+
tasks, including question answering, summarization, and reasoning. Their
|
43 |
+
relatively small size makes it possible to deploy them in environments with
|
44 |
+
limited resources such as laptops, desktops or your own cloud infrastructure,
|
45 |
+
democratizing access to state of the art AI models and helping foster innovation
|
46 |
+
for everyone.
|
47 |
+
|
48 |
+
### Inputs and outputs
|
49 |
+
|
50 |
+
- **Input:**
|
51 |
+
- Text string, such as a question, a prompt, or a document to be summarized
|
52 |
+
- Images, normalized to 896 x 896 resolution and encoded to 256 tokens
|
53 |
+
each
|
54 |
+
- Total input context of 128K tokens for the 4B, 12B, and 27B sizes, and
|
55 |
+
32K tokens for the 1B size
|
56 |
+
|
57 |
+
- **Output:**
|
58 |
+
- Generated text in response to the input, such as an answer to a
|
59 |
+
question, analysis of image content, or a summary of a document
|
60 |
+
- Total output context of 8192 tokens
|
61 |
+
|
62 |
+
### Usage
|
63 |
+
|
64 |
+
Below, there are some code snippets on how to get quickly started with running the model. First, install the Transformers library with the version made for Gemma 3:
|
65 |
+
|
66 |
+
```sh
|
67 |
+
|
68 |
+
$ pip install git+https://github.com/huggingface/[email protected]
|
69 |
+
|
70 |
+
```
|
71 |
+
|
72 |
+
Then, copy the snippet from the section that is relevant for your use case.
|
73 |
+
|
74 |
+
#### Running with the `pipeline` API
|
75 |
+
|
76 |
+
```python
|
77 |
+
from transformers import pipeline
|
78 |
+
import torch
|
79 |
+
|
80 |
+
pipe = pipeline(
|
81 |
+
"image-text-to-text",
|
82 |
+
model="google/gemma-3-12b-pt",
|
83 |
+
device="cuda",
|
84 |
+
torch_dtype=torch.bfloat16
|
85 |
+
)
|
86 |
+
|
87 |
+
output = pipe(
|
88 |
+
"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg",
|
89 |
+
text="<start_of_image> in this image, there is"
|
90 |
+
)
|
91 |
+
|
92 |
+
print(output)
|
93 |
+
# [{'input_text': '<start_of_image> in this image, there is',
|
94 |
+
# 'generated_text': '<start_of_image> in this image, there is a bumblebee on a pink flower.\n\n'}]
|
95 |
+
```
|
96 |
+
|
97 |
+
#### Running the model on a single / multi GPU
|
98 |
+
|
99 |
+
```python
|
100 |
+
# pip install accelerate
|
101 |
+
|
102 |
+
from transformers import AutoProcessor, Gemma3ForConditionalGeneration
|
103 |
+
from PIL import Image
|
104 |
+
import requests
|
105 |
+
import torch
|
106 |
+
|
107 |
+
model_id = "google/gemma-3-12b-pt"
|
108 |
+
|
109 |
+
url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg"
|
110 |
+
image = Image.open(requests.get(url, stream=True).raw)
|
111 |
+
|
112 |
+
model = Gemma3ForConditionalGeneration.from_pretrained(model_id).eval()
|
113 |
+
processor = AutoProcessor.from_pretrained(model_id)
|
114 |
+
|
115 |
+
prompt = "<start_of_image> in this image, there is"
|
116 |
+
model_inputs = processor(text=prompt, images=image, return_tensors="pt")
|
117 |
+
|
118 |
+
input_len = model_inputs["input_ids"].shape[-1]
|
119 |
+
|
120 |
+
with torch.inference_mode():
|
121 |
+
generation = model.generate(**model_inputs, max_new_tokens=100, do_sample=False)
|
122 |
+
generation = generation[0][input_len:]
|
123 |
+
|
124 |
+
decoded = processor.decode(generation, skip_special_tokens=True)
|
125 |
+
print(decoded)
|
126 |
+
```
|
127 |
+
|
128 |
+
### Citation
|
129 |
+
|
130 |
+
```none
|
131 |
+
@article{gemma_2025,
|
132 |
+
title={Gemma 3},
|
133 |
+
url={https://goo.gle/Gemma3Report},
|
134 |
+
publisher={Kaggle},
|
135 |
+
author={Gemma Team},
|
136 |
+
year={2025}
|
137 |
+
}
|
138 |
+
```
|
139 |
+
|
140 |
+
## Model Data
|
141 |
+
|
142 |
+
Data used for model training and how the data was processed.
|
143 |
+
|
144 |
+
### Training Dataset
|
145 |
+
|
146 |
+
These models were trained on a dataset of text data that includes a wide variety
|
147 |
+
of sources. The 27B model was trained with 14 trillion tokens, the 12B model was
|
148 |
+
trained with 12 trillion tokens, 4B model was trained with 4 trillion tokens and
|
149 |
+
1B with 2 trillion tokens. Here are the key components:
|
150 |
+
|
151 |
+
- Web Documents: A diverse collection of web text ensures the model is
|
152 |
+
exposed to a broad range of linguistic styles, topics, and vocabulary. The
|
153 |
+
training dataset includes content in over 140 languages.
|
154 |
+
- Code: Exposing the model to code helps it to learn the syntax and
|
155 |
+
patterns of programming languages, which improves its ability to generate
|
156 |
+
code and understand code-related questions.
|
157 |
+
- Mathematics: Training on mathematical text helps the model learn logical
|
158 |
+
reasoning, symbolic representation, and to address mathematical queries.
|
159 |
+
- Images: A wide range of images enables the model to perform image
|
160 |
+
analysis and visual data extraction tasks.
|
161 |
+
|
162 |
+
The combination of these diverse data sources is crucial for training a powerful
|
163 |
+
multimodal model that can handle a wide variety of different tasks and data
|
164 |
+
formats.
|
165 |
+
|
166 |
+
### Data Preprocessing
|
167 |
+
|
168 |
+
Here are the key data cleaning and filtering methods applied to the training
|
169 |
+
data:
|
170 |
+
|
171 |
+
- CSAM Filtering: Rigorous CSAM (Child Sexual Abuse Material) filtering
|
172 |
+
was applied at multiple stages in the data preparation process to ensure
|
173 |
+
the exclusion of harmful and illegal content.
|
174 |
+
- Sensitive Data Filtering: As part of making Gemma pre-trained models
|
175 |
+
safe and reliable, automated techniques were used to filter out certain
|
176 |
+
personal information and other sensitive data from training sets.
|
177 |
+
- Additional methods: Filtering based on content quality and safety in
|
178 |
+
line with [our policies][safety-policies].
|
179 |
+
|
180 |
+
## Implementation Information
|
181 |
+
|
182 |
+
Details about the model internals.
|
183 |
+
|
184 |
+
### Hardware
|
185 |
+
|
186 |
+
Gemma was trained using [Tensor Processing Unit (TPU)][tpu] hardware (TPUv4p,
|
187 |
+
TPUv5p and TPUv5e). Training vision-language models (VLMS) requires significant
|
188 |
+
computational power. TPUs, designed specifically for matrix operations common in
|
189 |
+
machine learning, offer several advantages in this domain:
|
190 |
+
|
191 |
+
- Performance: TPUs are specifically designed to handle the massive
|
192 |
+
computations involved in training VLMs. They can speed up training
|
193 |
+
considerably compared to CPUs.
|
194 |
+
- Memory: TPUs often come with large amounts of high-bandwidth memory,
|
195 |
+
allowing for the handling of large models and batch sizes during training.
|
196 |
+
This can lead to better model quality.
|
197 |
+
- Scalability: TPU Pods (large clusters of TPUs) provide a scalable
|
198 |
+
solution for handling the growing complexity of large foundation models.
|
199 |
+
You can distribute training across multiple TPU devices for faster and more
|
200 |
+
efficient processing.
|
201 |
+
- Cost-effectiveness: In many scenarios, TPUs can provide a more
|
202 |
+
cost-effective solution for training large models compared to CPU-based
|
203 |
+
infrastructure, especially when considering the time and resources saved
|
204 |
+
due to faster training.
|
205 |
+
- These advantages are aligned with
|
206 |
+
[Google's commitments to operate sustainably][sustainability].
|
207 |
+
|
208 |
+
### Software
|
209 |
+
|
210 |
+
Training was done using [JAX][jax] and [ML Pathways][ml-pathways].
|
211 |
+
|
212 |
+
JAX allows researchers to take advantage of the latest generation of hardware,
|
213 |
+
including TPUs, for faster and more efficient training of large models. ML
|
214 |
+
Pathways is Google's latest effort to build artificially intelligent systems
|
215 |
+
capable of generalizing across multiple tasks. This is specially suitable for
|
216 |
+
foundation models, including large language models like these ones.
|
217 |
+
|
218 |
+
Together, JAX and ML Pathways are used as described in the
|
219 |
+
[paper about the Gemini family of models][gemini-2-paper]; *"the 'single
|
220 |
+
controller' programming model of Jax and Pathways allows a single Python
|
221 |
+
process to orchestrate the entire training run, dramatically simplifying the
|
222 |
+
development workflow."*
|
223 |
+
|
224 |
+
## Evaluation
|
225 |
+
|
226 |
+
Model evaluation metrics and results.
|
227 |
+
|
228 |
+
### Benchmark Results
|
229 |
+
|
230 |
+
These models were evaluated against a large collection of different datasets and
|
231 |
+
metrics to cover different aspects of text generation:
|
232 |
+
|
233 |
+
#### Reasoning and factuality
|
234 |
+
|
235 |
+
| Benchmark | Metric | Gemma 3 PT 1B | Gemma 3 PT 4B | Gemma 3 PT 12B | Gemma 3 PT 27B |
|
236 |
+
| ------------------------------ |----------------|:--------------:|:-------------:|:--------------:|:--------------:|
|
237 |
+
| [HellaSwag][hellaswag] | 10-shot | 62.3 | 77.2 | 84.2 | 85.6 |
|
238 |
+
| [BoolQ][boolq] | 0-shot | 63.2 | 72.3 | 78.8 | 82.4 |
|
239 |
+
| [PIQA][piqa] | 0-shot | 73.8 | 79.6 | 81.8 | 83.3 |
|
240 |
+
| [SocialIQA][socialiqa] | 0-shot | 48.9 | 51.9 | 53.4 | 54.9 |
|
241 |
+
| [TriviaQA][triviaqa] | 5-shot | 39.8 | 65.8 | 78.2 | 85.5 |
|
242 |
+
| [Natural Questions][naturalq] | 5-shot | 9.48 | 20.0 | 31.4 | 36.1 |
|
243 |
+
| [ARC-c][arc] | 25-shot | 38.4 | 56.2 | 68.9 | 70.6 |
|
244 |
+
| [ARC-e][arc] | 0-shot | 73.0 | 82.4 | 88.3 | 89.0 |
|
245 |
+
| [WinoGrande][winogrande] | 5-shot | 58.2 | 64.7 | 74.3 | 78.8 |
|
246 |
+
| [BIG-Bench Hard][bbh] | few-shot | 28.4 | 50.9 | 72.6 | 77.7 |
|
247 |
+
| [DROP][drop] | 1-shot | 42.4 | 60.1 | 72.2 | 77.2 |
|
248 |
+
|
249 |
+
[hellaswag]: https://arxiv.org/abs/1905.07830
|
250 |
+
[boolq]: https://arxiv.org/abs/1905.10044
|
251 |
+
[piqa]: https://arxiv.org/abs/1911.11641
|
252 |
+
[socialiqa]: https://arxiv.org/abs/1904.09728
|
253 |
+
[triviaqa]: https://arxiv.org/abs/1705.03551
|
254 |
+
[naturalq]: https://github.com/google-research-datasets/natural-questions
|
255 |
+
[arc]: https://arxiv.org/abs/1911.01547
|
256 |
+
[winogrande]: https://arxiv.org/abs/1907.10641
|
257 |
+
[bbh]: https://paperswithcode.com/dataset/bbh
|
258 |
+
[drop]: https://arxiv.org/abs/1903.00161
|
259 |
+
|
260 |
+
#### STEM and code
|
261 |
+
|
262 |
+
| Benchmark | Metric | Gemma 3 PT 4B | Gemma 3 PT 12B | Gemma 3 PT 27B |
|
263 |
+
| ------------------------------ |----------------|:-------------:|:--------------:|:--------------:|
|
264 |
+
| [MMLU][mmlu] | 5-shot | 59.6 | 74.5 | 78.6 |
|
265 |
+
| [MMLU][mmlu] (Pro COT) | 5-shot | 29.2 | 45.3 | 52.2 |
|
266 |
+
| [AGIEval][agieval] | 3-5-shot | 42.1 | 57.4 | 66.2 |
|
267 |
+
| [MATH][math] | 4-shot | 24.2 | 43.3 | 50.0 |
|
268 |
+
| [GSM8K][gsm8k] | 8-shot | 38.4 | 71.0 | 82.6 |
|
269 |
+
| [GPQA][gpqa] | 5-shot | 15.0 | 25.4 | 24.3 |
|
270 |
+
| [MBPP][mbpp] | 3-shot | 46.0 | 60.4 | 65.6 |
|
271 |
+
| [HumanEval][humaneval] | 0-shot | 36.0 | 45.7 | 48.8 |
|
272 |
+
|
273 |
+
[mmlu]: https://arxiv.org/abs/2009.03300
|
274 |
+
[agieval]: https://arxiv.org/abs/2304.06364
|
275 |
+
[math]: https://arxiv.org/abs/2103.03874
|
276 |
+
[gsm8k]: https://arxiv.org/abs/2110.14168
|
277 |
+
[gpqa]: https://arxiv.org/abs/2311.12022
|
278 |
+
[mbpp]: https://arxiv.org/abs/2108.07732
|
279 |
+
[humaneval]: https://arxiv.org/abs/2107.03374
|
280 |
+
|
281 |
+
#### Multilingual
|
282 |
+
|
283 |
+
| Benchmark | Gemma 3 PT 1B | Gemma 3 PT 4B | Gemma 3 PT 12B | Gemma 3 PT 27B |
|
284 |
+
| ------------------------------------ |:-------------:|:-------------:|:--------------:|:--------------:|
|
285 |
+
| [MGSM][mgsm] | 2.04 | 34.7 | 64.3 | 74.3 |
|
286 |
+
| [Global-MMLU-Lite][global-mmlu-lite] | 24.9 | 57.0 | 69.4 | 75.7 |
|
287 |
+
| [WMT24++][wmt24pp] (ChrF) | 36.7 | 48.4 | 53.9 | 55.7 |
|
288 |
+
| [FloRes][flores] | 29.5 | 39.2 | 46.0 | 48.8 |
|
289 |
+
| [XQuAD][xquad] (all) | 43.9 | 68.0 | 74.5 | 76.8 |
|
290 |
+
| [ECLeKTic][eclektic] | 4.69 | 11.0 | 17.2 | 24.4 |
|
291 |
+
| [IndicGenBench][indicgenbench] | 41.4 | 57.2 | 61.7 | 63.4 |
|
292 |
+
|
293 |
+
[mgsm]: https://arxiv.org/abs/2210.03057
|
294 |
+
[flores]: https://arxiv.org/abs/2106.03193
|
295 |
+
[xquad]: https://arxiv.org/abs/1910.11856v3
|
296 |
+
[global-mmlu-lite]: https://huggingface.co/datasets/CohereForAI/Global-MMLU-Lite
|
297 |
+
[wmt24pp]: https://arxiv.org/abs/2502.12404v1
|
298 |
+
[eclektic]: https://arxiv.org/abs/2502.21228
|
299 |
+
[indicgenbench]: https://arxiv.org/abs/2404.16816
|
300 |
+
|
301 |
+
#### Multimodal
|
302 |
+
|
303 |
+
| Benchmark | Gemma 3 PT 4B | Gemma 3 PT 12B | Gemma 3 PT 27B |
|
304 |
+
| ------------------------------ |:-------------:|:--------------:|:--------------:|
|
305 |
+
| [COCOcap][coco-cap] | 102 | 111 | 116 |
|
306 |
+
| [DocVQA][docvqa] (val) | 72.8 | 82.3 | 85.6 |
|
307 |
+
| [InfoVQA][info-vqa] (val) | 44.1 | 54.8 | 59.4 |
|
308 |
+
| [MMMU][mmmu] (pt) | 39.2 | 50.3 | 56.1 |
|
309 |
+
| [TextVQA][textvqa] (val) | 58.9 | 66.5 | 68.6 |
|
310 |
+
| [RealWorldQA][realworldqa] | 45.5 | 52.2 | 53.9 |
|
311 |
+
| [ReMI][remi] | 27.3 | 38.5 | 44.8 |
|
312 |
+
| [AI2D][ai2d] | 63.2 | 75.2 | 79.0 |
|
313 |
+
| [ChartQA][chartqa] | 63.6 | 74.7 | 76.3 |
|
314 |
+
| [VQAv2][vqav2] | 63.9 | 71.2 | 72.9 |
|
315 |
+
| [BLINK][blinkvqa] | 38.0 | 35.9 | 39.6 |
|
316 |
+
| [OKVQA][okvqa] | 51.0 | 58.7 | 60.2 |
|
317 |
+
| [TallyQA][tallyqa] | 42.5 | 51.8 | 54.3 |
|
318 |
+
| [SpatialSense VQA][ss-vqa] | 50.9 | 60.0 | 59.4 |
|
319 |
+
| [CountBenchQA][countbenchqa] | 26.1 | 17.8 | 68.0 |
|
320 |
+
|
321 |
+
[coco-cap]: https://cocodataset.org/#home
|
322 |
+
[docvqa]: https://www.docvqa.org/
|
323 |
+
[info-vqa]: https://arxiv.org/abs/2104.12756
|
324 |
+
[mmmu]: https://arxiv.org/abs/2311.16502
|
325 |
+
[textvqa]: https://textvqa.org/
|
326 |
+
[realworldqa]: https://paperswithcode.com/dataset/realworldqa
|
327 |
+
[remi]: https://arxiv.org/html/2406.09175v1
|
328 |
+
[ai2d]: https://allenai.org/data/diagrams
|
329 |
+
[chartqa]: https://arxiv.org/abs/2203.10244
|
330 |
+
[vqav2]: https://visualqa.org/index.html
|
331 |
+
[blinkvqa]: https://arxiv.org/abs/2404.12390
|
332 |
+
[okvqa]: https://okvqa.allenai.org/
|
333 |
+
[tallyqa]: https://arxiv.org/abs/1810.12440
|
334 |
+
[ss-vqa]: https://arxiv.org/abs/1908.02660
|
335 |
+
[countbenchqa]: https://github.com/google-research/big_vision/blob/main/big_vision/datasets/countbenchqa/
|
336 |
+
|
337 |
+
## Ethics and Safety
|
338 |
+
|
339 |
+
Ethics and safety evaluation approach and results.
|
340 |
+
|
341 |
+
### Evaluation Approach
|
342 |
+
|
343 |
+
Our evaluation methods include structured evaluations and internal red-teaming
|
344 |
+
testing of relevant content policies. Red-teaming was conducted by a number of
|
345 |
+
different teams, each with different goals and human evaluation metrics. These
|
346 |
+
models were evaluated against a number of different categories relevant to
|
347 |
+
ethics and safety, including:
|
348 |
+
|
349 |
+
- **Child Safety**: Evaluation of text-to-text and image to text prompts
|
350 |
+
covering child safety policies, including child sexual abuse and
|
351 |
+
exploitation.
|
352 |
+
- **Content Safety:** Evaluation of text-to-text and image to text prompts
|
353 |
+
covering safety policies including, harassment, violence and gore, and hate
|
354 |
+
speech.
|
355 |
+
- **Representational Harms**: Evaluation of text-to-text and image to text
|
356 |
+
prompts covering safety policies including bias, stereotyping, and harmful
|
357 |
+
associations or inaccuracies.
|
358 |
+
|
359 |
+
In addition to development level evaluations, we conduct "assurance
|
360 |
+
evaluations" which are our 'arms-length' internal evaluations for responsibility
|
361 |
+
governance decision making. They are conducted separately from the model
|
362 |
+
development team, to inform decision making about release. High level findings
|
363 |
+
are fed back to the model team, but prompt sets are held-out to prevent
|
364 |
+
overfitting and preserve the results' ability to inform decision making.
|
365 |
+
Assurance evaluation results are reported to our Responsibility & Safety Council
|
366 |
+
as part of release review.
|
367 |
+
|
368 |
+
### Evaluation Results
|
369 |
+
|
370 |
+
For all areas of safety testing, we saw major improvements in the categories of
|
371 |
+
child safety, content safety, and representational harms relative to previous
|
372 |
+
Gemma models. All testing was conducted without safety filters to evaluate the
|
373 |
+
model capabilities and behaviors. For both text-to-text and image-to-text, and
|
374 |
+
across all model sizes, the model produced minimal policy violations, and showed
|
375 |
+
significant improvements over previous Gemma models' performance with respect
|
376 |
+
to ungrounded inferences. A limitation of our evaluations was they included only
|
377 |
+
English language prompts.
|
378 |
+
|
379 |
+
## Usage and Limitations
|
380 |
+
|
381 |
+
These models have certain limitations that users should be aware of.
|
382 |
+
|
383 |
+
### Intended Usage
|
384 |
+
|
385 |
+
Open vision-language models (VLMs) models have a wide range of applications
|
386 |
+
across various industries and domains. The following list of potential uses is
|
387 |
+
not comprehensive. The purpose of this list is to provide contextual information
|
388 |
+
about the possible use-cases that the model creators considered as part of model
|
389 |
+
training and development.
|
390 |
+
|
391 |
+
- Content Creation and Communication
|
392 |
+
- Text Generation: These models can be used to generate creative text
|
393 |
+
formats such as poems, scripts, code, marketing copy, and email drafts.
|
394 |
+
- Chatbots and Conversational AI: Power conversational interfaces
|
395 |
+
for customer service, virtual assistants, or interactive applications.
|
396 |
+
- Text Summarization: Generate concise summaries of a text corpus,
|
397 |
+
research papers, or reports.
|
398 |
+
- Image Data Extraction: These models can be used to extract,
|
399 |
+
interpret, and summarize visual data for text communications.
|
400 |
+
- Research and Education
|
401 |
+
- Natural Language Processing (NLP) and VLM Research: These
|
402 |
+
models can serve as a foundation for researchers to experiment with VLM
|
403 |
+
and NLP techniques, develop algorithms, and contribute to the
|
404 |
+
advancement of the field.
|
405 |
+
- Language Learning Tools: Support interactive language learning
|
406 |
+
experiences, aiding in grammar correction or providing writing practice.
|
407 |
+
- Knowledge Exploration: Assist researchers in exploring large
|
408 |
+
bodies of text by generating summaries or answering questions about
|
409 |
+
specific topics.
|
410 |
+
|
411 |
+
### Limitations
|
412 |
+
|
413 |
+
- Training Data
|
414 |
+
- The quality and diversity of the training data significantly
|
415 |
+
influence the model's capabilities. Biases or gaps in the training data
|
416 |
+
can lead to limitations in the model's responses.
|
417 |
+
- The scope of the training dataset determines the subject areas
|
418 |
+
the model can handle effectively.
|
419 |
+
- Context and Task Complexity
|
420 |
+
- Models are better at tasks that can be framed with clear
|
421 |
+
prompts and instructions. Open-ended or highly complex tasks might be
|
422 |
+
challenging.
|
423 |
+
- A model's performance can be influenced by the amount of context
|
424 |
+
provided (longer context generally leads to better outputs, up to a
|
425 |
+
certain point).
|
426 |
+
- Language Ambiguity and Nuance
|
427 |
+
- Natural language is inherently complex. Models might struggle
|
428 |
+
to grasp subtle nuances, sarcasm, or figurative language.
|
429 |
+
- Factual Accuracy
|
430 |
+
- Models generate responses based on information they learned
|
431 |
+
from their training datasets, but they are not knowledge bases. They
|
432 |
+
may generate incorrect or outdated factual statements.
|
433 |
+
- Common Sense
|
434 |
+
- Models rely on statistical patterns in language. They might
|
435 |
+
lack the ability to apply common sense reasoning in certain situations.
|
436 |
+
|
437 |
+
### Ethical Considerations and Risks
|
438 |
+
|
439 |
+
The development of vision-language models (VLMs) raises several ethical
|
440 |
+
concerns. In creating an open model, we have carefully considered the following:
|
441 |
+
|
442 |
+
- Bias and Fairness
|
443 |
+
- VLMs trained on large-scale, real-world text and image data can
|
444 |
+
reflect socio-cultural biases embedded in the training material. These
|
445 |
+
models underwent careful scrutiny, input data pre-processing described
|
446 |
+
and posterior evaluations reported in this card.
|
447 |
+
- Misinformation and Misuse
|
448 |
+
- VLMs can be misused to generate text that is false, misleading,
|
449 |
+
or harmful.
|
450 |
+
- Guidelines are provided for responsible use with the model, see the
|
451 |
+
[Responsible Generative AI Toolkit][rai-toolkit].
|
452 |
+
- Transparency and Accountability:
|
453 |
+
- This model card summarizes details on the models' architecture,
|
454 |
+
capabilities, limitations, and evaluation processes.
|
455 |
+
- A responsibly developed open model offers the opportunity to
|
456 |
+
share innovation by making VLM technology accessible to developers and
|
457 |
+
researchers across the AI ecosystem.
|
458 |
+
|
459 |
+
Risks identified and mitigations:
|
460 |
+
|
461 |
+
- **Perpetuation of biases**: It's encouraged to perform continuous
|
462 |
+
monitoring (using evaluation metrics, human review) and the exploration of
|
463 |
+
de-biasing techniques during model training, fine-tuning, and other use
|
464 |
+
cases.
|
465 |
+
- **Generation of harmful content**: Mechanisms and guidelines for content
|
466 |
+
safety are essential. Developers are encouraged to exercise caution and
|
467 |
+
implement appropriate content safety safeguards based on their specific
|
468 |
+
product policies and application use cases.
|
469 |
+
- **Misuse for malicious purposes**: Technical limitations and developer
|
470 |
+
and end-user education can help mitigate against malicious applications of
|
471 |
+
VLMs. Educational resources and reporting mechanisms for users to flag
|
472 |
+
misuse are provided. Prohibited uses of Gemma models are outlined in the
|
473 |
+
[Gemma Prohibited Use Policy][prohibited-use].
|
474 |
+
- **Privacy violations**: Models were trained on data filtered for removal
|
475 |
+
of certain personal information and other sensitive data. Developers are
|
476 |
+
encouraged to adhere to privacy regulations with privacy-preserving
|
477 |
+
techniques.
|
478 |
+
|
479 |
+
### Benefits
|
480 |
+
|
481 |
+
At the time of release, this family of models provides high-performance open
|
482 |
+
vision-language model implementations designed from the ground up for
|
483 |
+
responsible AI development compared to similarly sized models.
|
484 |
+
|
485 |
+
Using the benchmark evaluation metrics described in this document, these models
|
486 |
+
have shown to provide superior performance to other, comparably-sized open model
|
487 |
+
alternatives.
|
488 |
+
|
489 |
+
[g3-tech-report]: https://goo.gle/Gemma3Report
|
490 |
+
[rai-toolkit]: https://ai.google.dev/responsible
|
491 |
+
[kaggle-gemma]: https://www.kaggle.com/models/google/gemma-3
|
492 |
+
[vertex-mg-gemma3]: https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/gemma3
|
493 |
+
[terms]: https://ai.google.dev/gemma/terms
|
494 |
+
[safety-policies]: https://ai.google/static/documents/ai-responsibility-update-published-february-2025.pdf
|
495 |
+
[prohibited-use]: https://ai.google.dev/gemma/prohibited_use_policy
|
496 |
+
[tpu]: https://cloud.google.com/tpu/docs/intro-to-tpu
|
497 |
+
[sustainability]: https://sustainability.google/operating-sustainably/
|
498 |
+
[jax]: https://github.com/jax-ml/jax
|
499 |
+
[ml-pathways]: https://blog.google/technology/ai/introducing-pathways-next-generation-ai-architecture/
|
500 |
+
[sustainability]: https://sustainability.google/operating-sustainably/
|
501 |
+
[gemini-2-paper]: https://arxiv.org/abs/2312.11805
|
added_tokens.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"<image_soft_token>": 262144
|
3 |
+
}
|
config.json
ADDED
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"architectures": [
|
3 |
+
"Gemma3ForConditionalGeneration"
|
4 |
+
],
|
5 |
+
"boi_token_index": 255999,
|
6 |
+
"eoi_token_index": 256000,
|
7 |
+
"image_token_index": 262144,
|
8 |
+
"initializer_range": 0.02,
|
9 |
+
"mm_tokens_per_image": 256,
|
10 |
+
"model_type": "gemma3",
|
11 |
+
"text_config": {
|
12 |
+
"hidden_size": 3840,
|
13 |
+
"intermediate_size": 15360,
|
14 |
+
"model_type": "gemma3_text",
|
15 |
+
"num_attention_heads": 16,
|
16 |
+
"num_hidden_layers": 48,
|
17 |
+
"num_key_value_heads": 8,
|
18 |
+
"rope_scaling": {
|
19 |
+
"factor": 8.0,
|
20 |
+
"rope_type": "linear"
|
21 |
+
},
|
22 |
+
"sliding_window": 1024
|
23 |
+
},
|
24 |
+
"torch_dtype": "bfloat16",
|
25 |
+
"transformers_version": "4.50.0.dev0",
|
26 |
+
"vision_config": {
|
27 |
+
"hidden_size": 1152,
|
28 |
+
"image_size": 896,
|
29 |
+
"intermediate_size": 4304,
|
30 |
+
"model_type": "siglip_vision_model",
|
31 |
+
"num_attention_heads": 16,
|
32 |
+
"num_hidden_layers": 27,
|
33 |
+
"patch_size": 14,
|
34 |
+
"vision_use_head": false
|
35 |
+
}
|
36 |
+
}
|
generation_config.json
ADDED
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_from_model_config": true,
|
3 |
+
"bos_token_id": 2,
|
4 |
+
"cache_implementation": "hybrid",
|
5 |
+
"eos_token_id": 1,
|
6 |
+
"pad_token_id": 0,
|
7 |
+
"transformers_version": "4.50.0.dev0"
|
8 |
+
}
|
model-00001-of-00005.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:11edde44c55ae7a0ac6d284c0f37a1f8c0622b42b6a4d1d3d268d7671083b270
|
3 |
+
size 4979902192
|
model-00002-of-00005.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:69846170c3ea1d9f5f60fad89b9459f5f02ddec4155d7083d22c74bbe2f19caa
|
3 |
+
size 4931296592
|
model-00003-of-00005.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a069b463ee896af6ba10a1f88626cf8e76854eb24f86815bf8c02147229a0b45
|
3 |
+
size 4931296656
|
model-00004-of-00005.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a337c8d5775a1481659461714f30a0257f4002a2d497b02a560203fe3539200a
|
3 |
+
size 4931296656
|
model-00005-of-00005.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0078fffdb4390ea9737a8b843285a9960cb6cf56e1f50a1606f97099406316db
|
3 |
+
size 4601000928
|
model.safetensors.index.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
preprocessor_config.json
ADDED
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"do_convert_rgb": null,
|
3 |
+
"do_normalize": true,
|
4 |
+
"do_pan_and_scan": null,
|
5 |
+
"do_rescale": true,
|
6 |
+
"do_resize": true,
|
7 |
+
"image_mean": [
|
8 |
+
0.5,
|
9 |
+
0.5,
|
10 |
+
0.5
|
11 |
+
],
|
12 |
+
"image_processor_type": "Gemma3ImageProcessor",
|
13 |
+
"image_seq_length": 256,
|
14 |
+
"image_std": [
|
15 |
+
0.5,
|
16 |
+
0.5,
|
17 |
+
0.5
|
18 |
+
],
|
19 |
+
"pan_and_scan_max_num_crops": null,
|
20 |
+
"pan_and_scan_min_crop_size": null,
|
21 |
+
"pan_and_scan_min_ratio_to_activate": null,
|
22 |
+
"processor_class": "Gemma3Processor",
|
23 |
+
"resample": 2,
|
24 |
+
"rescale_factor": 0.00392156862745098,
|
25 |
+
"size": {
|
26 |
+
"height": 896,
|
27 |
+
"width": 896
|
28 |
+
}
|
29 |
+
}
|
processor_config.json
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"image_seq_length": 256,
|
3 |
+
"processor_class": "Gemma3Processor"
|
4 |
+
}
|
special_tokens_map.json
ADDED
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"boi_token": "<start_of_image>",
|
3 |
+
"bos_token": {
|
4 |
+
"content": "<bos>",
|
5 |
+
"lstrip": false,
|
6 |
+
"normalized": false,
|
7 |
+
"rstrip": false,
|
8 |
+
"single_word": false
|
9 |
+
},
|
10 |
+
"eoi_token": "<end_of_image>",
|
11 |
+
"eos_token": {
|
12 |
+
"content": "<eos>",
|
13 |
+
"lstrip": false,
|
14 |
+
"normalized": false,
|
15 |
+
"rstrip": false,
|
16 |
+
"single_word": false
|
17 |
+
},
|
18 |
+
"image_token": "<image_soft_token>",
|
19 |
+
"pad_token": {
|
20 |
+
"content": "<pad>",
|
21 |
+
"lstrip": false,
|
22 |
+
"normalized": false,
|
23 |
+
"rstrip": false,
|
24 |
+
"single_word": false
|
25 |
+
},
|
26 |
+
"unk_token": {
|
27 |
+
"content": "<unk>",
|
28 |
+
"lstrip": false,
|
29 |
+
"normalized": false,
|
30 |
+
"rstrip": false,
|
31 |
+
"single_word": false
|
32 |
+
}
|
33 |
+
}
|
tokenizer.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7d4046bf0505a327dd5a0abbb427ecd4fc82f99c2ceaa170bc61ecde12809b0c
|
3 |
+
size 33384570
|
tokenizer.model
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
|
3 |
+
size 4689074
|
tokenizer_config.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|