Remove need for trust_remote_code on HF version
Browse filesI opened for this one, but IMO should be valid for all! Also `AutoProcessor` will work directly with your path :)
README.md
CHANGED
@@ -369,10 +369,8 @@ from transformers import AutoTokenizer, AutoModel
|
|
369 |
path = "OpenGVLab/InternVL3_5-8B-HF"
|
370 |
model = AutoModel.from_pretrained(
|
371 |
path,
|
372 |
-
|
373 |
-
low_cpu_mem_usage=True
|
374 |
-
use_flash_attn=True,
|
375 |
-
trust_remote_code=True).eval().cuda()
|
376 |
```
|
377 |
|
378 |
#### BNB 8-bit Quantization
|
@@ -383,11 +381,10 @@ from transformers import AutoTokenizer, AutoModel
|
|
383 |
path = "OpenGVLab/InternVL3_5-8B-HF"
|
384 |
model = AutoModel.from_pretrained(
|
385 |
path,
|
386 |
-
|
387 |
load_in_8bit=True,
|
388 |
-
low_cpu_mem_usage=True
|
389 |
-
|
390 |
-
trust_remote_code=True).eval()
|
391 |
```
|
392 |
|
393 |
#### Multiple GPUs
|
@@ -400,10 +397,8 @@ from transformers import AutoTokenizer, AutoModel
|
|
400 |
path = "OpenGVLab/InternVL3_5-8B-HF"
|
401 |
model = AutoModel.from_pretrained(
|
402 |
path,
|
403 |
-
|
404 |
low_cpu_mem_usage=True,
|
405 |
-
use_flash_attn=True,
|
406 |
-
trust_remote_code=True,
|
407 |
device_map="auto").eval()
|
408 |
```
|
409 |
|
|
|
369 |
path = "OpenGVLab/InternVL3_5-8B-HF"
|
370 |
model = AutoModel.from_pretrained(
|
371 |
path,
|
372 |
+
dtype=torch.bfloat16,
|
373 |
+
low_cpu_mem_usage=True).eval()
|
|
|
|
|
374 |
```
|
375 |
|
376 |
#### BNB 8-bit Quantization
|
|
|
381 |
path = "OpenGVLab/InternVL3_5-8B-HF"
|
382 |
model = AutoModel.from_pretrained(
|
383 |
path,
|
384 |
+
dtype=torch.bfloat16,
|
385 |
load_in_8bit=True,
|
386 |
+
low_cpu_mem_usage=True
|
387 |
+
device_map="auto").eval()
|
|
|
388 |
```
|
389 |
|
390 |
#### Multiple GPUs
|
|
|
397 |
path = "OpenGVLab/InternVL3_5-8B-HF"
|
398 |
model = AutoModel.from_pretrained(
|
399 |
path,
|
400 |
+
dtype=torch.bfloat16,
|
401 |
low_cpu_mem_usage=True,
|
|
|
|
|
402 |
device_map="auto").eval()
|
403 |
```
|
404 |
|