| ## Inference Code | |
| ``` | |
| import os | |
| os.environ["CUDA_VISIBLE_DEVICES"]="0" | |
| from dataclasses import dataclass, field | |
| from typing import Optional | |
| import torch | |
| from datasets import load_dataset | |
| from peft import LoraConfig | |
| from transformers import ( | |
| AutoModelForCausalLM, | |
| AutoTokenizer, | |
| BitsAndBytesConfig, | |
| HfArgumentParser, | |
| AutoTokenizer, | |
| TrainingArguments, | |
| ) | |
| from trl import SFTTrainer | |
| from peft import ( | |
| prepare_model_for_kbit_training, | |
| LoraConfig, | |
| get_peft_model, | |
| PeftModel | |
| ) | |
| bnb_config = BitsAndBytesConfig( | |
| load_in_4bit=True, | |
| bnb_4bit_quant_type="nf4", | |
| bnb_4bit_compute_dtype="bfloat16", | |
| bnb_4bit_use_double_quant=False, | |
| ) | |
| device_map = {"": 0} | |
| tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-40b") | |
| model = AutoModelForCausalLM.from_pretrained( | |
| "tiiuae/falcon-40b", quantization_config=bnb_config, device_map=device_map, trust_remote_code=True | |
| ) | |
| model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=False) | |
| model_id = "smangrul/falcon-40B-int4-peft-lora-sfttrainer" | |
| model = PeftModel.from_pretrained(model, model_id) | |
| text = '### Human: Write a tweet celebrating the Apache-2 release of Falcon models which are generative Large Language Models (LLMs) on which you have been finetuned. Previously, it was under a bit of a restrictive license. Make the tweet punchy, energetic, exciting and marketable.### Assitant:' | |
| outputs = model.generate(input_ids=tokenizer(text, return_tensors="pt").input_ids, | |
| max_new_tokens=256, | |
| temperature=0.7, | |
| top_p=0.9, | |
| do_sample=True) | |
| print(tokenizer.batch_decode(outputs)) | |
| ``` | |
| ### Example completions: | |
| ``` | |
| ### Human: Write a tweet celebrating the Apache-2 release of Falcon models which are generative Large Language Models (LLMs) on which you have been finetuned. | |
| Previously, it was under a bit of a restrictive license. | |
| Make the tweet punchy, energetic, exciting and marketable. | |
| ### Assitant: The Apache-2 release of Falcon models is a huge milestone for the Open Source community! | |
| 🎉 Previously, Falcon was only available under a restrictive license, but now anyone can use and contribute to it. | |
| \n\nFalcon is a large language | |
| ``` | |
| ``` | |
| ### Human: Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions. | |
| ### Assitant: Here is a sample travel blog post about a recent trip to Hawaii:\n\n# | |
| A Guide to Cultural Experiences and Must-See Attractions in Hawaii\n\nIf you\'re looking for a tropical getaway, Hawaii is the perfect destination. | |
| With its stunning beaches, lush rainforests, and rich culture, there\'s something for everyone in this beautiful state. | |
| In this blog post, we\'ll explore some of the best cultural experiences and must-see attractions in Hawaii.\n\n## | |
| Experience the Aloha Spirit\n\nOne of the best things about Hawaii is the people. | |
| The locals are known for their hospitality and kindness, and you\'ll often hear them say "aloha" as a greeting. | |
| The aloha spirit is all about treating others with respect and kindness, and it\'s a big part of what makes Hawaii such a special place.\n\n | |
| ## Visit a Luau\n\nOne of the most popular cultural experiences in Hawaii is the luau. A luau is a traditional Hawaiian feast that includes food, music, and dance. | |
| It\'s a great way to experience the culture and learn more about Hawaiian history and traditions. | |
| There are many different luau options available, so you can find one that fits your interests and budget.\n\n## Explore the Natural W' | |
| ``` | |