Create oumi/oumi_infer.yaml

oumi/oumi_infer.yaml

@@ -0,0 +1,24 @@
+# Inference config for Llama 8B Instruct.
+#
+# Usage:
+#   oumi infer -i -c configs/recipes/llama3_1/inference/8b_infer.yaml
+#
+# See Also:
+#   - Documentation: https://oumi.ai/docs/en/latest/user_guides/infer/infer.html
+#   - Config class: oumi.core.configs.InferenceConfig
+#   - Config source: https://github.com/oumi-ai/oumi/blob/main/src/oumi/core/configs/inference_config.py
+#   - Other inference configs: configs/**/inference/
+
+model:
+  model_name: "uiuc-convai/CALM-8B"
+  adapter_model: null  # Update for LoRA-tuned models.
+  model_max_length: 2048  # Update based on your hardware constraints
+  torch_dtype_str: "bfloat16"
+  attn_implementation: "sdpa"
+  load_pretrained_weights: True
+  trust_remote_code: True
+
+generation:
+  max_new_tokens: 2048
+
+engine: NATIVE