docs: rename config and docs to reflect Ornith-35B model

training/configs/ornith-35b-lora.yaml

@@ -0,0 +1,72 @@
+# LoRA Training Configuration for Llama-2-7b
+# Dataset: cyron_summary_lora_dataset (20k examples)
+
+base_model: /data/models/Ornith-1.0-35B-FP8
+model_type: LlamaForCausalLM
+tokenizer_type: LlamaTokenizer
+
+# Model is already quantized (Ornith uses CompressedTensors)
+# No need for BitsAndBytes configuration
+
+# LoRA Configuration
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.05
+target_modules:
+  - q_proj
+  - v_proj
+  - k_proj
+  - o_proj
+  - gate_proj
+  - up_proj
+  - down_proj
+lora_task_type: CAUSAL_LM
+
+# Dataset
+dataset:
+  - path: ../combined_20k.jsonl
+    type: completion
+    text_column: text
+
+# Training Parameters
+train_params:
+  num_train_epochs: 3
+  per_device_train_batch_size: 1
+  gradient_accumulation_steps: 8
+  learning_rate: 2e-4
+  lr_scheduler_type: cosine
+  weight_decay: 0.01
+  warmup_ratio: 0.03
+  max_seq_length: 1024
+  logging_steps: 10
+  save_steps: 100
+  save_total_limit: 3
+  output_dir: ../../output/llama2-7b-lora
+
+# Precision
+mixed_precision: bf16
+
+# Distributed training (2x RTX 5090) - DeepSpeed ZeRO-3
+plugin: deepspeed
+huggingface_hub:
+  token: null
+deepspeed_config:
+  zero_optimization:
+    stage: 3
+    offload_optimizer:
+      device: cpu
+      pin_memory: true
+    offload_param:
+      device: cpu
+      pin_memory: true
+  gradient_clipping: 1.0
+  train_batch_size: auto
+  train_micro_batch_size_per_gpu: auto
+
+# Evaluation
+eval_strategy: steps
+eval_steps: 100
+eval_accumulation_steps: 10
+
+# Gradient Checkpointing
+gradient_checkpointing: true