fix: use 4-bit model, remove DeepSpeed, fix dataset path

training/configs/ornith-35b-lora.yaml

@@ -1,12 +1,12 @@
 # LoRA Training Configuration for Ornith-1.0-35B
 # Dataset: cyron_summary_lora_dataset (20k examples)
 
-base_model: /data/models/Ornith-1.0-35B
-model_type: LlamaForCausalLM
-tokenizer_type: LlamaTokenizer
+base_model: /data/models/Ornith-1.0-35B-4bit
+model_type: Qwen3_5MoeForCausalLM
+tokenizer_type: AutoTokenizer
 
-# Model is already quantized (Ornith uses CompressedTensors)
-# No need for BitsAndBytes configuration
+# Model is pre-quantized with CompressedTensors
+# Loading via accelerate device_map for DISTRIBUTED training
 
 # LoRA Configuration
 lora_r: 64
@@ -24,7 +24,7 @@ lora_task_type: CAUSAL_LM
 
 # Dataset
 dataset:
-  - path: ../combined_20k.jsonl
+  - path: ~/loras/agenx-lora-training/dataset/combined_20k.jsonl
     type: completion
     text_column: text
 
@@ -33,7 +33,7 @@ train_params:
   num_train_epochs: 3
   per_device_train_batch_size: 1
   gradient_accumulation_steps: 8
-  learning_rate: 2e-4
+  learning_rate: 0.0002
   lr_scheduler_type: cosine
   weight_decay: 0.01
   warmup_ratio: 0.03  # Will be converted to warmup_steps by TrainingArguments
@@ -46,16 +46,9 @@ train_params:
 # Precision
 mixed_precision: bf16
 
-# Distributed training (2x RTX 5090) - DeepSpeed ZeRO-3
-plugin: deepspeed
-huggingface_hub:
-  token: null
-deepspeed_config:
-  zero_optimization:
-    stage: 3
-  gradient_clipping: 1.0
-  train_batch_size: auto
-  train_micro_batch_size_per_gpu: auto
+# Distributed training (2x RTX 5090)
+# Using accelerate device_map for DISTRIBUTED loading
+# No DeepSpeed - model already quantized
 
 # Evaluation
 eval_strategy: steps