cmedina/agenx-lora-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use 4-bit model, remove DeepSpeed, fix dataset path

Browse Source
This commit is contained in:
Christian Medina
2026-07-02 15:23:26 -04:00
parent 4348323116
commit b8214f64cb
1 changed files with 10 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
training/configs/ornith-35b-lora.yaml
View File

@@ -1,12 +1,12 @@
# LoRA Training Configuration for Ornith-1.0-35B # LoRA Training Configuration for Ornith-1.0-35B
# Dataset: cyron_summary_lora_dataset (20k examples) # Dataset: cyron_summary_lora_dataset (20k examples)
base_model: /data/models/Ornith-1.0-35B base_model: /data/models/Ornith-1.0-35B-4bit
model_type: LlamaForCausalLM model_type: Qwen3_5MoeForCausalLM
tokenizer_type: LlamaTokenizer tokenizer_type: AutoTokenizer
# Model is already quantized (Ornith uses CompressedTensors) # Model is pre-quantized with CompressedTensors
# No need for BitsAndBytes configuration # Loading via accelerate device_map for DISTRIBUTED training
# LoRA Configuration # LoRA Configuration
lora_r: 64 lora_r: 64
@@ -24,7 +24,7 @@ lora_task_type: CAUSAL_LM
# Dataset # Dataset
dataset: dataset:
- path: ../combined_20k.jsonl - path: ~/loras/agenx-lora-training/dataset/combined_20k.jsonl
type: completion type: completion
text_column: text text_column: text
@@ -33,7 +33,7 @@ train_params:
num_train_epochs: 3 num_train_epochs: 3
per_device_train_batch_size: 1 per_device_train_batch_size: 1
gradient_accumulation_steps: 8 gradient_accumulation_steps: 8
learning_rate: 2e-4 learning_rate: 0.0002
lr_scheduler_type: cosine lr_scheduler_type: cosine
weight_decay: 0.01 weight_decay: 0.01
warmup_ratio: 0.03 # Will be converted to warmup_steps by TrainingArguments warmup_ratio: 0.03 # Will be converted to warmup_steps by TrainingArguments
@@ -46,16 +46,9 @@ train_params:
# Precision # Precision
mixed_precision: bf16 mixed_precision: bf16
# Distributed training (2x RTX 5090) - DeepSpeed ZeRO-3 # Distributed training (2x RTX 5090)
plugin: deepspeed # Using accelerate device_map for DISTRIBUTED loading
huggingface_hub: # No DeepSpeed - model already quantized
token: null
deepspeed_config:
zero_optimization:
stage: 3
gradient_clipping: 1.0
train_batch_size: auto
train_micro_batch_size_per_gpu: auto
# Evaluation # Evaluation
eval_strategy: steps eval_strategy: steps