60 lines
1.6 KiB
YAML
60 lines
1.6 KiB
YAML
# LoRA Training Configuration for Ornith-1.0-35B
|
|
# Dataset: cyron_summary_lora_dataset (20k examples)
|
|
|
|
base_model: /data/models/Ornith-1.0-35B-4bit
|
|
model_type: Qwen3_5MoeForCausalLM
|
|
tokenizer_type: AutoTokenizer
|
|
|
|
# Model is pre-quantized with CompressedTensors
|
|
# Loading via accelerate device_map for DISTRIBUTED training
|
|
|
|
# LoRA Configuration
|
|
lora_r: 64
|
|
lora_alpha: 128
|
|
lora_dropout: 0.05
|
|
target_modules:
|
|
- q_proj
|
|
- v_proj
|
|
- k_proj
|
|
- o_proj
|
|
- gate_proj
|
|
- up_proj
|
|
- down_proj
|
|
lora_task_type: CAUSAL_LM
|
|
|
|
# Dataset
|
|
dataset:
|
|
- path: /home/cyaren/loras/agenx-lora-training/dataset/combined_20k.jsonl
|
|
type: completion
|
|
text_column: output
|
|
|
|
# Training Parameters
|
|
train_params:
|
|
num_train_epochs: 3
|
|
per_device_train_batch_size: 1
|
|
gradient_accumulation_steps: 16 # Increased to reduce per-step memory
|
|
learning_rate: 0.0002
|
|
lr_scheduler_type: cosine
|
|
weight_decay: 0.01
|
|
warmup_ratio: 0.03 # Will be converted to warmup_steps by TrainingArguments
|
|
max_seq_length: 512 # Reduced from 1024 to save memory
|
|
logging_steps: 10
|
|
save_steps: 100
|
|
save_total_limit: 3
|
|
output_dir: ../../output/ornith-35b-lora
|
|
optim: adamw_bnb_8bit # 8-bit optimizer to save VRAM
|
|
optim_args: "offload_optimizer_device=cpu" # Offload optimizer to CPU RAM
|
|
|
|
# Precision
|
|
mixed_precision: bf16
|
|
|
|
# Distributed training (2x RTX 5090)
|
|
# Using accelerate device_map for DISTRIBUTED loading
|
|
# No DeepSpeed - model already quantized
|
|
|
|
# Evaluation (disable - no test split in dataset)
|
|
eval_strategy: "no"
|
|
|
|
# Gradient Checkpointing (disable - causes device issues with distributed MoE)
|
|
gradient_checkpointing: false
|