# LoRA Training Configuration for Ornith-1.0-35B
# Dataset: cyron_summary_lora_dataset (20k examples)

base_model: /data/models/Ornith-1.0-35B-4bit
model_type: Qwen3_5MoeForCausalLM
tokenizer_type: AutoTokenizer

# Model is pre-quantized with CompressedTensors
# Loading via accelerate device_map for DISTRIBUTED training

# LoRA Configuration
lora_r: 64
lora_alpha: 128
lora_dropout: 0.05
target_modules:
  - q_proj
  - v_proj
  - k_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj
lora_task_type: CAUSAL_LM

# Dataset
dataset:
  - path: /home/cyaren/loras/agenx-lora-training/dataset/combined_20k.jsonl
    type: completion
    text_column: output

# Training Parameters
train_params:
  num_train_epochs: 3
  per_device_train_batch_size: 1
  gradient_accumulation_steps: 8
  learning_rate: 0.0002
  lr_scheduler_type: cosine
  weight_decay: 0.01
  warmup_ratio: 0.03  # Will be converted to warmup_steps by TrainingArguments
  max_seq_length: 1024
  logging_steps: 10
  save_steps: 100
  save_total_limit: 3
  output_dir: ../../output/ornith-35b-lora
  optim: adamw_bnb_8bit  # 8-bit optimizer to save VRAM

# Precision
mixed_precision: bf16

# Distributed training (2x RTX 5090)
# Using accelerate device_map for DISTRIBUTED loading
# No DeepSpeed - model already quantized

# Evaluation (disable - no test split in dataset)
eval_strategy: "no"

# Gradient Checkpointing (disable - causes device issues with distributed MoE)
gradient_checkpointing: false