# LoRA Training Configuration for Llama-2-7b
# Dataset: cyron_summary_lora_dataset (20k examples)

base_model: deepreinforce-ai/Ornith-1.0-35B-FP8
model_type: LlamaForCausalLM
tokenizer_type: LlamaTokenizer

# Model is already quantized (Ornith uses CompressedTensors)
# No need for BitsAndBytes configuration

# LoRA Configuration
lora_r: 16
lora_alpha: 32
lora_dropout: 0.05
target_modules:
  - q_proj
  - v_proj
  - k_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj
lora_task_type: CAUSAL_LM

# Dataset
dataset:
  - path: ../combined_20k.jsonl
    type: completion
    text_column: text

# Training Parameters
train_params:
  num_train_epochs: 3
  per_device_train_batch_size: 1
  gradient_accumulation_steps: 8
  learning_rate: 2e-4
  lr_scheduler_type: cosine
  weight_decay: 0.01
  warmup_ratio: 0.03
  max_seq_length: 1024
  logging_steps: 10
  save_steps: 100
  save_total_limit: 3
  output_dir: ../../output/llama2-7b-lora

# Precision
mixed_precision: bf16

# Distributed training (2x RTX 5090) - DeepSpeed ZeRO-3
plugin: deepspeed
huggingface_hub:
  token: null
deeepspeed_config:
  zero_optimization:
    stage: 3
    offload_optimizer:
      device: cpu
      pin_memory: true
    offload_param:
      device: cpu
      pin_memory: true
  gradient_clipping: 1.0
  train_batch_size: auto
  train_micro_batch_size_per_gpu: auto

# Evaluation
eval_strategy: steps
eval_steps: 100
eval_accumulation_steps: 10

# Gradient Checkpointing
gradient_checkpointing: true