67 lines
1.4 KiB
YAML
67 lines
1.4 KiB
YAML
# LoRA Training Configuration for Llama-2-7b
|
|
# Dataset: cyron_summary_lora_dataset (20k examples)
|
|
|
|
base_model: /data/models/Ornith-1.0-35B
|
|
model_type: LlamaForCausalLM
|
|
tokenizer_type: LlamaTokenizer
|
|
|
|
# Model is already quantized (Ornith uses CompressedTensors)
|
|
# No need for BitsAndBytes configuration
|
|
|
|
# LoRA Configuration
|
|
lora_r: 16
|
|
lora_alpha: 32
|
|
lora_dropout: 0.05
|
|
target_modules:
|
|
- q_proj
|
|
- v_proj
|
|
- k_proj
|
|
- o_proj
|
|
- gate_proj
|
|
- up_proj
|
|
- down_proj
|
|
lora_task_type: CAUSAL_LM
|
|
|
|
# Dataset
|
|
dataset:
|
|
- path: ../combined_20k.jsonl
|
|
type: completion
|
|
text_column: text
|
|
|
|
# Training Parameters
|
|
train_params:
|
|
num_train_epochs: 3
|
|
per_device_train_batch_size: 1
|
|
gradient_accumulation_steps: 8
|
|
learning_rate: 2e-4
|
|
lr_scheduler_type: cosine
|
|
weight_decay: 0.01
|
|
warmup_ratio: 0.03 # Will be converted to warmup_steps by TrainingArguments
|
|
max_seq_length: 1024
|
|
logging_steps: 10
|
|
save_steps: 100
|
|
save_total_limit: 3
|
|
output_dir: ../../output/llama2-7b-lora
|
|
|
|
# Precision
|
|
mixed_precision: bf16
|
|
|
|
# Distributed training (2x RTX 5090) - DeepSpeed ZeRO-3
|
|
plugin: deepspeed
|
|
huggingface_hub:
|
|
token: null
|
|
deepspeed_config:
|
|
zero_optimization:
|
|
stage: 3
|
|
gradient_clipping: 1.0
|
|
train_batch_size: auto
|
|
train_micro_batch_size_per_gpu: auto
|
|
|
|
# Evaluation
|
|
eval_strategy: steps
|
|
eval_steps: 100
|
|
eval_accumulation_steps: 10
|
|
|
|
# Gradient Checkpointing
|
|
gradient_checkpointing: true
|