init: add LoRA training infrastructure and 20k dataset

training/configs/llama2-7b-lora.yaml

@@ -0,0 +1,55 @@
+# LoRA Training Configuration for Llama-2-7b
+# Dataset: cyron_summary_lora_dataset (20k examples)
+
+base_model: meta-llama/Llama-2-7b-hf
+model_type: LlamaForCausalLM
+tokenizer_type: LlamaTokenizer
+
+# Quantization (QLoRA)
+load_in_4bit: true
+bnb_4bit_compute_dtype: bfloat16
+bnb_4bit_quant_type: nf4
+use_nested_quant: false
+
+# LoRA Configuration
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.05
+target_modules:
+  - q_proj
+  - v_proj
+  - k_proj
+  - o_proj
+lora_task_type: CAUSAL_LM
+
+# Dataset
+dataset:
+  - path: ../combined_20k.jsonl
+    type: completion
+    text_column: text
+
+# Training Parameters
+train_params:
+  num_train_epochs: 3
+  per_device_train_batch_size: 4
+  gradient_accumulation_steps: 4
+  learning_rate: 2e-4
+  lr_scheduler_type: cosine
+  weight_decay: 0.01
+  warmup_ratio: 0.03
+  max_seq_length: 1024
+  logging_steps: 10
+  save_steps: 100
+  save_total_limit: 3
+  output_dir: ../../output/llama2-7b-lora
+
+# Precision
+mixed_precision: bf16
+
+# Evaluation
+eval_strategy: steps
+eval_steps: 100
+eval_accumulation_steps: 10
+
+# Gradient Checkpointing
+gradient_checkpointing: true