refactor: simplify deploy script, add train-on-this-server.sh
This commit is contained in:
Christian Medina
@@ -1,159 +1,68 @@
|
||||
#!/bin/bash
|
||||
# Update a GPU-server clone of agenx-lora-training and run the training.
|
||||
# Deploy LoRA training repo (clone + prepare dataset).
|
||||
# Training is run manually on the GPU server.
|
||||
#
|
||||
# Usage:
|
||||
# bash deploy-and-train.sh
|
||||
#
|
||||
# Optional overrides:
|
||||
# LORA_DEPLOY_REPO_URL=https://gitea.cyaren.com/cmedina/agenx-lora-training.git
|
||||
# LORA_DEPLOY_REPO_DIR=/opt/loras/agenx-lora-training
|
||||
# LORA_DEPLOY_BRANCH=main
|
||||
# This will:
|
||||
# 1. Clone/update the repo to /opt/loras/agenx-lora-training
|
||||
# 2. Print instructions for downloading dataset and training
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Load credentials from env file if it exists (override with env vars).
|
||||
if [ -f /home/cyaren/.deploy.env ]; then
|
||||
set -a; source /home/cyaren/.deploy.env; set +a
|
||||
fi
|
||||
|
||||
DEPLOY_USER="${LORA_DEPLOY_USER:-}"
|
||||
DEPLOY_TOKEN="${LORA_DEPLOY_TOKEN:-}"
|
||||
|
||||
# Build the repo URL; embed user:token for HTTPS auth when provided.
|
||||
# Use remote Gitea (local Gitea instances may not be accessible)
|
||||
if [ -n "${DEPLOY_USER}" ] && [ -n "${DEPLOY_TOKEN}" ]; then
|
||||
REPO_URL="${LORA_DEPLOY_REPO_URL:-https://${DEPLOY_USER}:${DEPLOY_TOKEN}@gitea.cyaren.com/cmedina/agenx-lora-training.git}"
|
||||
else
|
||||
# Configuration
|
||||
REPO_URL="${LORA_DEPLOY_REPO_URL:-https://gitea.cyaren.com/cmedina/agenx-lora-training.git}"
|
||||
fi
|
||||
REPO_DIR="${LORA_DEPLOY_REPO_DIR:-/opt/loras/agenx-lora-training}"
|
||||
BRANCH="${LORA_DEPLOY_BRANCH:-main}"
|
||||
REMOTE="${LORA_DEPLOY_REMOTE:-origin}"
|
||||
|
||||
RUN_USER="${SUDO_USER:-$(id -un)}"
|
||||
RUN_GROUP="$(id -gn "${RUN_USER}" 2>/dev/null || id -gn)"
|
||||
REPO_PARENT="$(dirname "${REPO_DIR}")"
|
||||
|
||||
echo "=== LoRA Training Git Deploy ==="
|
||||
echo "=== LoRA Training Deploy ==="
|
||||
echo "Repo: ${REPO_URL}"
|
||||
echo "Dir: ${REPO_DIR}"
|
||||
echo "Branch: ${BRANCH}"
|
||||
echo ""
|
||||
|
||||
# Create parent directory if needed
|
||||
if [ ! -d "${REPO_PARENT}" ]; then
|
||||
echo "Creating ${REPO_PARENT}..."
|
||||
sudo install -d -m 0755 -o "${RUN_USER}" -g "${RUN_GROUP}" "${REPO_PARENT}"
|
||||
fi
|
||||
|
||||
# Clone or update repo
|
||||
if [ ! -d "${REPO_DIR}/.git" ]; then
|
||||
echo "Cloning repository..."
|
||||
git clone --branch "${BRANCH}" "${REPO_URL}" "${REPO_DIR}"
|
||||
fi
|
||||
|
||||
else
|
||||
echo "Repository exists, pulling latest..."
|
||||
cd "${REPO_DIR}"
|
||||
|
||||
# Ensure the remote URL matches the authenticated HTTPS URL.
|
||||
git remote set-url "${REMOTE}" "${REPO_URL}"
|
||||
|
||||
git fetch "${REMOTE}" "${BRANCH}"
|
||||
git fetch origin "${BRANCH}"
|
||||
git checkout "${BRANCH}"
|
||||
git pull --ff-only "${REMOTE}" "${BRANCH}"
|
||||
|
||||
# Setup Python environment and train
|
||||
echo ""
|
||||
echo "=== Setting up Python environment ==="
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install --upgrade pip
|
||||
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
|
||||
pip install transformers datasets trl peft accelerate bitsandbytes
|
||||
|
||||
# Prepare dataset
|
||||
echo ""
|
||||
echo "=== Preparing dataset ==="
|
||||
python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data
|
||||
|
||||
# Train
|
||||
echo ""
|
||||
echo "=== Starting LoRA training ==="
|
||||
python3 training/scripts/train.py --config training/configs/llama2-7b-lora.yaml
|
||||
|
||||
# Step 3: Setup Python environment
|
||||
echo "[3/5] Setting up Python environment..."
|
||||
cd "$INSTALL_DIR"
|
||||
|
||||
# Check if Python is available
|
||||
if ! command -v python3 &> /dev/null; then
|
||||
echo "ERROR: Python3 not found. Please install Python $PYTHON_VERSION or later."
|
||||
exit 1
|
||||
git pull --ff-only origin "${BRANCH}"
|
||||
fi
|
||||
|
||||
# Create virtual environment
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
|
||||
# Upgrade pip
|
||||
pip install --upgrade pip
|
||||
|
||||
# Install PyTorch with CUDA support
|
||||
echo " Installing PyTorch with CUDA support..."
|
||||
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
|
||||
|
||||
# Install other dependencies
|
||||
echo " Installing training dependencies..."
|
||||
pip install transformers datasets trl peft accelerate bitsandbytes
|
||||
|
||||
# Verify GPU availability
|
||||
echo ""
|
||||
echo " Checking GPU availability..."
|
||||
python3 -c "
|
||||
import torch
|
||||
if torch.cuda.is_available():
|
||||
print(f' ✓ CUDA available: {torch.cuda.get_device_name(0)}')
|
||||
print(f' ✓ VRAM: {torch.cuda.get_device_properties(0).total_mem / 1e9:.1f} GB')
|
||||
else:
|
||||
print(' ✗ CUDA not available. GPU training will not work.')
|
||||
print(' Please ensure CUDA drivers are installed.')
|
||||
exit(1)
|
||||
"
|
||||
|
||||
# Step 4: Prepare dataset
|
||||
echo "[4/5] Preparing dataset..."
|
||||
python3 training/scripts/prepare_dataset.py \
|
||||
--input dataset/combined_20k.jsonl \
|
||||
--output training/data
|
||||
|
||||
echo " Dataset prepared:"
|
||||
echo " - training/data/train.jsonl"
|
||||
echo " - training/data/test.jsonl"
|
||||
|
||||
# Step 5: Train the model
|
||||
echo "[5/5] Starting training..."
|
||||
echo "=== Repository ready ==="
|
||||
echo "Location: ${REPO_DIR}"
|
||||
echo ""
|
||||
echo " Training configuration:"
|
||||
echo " - Model: meta-llama/Llama-2-7b-hf"
|
||||
echo " - Method: QLoRA (4-bit quantization)"
|
||||
echo " - Epochs: 3"
|
||||
echo " - Batch size: 4"
|
||||
echo " - Learning rate: 2e-4"
|
||||
echo "=== Next Steps ==="
|
||||
echo ""
|
||||
echo " Estimated training time: 6-24 hours (depending on GPU)"
|
||||
echo "1. Download dataset (8.77 MB):"
|
||||
echo " cd ${REPO_DIR}"
|
||||
echo " curl -O https://gitea.cyaren.com/cmedina/agenx-lora-training/raw/branch/main/dataset/combined_20k.jsonl"
|
||||
echo ""
|
||||
|
||||
python3 training/scripts/train.py \
|
||||
--config training/configs/llama2-7b-lora.yaml
|
||||
|
||||
echo "2. Setup Python + GPU drivers on your training server"
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
echo "Training complete!"
|
||||
echo "=============================================="
|
||||
echo "3. Prepare dataset:"
|
||||
echo " python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data"
|
||||
echo ""
|
||||
echo "Trained model saved to: $INSTALL_DIR/training/output/llama2-7b-lora/"
|
||||
echo "4. Train LoRA (on GPU server):"
|
||||
echo " python3 training/scripts/train.py --config training/configs/llama2-7b-lora.yaml"
|
||||
echo ""
|
||||
echo "To generate summaries:"
|
||||
echo " cd $INSTALL_DIR"
|
||||
echo " source venv/bin/activate"
|
||||
echo " python3 training/scripts/inference.py \\"
|
||||
echo " --model training/output/llama2-7b-lora \\"
|
||||
echo " --task \"Your task here\" \\"
|
||||
echo " --files src/file.py \\"
|
||||
echo " --tests-run --test-count 100"
|
||||
echo "=== To train now (if on GPU server) ==="
|
||||
echo " bash ${REPO_DIR}/train-on-this-server.sh"
|
||||
echo ""
|
||||
|
||||
74
train-on-this-server.sh
Normal file
74
train-on-this-server.sh
Normal file
@@ -0,0 +1,74 @@
|
||||
#!/bin/bash
|
||||
# Train LoRA on this server (assumes GPU is available).
|
||||
# Run this after deploy-and-train.sh has cloned the repo.
|
||||
|
||||
set -e
|
||||
|
||||
REPO_DIR="${1:-$(pwd)}"
|
||||
|
||||
echo "=== LoRA Training Setup ==="
|
||||
echo "Repo: ${REPO_DIR}"
|
||||
echo ""
|
||||
|
||||
# Check Python
|
||||
if ! command -v python3 &> /dev/null; then
|
||||
echo "ERROR: python3 not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[1/4] Setup Python environment..."
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install --upgrade pip
|
||||
|
||||
echo "[2/4] Install PyTorch with CUDA..."
|
||||
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
|
||||
|
||||
echo "[3/4] Install training dependencies..."
|
||||
pip install transformers datasets trl peft accelerate bitsandbytes
|
||||
|
||||
# Check GPU
|
||||
echo ""
|
||||
echo "[4/4] Checking GPU..."
|
||||
python3 -c "
|
||||
import torch
|
||||
if torch.cuda.is_available():
|
||||
print(f' ✓ GPU: {torch.cuda.get_device_name(0)}')
|
||||
print(f' ✓ VRAM: {torch.cuda.get_device_properties(0).total_mem / 1e9:.1f} GB')
|
||||
else:
|
||||
print(' ✗ No GPU detected')
|
||||
print(' Please ensure CUDA drivers are installed')
|
||||
exit(1)
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== Dataset Preparation ==="
|
||||
echo "Checking for combined_20k.jsonl..."
|
||||
if [ ! -f "dataset/combined_20k.jsonl" ]; then
|
||||
echo "Dataset not found. Downloading..."
|
||||
curl -L -o dataset/combined_20k.jsonl https://gitea.cyaren.com/cmedina/agenx-lora-training/raw/branch/main/dataset/combined_20k.jsonl
|
||||
fi
|
||||
|
||||
echo "Preparing dataset..."
|
||||
python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data
|
||||
|
||||
echo ""
|
||||
echo "=== Starting Training ==="
|
||||
echo "Config: training/configs/llama2-7b-lora.yaml"
|
||||
echo "Output: training/output/llama2-7b-lora/"
|
||||
echo ""
|
||||
echo "Training will take 6-24 hours depending on GPU."
|
||||
echo "Press Ctrl+C to stop (model will be saved at checkpoint)."
|
||||
echo ""
|
||||
|
||||
python3 training/scripts/train.py --config training/configs/llama2-7b-lora.yaml
|
||||
|
||||
echo ""
|
||||
echo "=== Training Complete ==="
|
||||
echo "Model saved to: training/output/llama2-7b-lora/"
|
||||
echo ""
|
||||
echo "To generate summaries:"
|
||||
echo " python3 training/scripts/inference.py \\"
|
||||
echo " --model training/output/llama2-7b-lora \\"
|
||||
echo " --task \"Your task here\" \\"
|
||||
echo " --files src/file.py"
|
||||
Reference in New Issue
Block a user