refactor: simplify deploy script, add train-on-this-server.sh
This commit is contained in:
Christian Medina
@@ -1,159 +1,68 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# Update a GPU-server clone of agenx-lora-training and run the training.
|
# Deploy LoRA training repo (clone + prepare dataset).
|
||||||
|
# Training is run manually on the GPU server.
|
||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
# bash deploy-and-train.sh
|
# bash deploy-and-train.sh
|
||||||
#
|
#
|
||||||
# Optional overrides:
|
# This will:
|
||||||
# LORA_DEPLOY_REPO_URL=https://gitea.cyaren.com/cmedina/agenx-lora-training.git
|
# 1. Clone/update the repo to /opt/loras/agenx-lora-training
|
||||||
# LORA_DEPLOY_REPO_DIR=/opt/loras/agenx-lora-training
|
# 2. Print instructions for downloading dataset and training
|
||||||
# LORA_DEPLOY_BRANCH=main
|
|
||||||
#
|
#
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# Load credentials from env file if it exists (override with env vars).
|
# Configuration
|
||||||
if [ -f /home/cyaren/.deploy.env ]; then
|
|
||||||
set -a; source /home/cyaren/.deploy.env; set +a
|
|
||||||
fi
|
|
||||||
|
|
||||||
DEPLOY_USER="${LORA_DEPLOY_USER:-}"
|
|
||||||
DEPLOY_TOKEN="${LORA_DEPLOY_TOKEN:-}"
|
|
||||||
|
|
||||||
# Build the repo URL; embed user:token for HTTPS auth when provided.
|
|
||||||
# Use remote Gitea (local Gitea instances may not be accessible)
|
|
||||||
if [ -n "${DEPLOY_USER}" ] && [ -n "${DEPLOY_TOKEN}" ]; then
|
|
||||||
REPO_URL="${LORA_DEPLOY_REPO_URL:-https://${DEPLOY_USER}:${DEPLOY_TOKEN}@gitea.cyaren.com/cmedina/agenx-lora-training.git}"
|
|
||||||
else
|
|
||||||
REPO_URL="${LORA_DEPLOY_REPO_URL:-https://gitea.cyaren.com/cmedina/agenx-lora-training.git}"
|
REPO_URL="${LORA_DEPLOY_REPO_URL:-https://gitea.cyaren.com/cmedina/agenx-lora-training.git}"
|
||||||
fi
|
|
||||||
REPO_DIR="${LORA_DEPLOY_REPO_DIR:-/opt/loras/agenx-lora-training}"
|
REPO_DIR="${LORA_DEPLOY_REPO_DIR:-/opt/loras/agenx-lora-training}"
|
||||||
BRANCH="${LORA_DEPLOY_BRANCH:-main}"
|
BRANCH="${LORA_DEPLOY_BRANCH:-main}"
|
||||||
REMOTE="${LORA_DEPLOY_REMOTE:-origin}"
|
|
||||||
|
|
||||||
RUN_USER="${SUDO_USER:-$(id -un)}"
|
RUN_USER="${SUDO_USER:-$(id -un)}"
|
||||||
RUN_GROUP="$(id -gn "${RUN_USER}" 2>/dev/null || id -gn)"
|
RUN_GROUP="$(id -gn "${RUN_USER}" 2>/dev/null || id -gn)"
|
||||||
REPO_PARENT="$(dirname "${REPO_DIR}")"
|
REPO_PARENT="$(dirname "${REPO_DIR}")"
|
||||||
|
|
||||||
echo "=== LoRA Training Git Deploy ==="
|
echo "=== LoRA Training Deploy ==="
|
||||||
echo "Repo: ${REPO_URL}"
|
echo "Repo: ${REPO_URL}"
|
||||||
echo "Dir: ${REPO_DIR}"
|
echo "Dir: ${REPO_DIR}"
|
||||||
echo "Branch: ${BRANCH}"
|
echo "Branch: ${BRANCH}"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Create parent directory if needed
|
||||||
if [ ! -d "${REPO_PARENT}" ]; then
|
if [ ! -d "${REPO_PARENT}" ]; then
|
||||||
|
echo "Creating ${REPO_PARENT}..."
|
||||||
sudo install -d -m 0755 -o "${RUN_USER}" -g "${RUN_GROUP}" "${REPO_PARENT}"
|
sudo install -d -m 0755 -o "${RUN_USER}" -g "${RUN_GROUP}" "${REPO_PARENT}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Clone or update repo
|
||||||
if [ ! -d "${REPO_DIR}/.git" ]; then
|
if [ ! -d "${REPO_DIR}/.git" ]; then
|
||||||
|
echo "Cloning repository..."
|
||||||
git clone --branch "${BRANCH}" "${REPO_URL}" "${REPO_DIR}"
|
git clone --branch "${BRANCH}" "${REPO_URL}" "${REPO_DIR}"
|
||||||
fi
|
else
|
||||||
|
echo "Repository exists, pulling latest..."
|
||||||
cd "${REPO_DIR}"
|
cd "${REPO_DIR}"
|
||||||
|
git fetch origin "${BRANCH}"
|
||||||
# Ensure the remote URL matches the authenticated HTTPS URL.
|
|
||||||
git remote set-url "${REMOTE}" "${REPO_URL}"
|
|
||||||
|
|
||||||
git fetch "${REMOTE}" "${BRANCH}"
|
|
||||||
git checkout "${BRANCH}"
|
git checkout "${BRANCH}"
|
||||||
git pull --ff-only "${REMOTE}" "${BRANCH}"
|
git pull --ff-only origin "${BRANCH}"
|
||||||
|
|
||||||
# Setup Python environment and train
|
|
||||||
echo ""
|
|
||||||
echo "=== Setting up Python environment ==="
|
|
||||||
python3 -m venv venv
|
|
||||||
source venv/bin/activate
|
|
||||||
pip install --upgrade pip
|
|
||||||
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
|
|
||||||
pip install transformers datasets trl peft accelerate bitsandbytes
|
|
||||||
|
|
||||||
# Prepare dataset
|
|
||||||
echo ""
|
|
||||||
echo "=== Preparing dataset ==="
|
|
||||||
python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data
|
|
||||||
|
|
||||||
# Train
|
|
||||||
echo ""
|
|
||||||
echo "=== Starting LoRA training ==="
|
|
||||||
python3 training/scripts/train.py --config training/configs/llama2-7b-lora.yaml
|
|
||||||
|
|
||||||
# Step 3: Setup Python environment
|
|
||||||
echo "[3/5] Setting up Python environment..."
|
|
||||||
cd "$INSTALL_DIR"
|
|
||||||
|
|
||||||
# Check if Python is available
|
|
||||||
if ! command -v python3 &> /dev/null; then
|
|
||||||
echo "ERROR: Python3 not found. Please install Python $PYTHON_VERSION or later."
|
|
||||||
exit 1
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Create virtual environment
|
|
||||||
python3 -m venv venv
|
|
||||||
source venv/bin/activate
|
|
||||||
|
|
||||||
# Upgrade pip
|
|
||||||
pip install --upgrade pip
|
|
||||||
|
|
||||||
# Install PyTorch with CUDA support
|
|
||||||
echo " Installing PyTorch with CUDA support..."
|
|
||||||
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
|
|
||||||
|
|
||||||
# Install other dependencies
|
|
||||||
echo " Installing training dependencies..."
|
|
||||||
pip install transformers datasets trl peft accelerate bitsandbytes
|
|
||||||
|
|
||||||
# Verify GPU availability
|
|
||||||
echo ""
|
echo ""
|
||||||
echo " Checking GPU availability..."
|
echo "=== Repository ready ==="
|
||||||
python3 -c "
|
echo "Location: ${REPO_DIR}"
|
||||||
import torch
|
|
||||||
if torch.cuda.is_available():
|
|
||||||
print(f' ✓ CUDA available: {torch.cuda.get_device_name(0)}')
|
|
||||||
print(f' ✓ VRAM: {torch.cuda.get_device_properties(0).total_mem / 1e9:.1f} GB')
|
|
||||||
else:
|
|
||||||
print(' ✗ CUDA not available. GPU training will not work.')
|
|
||||||
print(' Please ensure CUDA drivers are installed.')
|
|
||||||
exit(1)
|
|
||||||
"
|
|
||||||
|
|
||||||
# Step 4: Prepare dataset
|
|
||||||
echo "[4/5] Preparing dataset..."
|
|
||||||
python3 training/scripts/prepare_dataset.py \
|
|
||||||
--input dataset/combined_20k.jsonl \
|
|
||||||
--output training/data
|
|
||||||
|
|
||||||
echo " Dataset prepared:"
|
|
||||||
echo " - training/data/train.jsonl"
|
|
||||||
echo " - training/data/test.jsonl"
|
|
||||||
|
|
||||||
# Step 5: Train the model
|
|
||||||
echo "[5/5] Starting training..."
|
|
||||||
echo ""
|
echo ""
|
||||||
echo " Training configuration:"
|
echo "=== Next Steps ==="
|
||||||
echo " - Model: meta-llama/Llama-2-7b-hf"
|
|
||||||
echo " - Method: QLoRA (4-bit quantization)"
|
|
||||||
echo " - Epochs: 3"
|
|
||||||
echo " - Batch size: 4"
|
|
||||||
echo " - Learning rate: 2e-4"
|
|
||||||
echo ""
|
echo ""
|
||||||
echo " Estimated training time: 6-24 hours (depending on GPU)"
|
echo "1. Download dataset (8.77 MB):"
|
||||||
|
echo " cd ${REPO_DIR}"
|
||||||
|
echo " curl -O https://gitea.cyaren.com/cmedina/agenx-lora-training/raw/branch/main/dataset/combined_20k.jsonl"
|
||||||
echo ""
|
echo ""
|
||||||
|
echo "2. Setup Python + GPU drivers on your training server"
|
||||||
python3 training/scripts/train.py \
|
|
||||||
--config training/configs/llama2-7b-lora.yaml
|
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "=============================================="
|
echo "3. Prepare dataset:"
|
||||||
echo "Training complete!"
|
echo " python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data"
|
||||||
echo "=============================================="
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "Trained model saved to: $INSTALL_DIR/training/output/llama2-7b-lora/"
|
echo "4. Train LoRA (on GPU server):"
|
||||||
|
echo " python3 training/scripts/train.py --config training/configs/llama2-7b-lora.yaml"
|
||||||
echo ""
|
echo ""
|
||||||
echo "To generate summaries:"
|
echo "=== To train now (if on GPU server) ==="
|
||||||
echo " cd $INSTALL_DIR"
|
echo " bash ${REPO_DIR}/train-on-this-server.sh"
|
||||||
echo " source venv/bin/activate"
|
|
||||||
echo " python3 training/scripts/inference.py \\"
|
|
||||||
echo " --model training/output/llama2-7b-lora \\"
|
|
||||||
echo " --task \"Your task here\" \\"
|
|
||||||
echo " --files src/file.py \\"
|
|
||||||
echo " --tests-run --test-count 100"
|
|
||||||
echo ""
|
echo ""
|
||||||
|
|||||||
74
train-on-this-server.sh
Normal file
74
train-on-this-server.sh
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Train LoRA on this server (assumes GPU is available).
|
||||||
|
# Run this after deploy-and-train.sh has cloned the repo.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
REPO_DIR="${1:-$(pwd)}"
|
||||||
|
|
||||||
|
echo "=== LoRA Training Setup ==="
|
||||||
|
echo "Repo: ${REPO_DIR}"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Check Python
|
||||||
|
if ! command -v python3 &> /dev/null; then
|
||||||
|
echo "ERROR: python3 not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "[1/4] Setup Python environment..."
|
||||||
|
python3 -m venv venv
|
||||||
|
source venv/bin/activate
|
||||||
|
pip install --upgrade pip
|
||||||
|
|
||||||
|
echo "[2/4] Install PyTorch with CUDA..."
|
||||||
|
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
|
||||||
|
|
||||||
|
echo "[3/4] Install training dependencies..."
|
||||||
|
pip install transformers datasets trl peft accelerate bitsandbytes
|
||||||
|
|
||||||
|
# Check GPU
|
||||||
|
echo ""
|
||||||
|
echo "[4/4] Checking GPU..."
|
||||||
|
python3 -c "
|
||||||
|
import torch
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
print(f' ✓ GPU: {torch.cuda.get_device_name(0)}')
|
||||||
|
print(f' ✓ VRAM: {torch.cuda.get_device_properties(0).total_mem / 1e9:.1f} GB')
|
||||||
|
else:
|
||||||
|
print(' ✗ No GPU detected')
|
||||||
|
print(' Please ensure CUDA drivers are installed')
|
||||||
|
exit(1)
|
||||||
|
"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Dataset Preparation ==="
|
||||||
|
echo "Checking for combined_20k.jsonl..."
|
||||||
|
if [ ! -f "dataset/combined_20k.jsonl" ]; then
|
||||||
|
echo "Dataset not found. Downloading..."
|
||||||
|
curl -L -o dataset/combined_20k.jsonl https://gitea.cyaren.com/cmedina/agenx-lora-training/raw/branch/main/dataset/combined_20k.jsonl
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Preparing dataset..."
|
||||||
|
python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Starting Training ==="
|
||||||
|
echo "Config: training/configs/llama2-7b-lora.yaml"
|
||||||
|
echo "Output: training/output/llama2-7b-lora/"
|
||||||
|
echo ""
|
||||||
|
echo "Training will take 6-24 hours depending on GPU."
|
||||||
|
echo "Press Ctrl+C to stop (model will be saved at checkpoint)."
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
python3 training/scripts/train.py --config training/configs/llama2-7b-lora.yaml
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Training Complete ==="
|
||||||
|
echo "Model saved to: training/output/llama2-7b-lora/"
|
||||||
|
echo ""
|
||||||
|
echo "To generate summaries:"
|
||||||
|
echo " python3 training/scripts/inference.py \\"
|
||||||
|
echo " --model training/output/llama2-7b-lora \\"
|
||||||
|
echo " --task \"Your task here\" \\"
|
||||||
|
echo " --files src/file.py"
|
||||||
Reference in New Issue
Block a user