diff --git a/deploy-and-train.sh b/deploy-and-train.sh index d9bb4e5..dc78cd1 100755 --- a/deploy-and-train.sh +++ b/deploy-and-train.sh @@ -1,159 +1,68 @@ #!/bin/bash -# Update a GPU-server clone of agenx-lora-training and run the training. +# Deploy LoRA training repo (clone + prepare dataset). +# Training is run manually on the GPU server. # # Usage: # bash deploy-and-train.sh # -# Optional overrides: -# LORA_DEPLOY_REPO_URL=https://gitea.cyaren.com/cmedina/agenx-lora-training.git -# LORA_DEPLOY_REPO_DIR=/opt/loras/agenx-lora-training -# LORA_DEPLOY_BRANCH=main +# This will: +# 1. Clone/update the repo to /opt/loras/agenx-lora-training +# 2. Print instructions for downloading dataset and training # set -euo pipefail -# Load credentials from env file if it exists (override with env vars). -if [ -f /home/cyaren/.deploy.env ]; then - set -a; source /home/cyaren/.deploy.env; set +a -fi - -DEPLOY_USER="${LORA_DEPLOY_USER:-}" -DEPLOY_TOKEN="${LORA_DEPLOY_TOKEN:-}" - -# Build the repo URL; embed user:token for HTTPS auth when provided. -# Use remote Gitea (local Gitea instances may not be accessible) -if [ -n "${DEPLOY_USER}" ] && [ -n "${DEPLOY_TOKEN}" ]; then - REPO_URL="${LORA_DEPLOY_REPO_URL:-https://${DEPLOY_USER}:${DEPLOY_TOKEN}@gitea.cyaren.com/cmedina/agenx-lora-training.git}" -else - REPO_URL="${LORA_DEPLOY_REPO_URL:-https://gitea.cyaren.com/cmedina/agenx-lora-training.git}" -fi +# Configuration +REPO_URL="${LORA_DEPLOY_REPO_URL:-https://gitea.cyaren.com/cmedina/agenx-lora-training.git}" REPO_DIR="${LORA_DEPLOY_REPO_DIR:-/opt/loras/agenx-lora-training}" BRANCH="${LORA_DEPLOY_BRANCH:-main}" -REMOTE="${LORA_DEPLOY_REMOTE:-origin}" RUN_USER="${SUDO_USER:-$(id -un)}" RUN_GROUP="$(id -gn "${RUN_USER}" 2>/dev/null || id -gn)" REPO_PARENT="$(dirname "${REPO_DIR}")" -echo "=== LoRA Training Git Deploy ===" +echo "=== LoRA Training Deploy ===" echo "Repo: ${REPO_URL}" echo "Dir: ${REPO_DIR}" echo "Branch: ${BRANCH}" +echo "" +# Create parent directory if needed if [ ! -d "${REPO_PARENT}" ]; then + echo "Creating ${REPO_PARENT}..." sudo install -d -m 0755 -o "${RUN_USER}" -g "${RUN_GROUP}" "${REPO_PARENT}" fi +# Clone or update repo if [ ! -d "${REPO_DIR}/.git" ]; then + echo "Cloning repository..." git clone --branch "${BRANCH}" "${REPO_URL}" "${REPO_DIR}" +else + echo "Repository exists, pulling latest..." + cd "${REPO_DIR}" + git fetch origin "${BRANCH}" + git checkout "${BRANCH}" + git pull --ff-only origin "${BRANCH}" fi -cd "${REPO_DIR}" - -# Ensure the remote URL matches the authenticated HTTPS URL. -git remote set-url "${REMOTE}" "${REPO_URL}" - -git fetch "${REMOTE}" "${BRANCH}" -git checkout "${BRANCH}" -git pull --ff-only "${REMOTE}" "${BRANCH}" - -# Setup Python environment and train echo "" -echo "=== Setting up Python environment ===" -python3 -m venv venv -source venv/bin/activate -pip install --upgrade pip -pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 -pip install transformers datasets trl peft accelerate bitsandbytes - -# Prepare dataset +echo "=== Repository ready ===" +echo "Location: ${REPO_DIR}" echo "" -echo "=== Preparing dataset ===" -python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data - -# Train +echo "=== Next Steps ===" echo "" -echo "=== Starting LoRA training ===" -python3 training/scripts/train.py --config training/configs/llama2-7b-lora.yaml - -# Step 3: Setup Python environment -echo "[3/5] Setting up Python environment..." -cd "$INSTALL_DIR" - -# Check if Python is available -if ! command -v python3 &> /dev/null; then - echo "ERROR: Python3 not found. Please install Python $PYTHON_VERSION or later." - exit 1 -fi - -# Create virtual environment -python3 -m venv venv -source venv/bin/activate - -# Upgrade pip -pip install --upgrade pip - -# Install PyTorch with CUDA support -echo " Installing PyTorch with CUDA support..." -pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 - -# Install other dependencies -echo " Installing training dependencies..." -pip install transformers datasets trl peft accelerate bitsandbytes - -# Verify GPU availability +echo "1. Download dataset (8.77 MB):" +echo " cd ${REPO_DIR}" +echo " curl -O https://gitea.cyaren.com/cmedina/agenx-lora-training/raw/branch/main/dataset/combined_20k.jsonl" echo "" -echo " Checking GPU availability..." -python3 -c " -import torch -if torch.cuda.is_available(): - print(f' ✓ CUDA available: {torch.cuda.get_device_name(0)}') - print(f' ✓ VRAM: {torch.cuda.get_device_properties(0).total_mem / 1e9:.1f} GB') -else: - print(' ✗ CUDA not available. GPU training will not work.') - print(' Please ensure CUDA drivers are installed.') - exit(1) -" - -# Step 4: Prepare dataset -echo "[4/5] Preparing dataset..." -python3 training/scripts/prepare_dataset.py \ - --input dataset/combined_20k.jsonl \ - --output training/data - -echo " Dataset prepared:" -echo " - training/data/train.jsonl" -echo " - training/data/test.jsonl" - -# Step 5: Train the model -echo "[5/5] Starting training..." +echo "2. Setup Python + GPU drivers on your training server" echo "" -echo " Training configuration:" -echo " - Model: meta-llama/Llama-2-7b-hf" -echo " - Method: QLoRA (4-bit quantization)" -echo " - Epochs: 3" -echo " - Batch size: 4" -echo " - Learning rate: 2e-4" +echo "3. Prepare dataset:" +echo " python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data" echo "" -echo " Estimated training time: 6-24 hours (depending on GPU)" +echo "4. Train LoRA (on GPU server):" +echo " python3 training/scripts/train.py --config training/configs/llama2-7b-lora.yaml" echo "" - -python3 training/scripts/train.py \ - --config training/configs/llama2-7b-lora.yaml - -echo "" -echo "==============================================" -echo "Training complete!" -echo "==============================================" -echo "" -echo "Trained model saved to: $INSTALL_DIR/training/output/llama2-7b-lora/" -echo "" -echo "To generate summaries:" -echo " cd $INSTALL_DIR" -echo " source venv/bin/activate" -echo " python3 training/scripts/inference.py \\" -echo " --model training/output/llama2-7b-lora \\" -echo " --task \"Your task here\" \\" -echo " --files src/file.py \\" -echo " --tests-run --test-count 100" +echo "=== To train now (if on GPU server) ===" +echo " bash ${REPO_DIR}/train-on-this-server.sh" echo "" diff --git a/train-on-this-server.sh b/train-on-this-server.sh new file mode 100644 index 0000000..79ad3c6 --- /dev/null +++ b/train-on-this-server.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Train LoRA on this server (assumes GPU is available). +# Run this after deploy-and-train.sh has cloned the repo. + +set -e + +REPO_DIR="${1:-$(pwd)}" + +echo "=== LoRA Training Setup ===" +echo "Repo: ${REPO_DIR}" +echo "" + +# Check Python +if ! command -v python3 &> /dev/null; then + echo "ERROR: python3 not found" + exit 1 +fi + +echo "[1/4] Setup Python environment..." +python3 -m venv venv +source venv/bin/activate +pip install --upgrade pip + +echo "[2/4] Install PyTorch with CUDA..." +pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 + +echo "[3/4] Install training dependencies..." +pip install transformers datasets trl peft accelerate bitsandbytes + +# Check GPU +echo "" +echo "[4/4] Checking GPU..." +python3 -c " +import torch +if torch.cuda.is_available(): + print(f' ✓ GPU: {torch.cuda.get_device_name(0)}') + print(f' ✓ VRAM: {torch.cuda.get_device_properties(0).total_mem / 1e9:.1f} GB') +else: + print(' ✗ No GPU detected') + print(' Please ensure CUDA drivers are installed') + exit(1) +" + +echo "" +echo "=== Dataset Preparation ===" +echo "Checking for combined_20k.jsonl..." +if [ ! -f "dataset/combined_20k.jsonl" ]; then + echo "Dataset not found. Downloading..." + curl -L -o dataset/combined_20k.jsonl https://gitea.cyaren.com/cmedina/agenx-lora-training/raw/branch/main/dataset/combined_20k.jsonl +fi + +echo "Preparing dataset..." +python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data + +echo "" +echo "=== Starting Training ===" +echo "Config: training/configs/llama2-7b-lora.yaml" +echo "Output: training/output/llama2-7b-lora/" +echo "" +echo "Training will take 6-24 hours depending on GPU." +echo "Press Ctrl+C to stop (model will be saved at checkpoint)." +echo "" + +python3 training/scripts/train.py --config training/configs/llama2-7b-lora.yaml + +echo "" +echo "=== Training Complete ===" +echo "Model saved to: training/output/llama2-7b-lora/" +echo "" +echo "To generate summaries:" +echo " python3 training/scripts/inference.py \\" +echo " --model training/output/llama2-7b-lora \\" +echo " --task \"Your task here\" \\" +echo " --files src/file.py"