fix: auto-split dataset if split files missing
This commit is contained in:
@@ -60,8 +60,13 @@ if [ ! -f "dataset/combined_20k.jsonl" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Preparing dataset..."
|
||||
echo "Checking for train.jsonl and test.jsonl..."
|
||||
if [ ! -f "training/data/train.jsonl" ] || [ ! -f "training/data/test.jsonl" ]; then
|
||||
echo "Split files not found, creating them..."
|
||||
python3 training/scripts/prepare_dataset.py --input dataset/combined_20k.jsonl --output training/data
|
||||
else
|
||||
echo "Split files already exist, using them."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Starting Training ==="
|
||||
|
||||
Reference in New Issue
Block a user