Draft: Resolve "Konfigurieren für llama3:70b und slrum"

Closes #3 (closed)

scripts/batch.sh

+#!/bin/bash
+#SBATCH -e slurm-%j.err     # error file-path
+#SBATCH -o slurm-%j.out     # output file-path
+#SBATCH --nodes=1           # number of work nodes
+#SBATCH --ntasks=1          # number of tasks
+#SBATCH --mem=4G            # reserved RAM
+#SBATCH --cpus-per-task=1   # number of CPU cores per task, alternative: --cpus-per-gpu
+#SBATCH --gres=gpu:0        # number of GPUs
+#SBATCH --time=00:15:00     # Time limit for the entire job
+
+KUERZEL=$1
+REPOPATH=$2
+
+# Check argurment number
+if [ "$#" -ne 2 ]; then
+  echo "Found incorrect number of arguments ($#)."
+  echo "Please one user shorthand and one path to the zip file of the re-CHESS repo:"
+  printf '\tbatch.sh <user-shorthand> <repo-path>\n'
+  exit 1
+fi
+
+# Check if the second argument is an existing file
+if [ ! -f $2 ]; then
+  echo "File $2 does not exist."
+  echo " Please specify the path to the zip file of the re-CHESS repo:"
+  printf '\tbatch.sh <user-shorthand> <repo-path>\n'
+  exit 1
+fi
+
+# Create user directory on /zpool1/slurm_data/ if it doesn't exist yet
+srun if [ ! -d /zpool1/slurm_data/${KUERZEL} ]; then mkdir /zpool1/slurm_data/${KUERZEL}; fi
+
+# Create directory for the job
+srun mkdir /zpool1/slurm_data/${KUERZEL}/${SLURM_JOBID}
+srun mkdir /zpool1/slurm_data/${KUERZEL}/${SLURM_JOBID}/re-chess
+
+RECHESSROOT=/zpool1/slurm_data/${KUERZEL}/${SLURM_JOBID}/re-chess
+
+# Copy re-CHESS repo to the jobs folder and unpack
+sbcast $2 /zpool1/slurm_data/${KUERZEL}/${SLURM_JOBID}/re-chess.zip
+srun unzip /zpool1/slurm_data/${KUERZEL}/${SLURM_JOBID}/re-chess.zip -d ${RECHESSROOT}
+
+# Download and unpack Ollama
+srun curl -L https://ollama.com/download/ollama-linux-amd64.tgz -O ${RECHESSROOT}/ollama/ollama-linux-amd64.tgz
+srun tar -xzvf ${RECHESSROOT}/ollama/ollama-linux-amd64.tgz -C ${RECHESSROOT}/ollama
+OLPATH=${RECHESSROOT}/ollama
+OLLOGFILE=/dev/null
+OLLAMA_MODELS=${OLPATH}/model
+
+# Pull llama3:70b
+srun ${OLPATH}/./bin/ollama serve & sleep 10 ; ${OLPATH}/./bin/ollama pull llama3:70b ; kill %1
+
+# Download the BIRD & Spider dataset
+BIRDPATH=$RECHESSROOT/CHESS/data/BIRD
+SPIDERPATH=$RECHESSROOT/CHESS/data/Spider
+srun if [ ! -d ${BIRDPATH} ]; then mkdir ${BIRDPATH}; fi
+srun if [ ! -d ${SPIDERPATH} ]; then mkdir ${SPIDERPATH}; fi
+srun curl -L 'https://bird-bench.oss-cn-beijing.aliyuncs.com/dev.zip' -o ${BIRDPATH}/dev.zip
+srun curl -L 'https://bird-bench.oss-cn-beijing.aliyuncs.com/train.zip' -o ${BIRDPATH}/train.zip
+srun curl -L 'https://drive.usercontent.google.com/download?id=1403EGqzIDoHMdQF4c9Bkyl7dZLZ5Wt6J&confirm=t' -o ${SPIDERPATH}/spider_data.zip
+
+# Unpack BIRD & Spider datasets
+srun unzip ${BIRDPATH}/dev.zip -d ${BIRDPATH}
+srun unzip ${BIRDPATH}/train.zip -d ${BIRDPATH}
+srun unzip ${SPIDERPATH}/spider_data.zip -d ${SPIDERPATH}
+
+# Unpack BIRD specific sub-directory
+srun mv ${BIRDPATH}/dev_20240627 ${BIRDPATH}/dev
+srun unzip ${BIRDPATH}/dev_databases.zip -d ${BIRDPATH}/dev
+srun unzip ${BIRDPATH}/train_databases.zip -d ${BIRDPATH}/train
+
+# Create a Python venv
+PYTHONVER=$(py3versions --default)
+srun ${PYTHONVER} -m venv ${RECHESSROOT}/CHESS/venv
+VENVPATH=${RECHESSROOT}/CHESS/venv
+srun source ${VENVPATH}/bin/activate ; pip3 install -r ${RECHESSROOT}/CHESS/requirements.txt ; deactivate
+
+# copy the .env.rechess file to .env
+srun cp ${RECHESSROOT}/CHESS/.env.rechess ${RECHESSROOT}/CHESS/.env
+
+# run preprocessing script
+srun cd ${RECHESSROOT}; source ${VENVPATH}/bin/activate; ${OLPATH}/./bin/ollama serve &> ${OLLOGFILE} & sleep 10; /run/./run-preprocess.sh ; kill %1; deactivate; cd $OLDPWD
+
+# run the agent script
+srun cd ${RECHESSROOT}; source ${VENVPATH}/bin/activate; ${OLPATH}/./bin/ollama serve &> ${OLLOGFILE} & sleep 10; /run/./run_main_ir_ss_cg_rechess.sh ; kill %1; deactivate; cd $OLDPWD
+
+# gather results
+
+##srun /zpool1/
+##srun ls -la /lib
+##srun ls -la /bin
+##srun ls -la /zpool1/slurm_data/${KUERZEL}/${SLURM_JOBID}
+##srun ls -la ${OLPATH}
+##srun ls -la ${OLPATH}/model
+
+# Remove all the data of the job to free memory on the node
+srun rm -r /zpool1/slurm_data/${KUERZEL}/${SLURM_JOBID}