Data Mining Projekt

"transformers=4.49.0",
"datasets=3.3.1",
"tiktoken=0.5.2",
"torch=2.1.2",
"peft=0.4.0",
"trl=0.10.1",
"prettytable",
"func-timeout",
"sqlparse",
"jsonlines",
"rouge-chinese=1.0.3",
"jieba=0.42.1",
"nltk=3.9.1",
"matplotlib=3.10.0",
"bitsandbytes=0.41.1",
"accelerate"

cd DB-GPT-Hub/src/dbgpt_hub_sql
sh dbgpt_hub_sql/scripts/gen_train_eval_data.sh

dataset="example_text2sql_train"
model_name_or_path=${model_name_or_path-"Qwen/Qwen2.5-14B"}
output_dir="src/dbgpt_hub_sql/dbgpt_hub_sql/output/adapter/qwen14b_qlora"
template="chatml"

CUDA_VISIBLE_DEVICES=0 python src/dbgpt_hub_sql/dbgpt_hub_sql/train/sft_train.py \
    --model_name_or_path $model_name_or_path \
    --quantization_bit 4 \
    --do_train \
    --dataset $dataset \
    --max_source_length 2048 \
    --max_target_length 512 \
    --finetuning_type lora \
    --lora_target q_proj,v_proj \
    --template $template \
    --lora_rank 64 \
    --lora_alpha 32 \
    --output_dir $output_dir \
    --overwrite_cache \
    --overwrite_output_dir \
    --per_device_train_batch_size 1 \
    --gradient_accumulation_steps 16 \
    --lr_scheduler_type cosine_with_restarts \
    --logging_steps 50 \
    --save_steps 2000 \
    --learning_rate 2e-4 \
    --num_train_epochs 8 \
    --plot_loss \
    --bf16  >> ${train_log}
    
#-------PREDICT---------------

CUDA_VISIBLE_DEVICES=0 python src/dbgpt_hub_sql/dbgpt_hub_sql/predict/predict.py \
    --model_name_or_path $model_name_or_path \
    --quantization_bit 4 \
    --template $template \
    --finetuning_type lora \
    --checkpoint_dir $output_dir \
    --predicted_input_filename src/dbgpt_hub_sql/dbgpt_hub_sql/data/spider_dev.json \
    --predicted_out_filename src/dbgpt_hub_sql/dbgpt_hub_sql/output/pred/qwen14b_qlora.sql >> ${pred_log}

#!/bin/bash
#SBATCH --job-name=pipe1
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-gpu=8
#SBATCH --mem-per-cpu=16G
#SBATCH --output=./Slurm-Outputs/%j_%x.out

source ../venv/bin/activate
cd ./DB-GPT-Hub

sh src/dbgpt_hub_sql/dbgpt_hub_sql/scripts/pipeline.sh

sbatch --nodelist=workg01 batch_pipeline.sh

cd DB-GPT-Hub/src/dbgpt_hub_sql/
python dbgpt_hub_sql/eval/evaluation.py --plug_value --input [predicted file] --etype [evaluation type]

  [evaluation type]  "match" for EM, "exec" for EX

Bsp.:
 python dbgpt_hub_sql/eval/evaluation.py --plug_value --input dbgpt_hub_sql/output/pred/qwen7bchat_qlora.sql --etype "exec"