
factor=$1
scaling_type=$2
model_type=$3
task_name=$4
step=$5


if [[ "$@" == *"flash_attn"* ]]; then
  flash_attn=1
else
  flash_attn=0
fi
echo "flash_attn: ${flash_attn}"


prefix=/scratch2/nlp/wutong/${task_name}/${model_type}_results


echo $scaling_type and $factor
python -u src/eval_ppl.py \
    --path_to_ckp ${prefix}/4k-$((factor*4))k-${scaling_type}/checkpoint-${step} \
    --model_name 7b-4k-$((factor*4))k-${scaling_type} \
    --rope_scaling_type ${scaling_type} \
    --rope_scaling_factor ${factor} \
    --model_max_position_embeddings 4096 \
    --max_input_tokens 262144 \
    --min_input_tokens 262144 \
    --sliding_window_step 4096 \
    --window_length_list 65536 98304 131072 196608 262144 \
    --batch_size 1 \
    --eval_nums 20 \
    --use_flash_attn ${flash_attn} \
    --dataset_name pg19 \
    --path_to_dataset /scratch/nlp/wutong/dataset/PoSE-Datasets/pile/test_pg19.jsonl \
    --path_to_output_dir eval_output/PPL/${model_type}-${task_name}-${step}

python -u src/eval_ppl.py \
    --path_to_ckp ${prefix}/4k-$((factor*4))k-${scaling_type}/checkpoint-${step} \
    --model_name 7b-4k-$((factor*4))k-${scaling_type} \
    --rope_scaling_type ${scaling_type} \
    --rope_scaling_factor ${factor} \
    --model_max_position_embeddings 4096 \
    --max_input_tokens 262144 \
    --min_input_tokens 262144 \
    --sliding_window_step 4096 \
    --window_length_list 65536 98304 131072 196608 262144 \
    --batch_size 1 \
    --eval_nums 20 \
    --use_flash_attn ${flash_attn} \
    --dataset_name books3 \
    --path_to_dataset /scratch/nlp/wutong/dataset/PoSE-Datasets/pile/test_books3.jsonl \
    --path_to_output_dir eval_output/PPL/${model_type}-${task_name}-${step}

