tower-eval/run_paper_benchmark.sh at main · deep-spin/tower-eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
#!/bin/bash

OPENAI=false

while (( "$#" )); do
  case "$1" in
    --openai)
      OPENAI=true
      shift
      ;;
    --) # end argument parsing
      shift
      break
      ;;
    -*|--*=) # unsupported flags
      echo "Error: Unsupported flag $1" >&2
      exit 1
      ;;
    *) # preserve positional arguments
      PARAMS="$PARAMS $1"
      shift
      ;;
  esac
done

if [ "$OPENAI" = true ] ; then
    configs=(configs/tower_paper/0_shot_openai.yaml configs/tower_paper/5_shot_generic_models.yaml configs/tower_paper/5_shot_openai.yaml configs/tower_paper/tower_instruct_0_shot.yaml configs/tower_paper/tower_instruct_5_shot.yaml)
    echo "Running Tower paper benchmark including open-ai models."
else
    configs=(configs/tower_paper/5_shot_generic_models.yaml configs/tower_paper/tower_instruct_0_shot.yaml configs/tower_paper/tower_instruct_5_shot.yaml)
    echo "Running Tower paper benchmark for open models only."
fi

for config in "${configs[@]}"; do
    echo "Running $config"
    python -m tower_eval.cli gen-eval --config $config
done