Skip to content

Commit 5bc8713

Browse files
Dando18Dando18
authored
Add magicoder (#20)
add magicoder outputs
1 parent 2912d7a commit 5bc8713

30 files changed

Lines changed: 595689 additions & 56 deletions

‎drivers/cpp/cpp_driver_wrapper.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ def run(self, executable: PathLike, **run_config) -> RunOutput:
106106
run_process = run_command(launch_cmd, timeout=self.run_timeout, dry=self.dry)
107107
except subprocess.TimeoutExpired as e:
108108
return RunOutput(-1, str(e.stdout), f"[Timeout] {str(e.stderr)}", config=run_config)
109+
except UnicodeDecodeError as e:
110+
logging.warning(f"UnicodeDecodeError: {str(e)}\nRunnning command: {launch_cmd}")
111+
return RunOutput(-1, "", f"UnicodeDecodeError: {str(e)}", config=run_config)
109112
return RunOutput(run_process.returncode, run_process.stdout, run_process.stderr, config=run_config)
110113

111114
def test_single_output(self, prompt: str, output: str, test_driver_file: PathLike, problem_size: str) -> GeneratedTextResult:

‎drivers/driver_wrapper.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ def __init__(self, exit_code: int, stdout: str, stderr: str, config: dict = {}):
4646
logging.warning(f"Runtime is 0 for run with config {self.config}. Try increasing the problem size.")
4747
if self.is_valid and self.best_sequential_runtime == 0:
4848
logging.warning(f"The best sequential runtime is 0 for run with config {self.config}. Try increasing the problem size.")
49-
if self.is_valid and self.best_sequential_runtime < 0.001:
49+
if self.is_valid and self.best_sequential_runtime and self.best_sequential_runtime < 0.001:
5050
logging.warning(f"The best sequential runtime is very small ({self.best_sequential_runtime}) for run with config {self.config}. Try increasing the problem size.")
5151

5252
def __repr__(self) -> str:

‎drivers/run-cuda.sbatch‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
#SBATCH --gpus=a100:1
55
#SBATCH -t 05:00:00
66
#SBATCH -A bhatele-lab-cmsc
7-
#SBATCH -J cuda-gemini-pro
8-
#SBATCH -o run-outputs/gemini-pro-cuda-%A.out
7+
#SBATCH -J cuda-magicoder-s-ds-6.7b
8+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-cuda-%A.out
99

1010
# settings
1111
HASH="015cff6f"
12-
MODEL="gemini-pro"
12+
MODEL="magicoder-s-ds-6.7b"
1313
TEMP="0.2"
1414

1515
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-hip.sbatch‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
#!/bin/bash
22
#SBATCH -N 1
33
#SBATCH -t 08:00:00
4-
#SBATCH -J hip-gemini-pro
5-
#SBATCH -o run-outputs/gemini-pro-hip-{{id}}.out
4+
#SBATCH -J hip-magicoder-s-ds-6.7b
5+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-hip-{{id}}.out
66

77
# settings
88
HASH="015cff6f"
9-
MODEL="gemini-pro"
9+
MODEL="magicoder-s-ds-6.7b"
1010
TEMP="0.2"
1111

1212
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-kokkos.sbatch‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
#SBATCH -p serial
55
#SBATCH -t 05:00:00
66
#SBATCH -A bhatele-lab-cmsc
7-
#SBATCH -J kokkos-gemini-pro
8-
#SBATCH -o run-outputs/gemini-pro-kokkos-%A.out
7+
#SBATCH -J kokkos-magicoder-s-ds-6.7b
8+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-kokkos-%A.out
99

1010
# settings
1111
HASH="015cff6f"
12-
MODEL="gemini-pro"
12+
MODEL="magicoder-s-ds-6.7b"
1313
TEMP="0.2"
1414

1515
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-mpi+omp.sbatch‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,14 @@
22
#SBATCH -N 4
33
#SBATCH --exclusive
44
#SBATCH -t 05:00:00
5+
#SBATCH -t 05:00:00
56
#SBATCH -A bhatele-lab-cmsc
6-
#SBATCH -J mpi+omp-gemini-pro
7-
#SBATCH -o run-outputs/gemini-pro-mpi+omp-%A.out
7+
#SBATCH -J mpi+omp-magicoder-s-ds-6.7b
8+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-mpi+omp-%A.out
89

910
# settings
1011
HASH="015cff6f"
11-
MODEL="gemini-pro"
12+
MODEL="magicoder-s-ds-6.7b"
1213
TEMP="0.2"
1314

1415
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-mpi.sbatch‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22
#SBATCH -n 512
33
#SBATCH -t 04:00:00
44
#SBATCH -A bhatele-lab-cmsc
5-
#SBATCH -J mpi-gemini-pro
6-
#SBATCH -o run-outputs/gemini-pro-mpi-%A.out
5+
#SBATCH -J mpi-magicoder-s-ds-6.7b
6+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-mpi-%A.out
77

88
# settings
99
HASH="015cff6f"
10-
MODEL="gemini-pro"
10+
MODEL="magicoder-s-ds-6.7b"
1111
TEMP="0.2"
1212

1313
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"
@@ -19,7 +19,7 @@ ml python gcc openmpi
1919
export OMPI_MCA_opal_warn_on_missing_libcuda=0
2020

2121
python run-all.py \
22-
$OUTPUT \
22+
$GENERATED_PROMPTS \
2323
--output $OUTPUT \
2424
--scratch-dir $SCRATCH_DIR \
2525
--launch-configs launch-configs.json \

‎drivers/run-omp.sbatch‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
#SBATCH --exclusive
44
#SBATCH -p serial
55
#SBATCH -t 05:00:00
6+
#SBATCH -t 05:00:00
67
#SBATCH -A bhatele-lab-cmsc
7-
#SBATCH -J omp-gemini-pro
8-
#SBATCH -o run-outputs/gemini-pro-omp-%A.out
8+
#SBATCH -J omp-magicoder-s-ds-6.7b
9+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-omp-%A.out
910

1011
# settings
1112
HASH="015cff6f"
12-
MODEL="gemini-pro"
13+
MODEL="magicoder-s-ds-6.7b"
1314
TEMP="0.2"
1415

1516
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎drivers/run-serial.sbatch‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
#SBATCH --exclusive
44
#SBATCH -p serial
55
#SBATCH -t 04:00:00
6+
#SBATCH -t 04:00:00
67
#SBATCH -A bhatele-lab-cmsc
7-
#SBATCH -J serial-gemini-pro
8-
#SBATCH -o run-outputs/gemini-pro-serial-%A.out
8+
#SBATCH -J serial-magicoder-s-ds-6.7b
9+
#SBATCH -o run-outputs/magicoder-s-ds-6.7b-serial-%A.out
910

1011
# settings
1112
HASH="015cff6f"
12-
MODEL="gemini-pro"
13+
MODEL="magicoder-s-ds-6.7b"
1314
TEMP="0.2"
1415

1516
GENERATED_PROMPTS="../results/${HASH}/${MODEL}_temp${TEMP}/all.json"

‎generate/generate-magicoder-s-ds-prompted.sbatch‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
#SBATCH -n 1
3+
#SBATCH -c 4
4+
#SBATCH --ntasks-per-node=1
5+
#SBATCH --gpus-per-task=1
6+
#SBATCH --mem=128000
7+
#SBATCH -t 05:00:00
8+
#SBATCH -A m2404
9+
#SBATCH -C gpu&hbm80g
10+
#SBATCH -q regular
11+
#SBATCH -J generate-magicoder-s-ds-prompted
12+
#SBATCH -o generation-job-logs/generate-magicoder-s-ds-prompted-%A.out
13+
14+
# settings
15+
MODEL="ise-uiuc/Magicoder-S-DS-6.7B"
16+
TEMP=0.2
17+
TOPP=0.95
18+
MAX_NEW_TKNS=2048
19+
SAMPLES_PER_PROMPT=20
20+
BATCH_SIZE=16
21+
hash=$(md5sum ../prompts/generation-prompts.json | cut -d' ' -f1)
22+
OUTPUT="../outputs/output_${hash:0:8}_${MODEL//\//--}_prompted_temp${TEMP}.json"
23+
CACHE="../outputs/cache/cache_${hash:0:8}_${MODEL//\//--}_prompted_temp${TEMP}.jsonl"
24+
echo "Writing to $OUTPUT"
25+
echo "model=$MODEL MAX_NEW_TKNS=$MAX_NEW_TKNS SAMPLES_PER_PROMPT=$SAMPLES_PER_PROMPT BATCH_SIZE=$BATCH_SIZE"
26+
27+
# setup
28+
#ml cuda/11.8.0
29+
source .env/bin/activate
30+
export HF_HOME=/pscratch/sd/d/dnicho/.cache/huggingface
31+
export OMP_NUM_THREADS=4
32+
33+
# generate
34+
srun python generate.py \
35+
--model $MODEL \
36+
--prompts ../prompts/generation-prompts.json \
37+
--output $OUTPUT \
38+
--cache $CACHE \
39+
--temperature $TEMP \
40+
--top_p $TOPP \
41+
--do_sample \
42+
--max_new_tokens $MAX_NEW_TKNS \
43+
--num_samples_per_prompt $SAMPLES_PER_PROMPT \
44+
--batch_size $BATCH_SIZE \
45+
--prompted

0 commit comments

Comments
 (0)