Skip to content

Commit e171e90

Browse files
seofernando25seofernando25
authored
Update report (#10)
* remove train cnn * update notebook labels * update report * Add current fitness evaluation figure * Add current fitness overlay visualization * Update report * Update report * Update report
1 parent 8416eae commit e171e90

3 files changed

Lines changed: 120 additions & 189 deletions

File tree

‎notebooks/experiment_analysis.ipynb‎

Lines changed: 117 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,16 @@
2727
"FINAL_TRAINING_DIR = REPO_ROOT / '.cache' / 'final_training'\n",
2828
"COLORS = {'RS': '#1f77b4', 'GA-STANDARD': '#ff7f0e', 'GA-MEMETIC': '#d62728', 'PSO': '#2ca02c'}\n",
2929
"\n",
30+
"\n",
31+
"def _exp_smooth(values: np.ndarray, smoothing_slider: float) -> np.ndarray:\n",
32+
" smoothed = np.empty_like(values, dtype=np.float32)\n",
33+
" last = values[0]\n",
34+
" for idx, point in enumerate(values):\n",
35+
" last = smoothing_slider * last + (1.0 - smoothing_slider) * point\n",
36+
" smoothed[idx] = last\n",
37+
" return smoothed\n",
38+
"\n",
39+
"\n",
3040
"def parse_experiment_name(exp_name):\n",
3141
" if '-' not in exp_name:\n",
3242
" return None, None\n",
@@ -95,17 +105,13 @@
95105
"for ax, model in zip(axes, models):\n",
96106
" optimizers = sorted(hpo_grouped[model].keys())\n",
97107
" data_to_plot = [hpo_grouped[model][opt] for opt in optimizers]\n",
98-
" all_values = [val for sublist in data_to_plot for val in sublist]\n",
99108
" \n",
100-
" if all_values:\n",
101-
" y_min, y_max = np.percentile(all_values, [2, 98])\n",
102-
" ax.set_ylim(y_min - (y_max - y_min) * 0.1, y_max + (y_max - y_min) * 0.1)\n",
103109
" \n",
104110
" bp = ax.boxplot(data_to_plot, tick_labels=optimizers, patch_artist=True)\n",
105111
" for patch in bp['boxes']:\n",
106112
" patch.set_facecolor('lightblue')\n",
107113
" \n",
108-
" ax.set_title(f'{model} - Final Fitness (HPO)', fontweight='bold')\n",
114+
" ax.set_title(f'{model}', fontweight='bold')\n",
109115
" ax.set_xlabel('Optimizer')\n",
110116
" ax.set_ylabel('Composite Fitness')\n",
111117
" ax.grid(True, alpha=0.3)\n",
@@ -180,7 +186,7 @@
180186
" f'{val:.4f}', ha='center', va='bottom', fontsize=9)\n",
181187
" \n",
182188
" ax.set_ylim(0, 1)\n",
183-
" ax.set_title(f'{model} - Test Performance', fontweight='bold')\n",
189+
" ax.set_title(f'{model}', fontweight='bold')\n",
184190
" ax.set_xlabel('Optimizer')\n",
185191
" ax.set_ylabel('Composite Fitness')\n",
186192
" ax.grid(True, alpha=0.3, axis='y')\n",
@@ -252,8 +258,8 @@
252258
" ax.fill_between(generations, mean_curve - std_curve, mean_curve + std_curve, \n",
253259
" color=color, alpha=0.2)\n",
254260
" \n",
255-
" ax.set_title(f'{model} - Convergence', fontweight='bold')\n",
256-
" ax.set_xlabel('Generation')\n",
261+
" ax.set_title(f'{model}', fontweight='bold')\n",
262+
" ax.set_xlabel('Evaluation Count ($n$)')\n",
257263
" ax.set_ylabel('Best Fitness')\n",
258264
" ax.legend()\n",
259265
" ax.grid(True, alpha=0.3)\n",
@@ -262,6 +268,109 @@
262268
"plt.show()\n"
263269
]
264270
},
271+
{
272+
"cell_type": "markdown",
273+
"metadata": {},
274+
"source": [
275+
"## Current Fitness Overlays\n"
276+
]
277+
},
278+
{
279+
"cell_type": "code",
280+
"execution_count": null,
281+
"metadata": {},
282+
"outputs": [],
283+
"source": [
284+
"SMOOTHING_SLIDER_CURRENT_FITNESS = 0.5\n",
285+
"current_overlay = defaultdict(lambda: defaultdict(list))\n",
286+
"\n",
287+
"for exp_dir in sorted(EXPERIMENT_DIR.iterdir()):\n",
288+
" if not exp_dir.is_dir():\n",
289+
" continue\n",
290+
" model, optimizer = parse_experiment_name(exp_dir.name)\n",
291+
" if not optimizer:\n",
292+
" continue\n",
293+
" for run_dir in sorted(exp_dir.iterdir()):\n",
294+
" conv_path = run_dir / 'convergence.json'\n",
295+
" with open(conv_path) as f:\n",
296+
" conv = json.load(f)\n",
297+
" evaluations = conv.get('evaluations')\n",
298+
" current_fitness = conv.get('current_fitness')\n",
299+
" if evaluations and current_fitness:\n",
300+
" current_overlay[model][optimizer].append((evaluations, current_fitness))\n",
301+
"\n",
302+
"total_runs = sum(len(v) for model_data in current_overlay.values() for v in model_data.values())\n",
303+
"print(f\"Loaded current-fitness traces for {total_runs} runs\\n\")\n",
304+
"\n",
305+
"models = sorted(current_overlay.keys())\n",
306+
"# Prefer consistent optimizer ordering; fallback to sorted discovered optimizers\n",
307+
"preferred_order = [\"RS\", \"GA-STANDARD\", \"GA-MEMETIC\", \"PSO\"]\n",
308+
"observed_opts = sorted({opt for model_data in current_overlay.values() for opt in model_data.keys()})\n",
309+
"optimizer_order = [opt for opt in preferred_order if opt in observed_opts]\n",
310+
"optimizer_order += [opt for opt in observed_opts if opt not in optimizer_order]\n",
311+
"\n",
312+
"fig, axes = plt.subplots(len(optimizer_order), len(models), figsize=(6 * len(models), 3 * len(optimizer_order)))\n",
313+
"if len(optimizer_order) == 1 and len(models) == 1:\n",
314+
" axes = [[axes]]\n",
315+
"elif len(optimizer_order) == 1:\n",
316+
" axes = [axes]\n",
317+
"elif len(models) == 1:\n",
318+
" axes = [[ax] for ax in axes]\n",
319+
"\n",
320+
"for row_idx, optimizer in enumerate(optimizer_order):\n",
321+
" for col_idx, model in enumerate(models):\n",
322+
" ax = axes[row_idx][col_idx]\n",
323+
" runs = current_overlay[model].get(optimizer, [])\n",
324+
" if not runs:\n",
325+
" ax.text(0.5, 0.5, 'N/A', ha='center', va='center')\n",
326+
" ax.set_xticks([])\n",
327+
" ax.set_yticks([])\n",
328+
" if row_idx == 0:\n",
329+
" ax.set_title(model, fontweight='bold')\n",
330+
" if col_idx == 0:\n",
331+
" ax.set_ylabel(optimizer)\n",
332+
" continue\n",
333+
"\n",
334+
" base_eval = np.array(runs[0][0])\n",
335+
" aligned = []\n",
336+
" for evals, curr in runs:\n",
337+
" evals_arr = np.array(evals)\n",
338+
" curr_arr = np.array(curr)\n",
339+
" if len(evals_arr) != len(base_eval) or not np.allclose(evals_arr, base_eval):\n",
340+
" curr_arr = np.interp(base_eval, evals_arr, curr_arr)\n",
341+
" aligned.append(curr_arr)\n",
342+
" aligned = np.array(aligned)\n",
343+
" mean_curve = aligned.mean(axis=0)\n",
344+
" std_curve = aligned.std(axis=0)\n",
345+
" mean_smoothed = _exp_smooth(mean_curve, SMOOTHING_SLIDER_CURRENT_FITNESS)\n",
346+
" std_smoothed = _exp_smooth(std_curve, SMOOTHING_SLIDER_CURRENT_FITNESS)\n",
347+
" color = COLORS.get(optimizer, '#888888')\n",
348+
" ax.fill_between(base_eval, mean_smoothed - std_smoothed, mean_smoothed + std_smoothed,\n",
349+
" color=color, alpha=0.2)\n",
350+
" ax.plot(base_eval, mean_smoothed, label=optimizer, color=color, linewidth=2)\n",
351+
"\n",
352+
" # set per-model y-limits\n",
353+
" model_key = model.lower()\n",
354+
" if model_key == 'cnn':\n",
355+
" ax.set_ylim(0.40, 0.85)\n",
356+
" elif model_key == 'dt':\n",
357+
" ax.set_ylim(0.275, 0.36)\n",
358+
" elif model_key == 'knn':\n",
359+
" ax.set_ylim(0.225, 0.475)\n",
360+
"\n",
361+
" ax.grid(True, alpha=0.3)\n",
362+
" if row_idx == len(optimizer_order) - 1:\n",
363+
" ax.set_xlabel('Evaluation ($n$)')\n",
364+
" if col_idx == 0:\n",
365+
" ax.set_ylabel(f'{optimizer}\\nCurrent Fitness')\n",
366+
" if row_idx == 0:\n",
367+
" ax.set_title(f'{model}', fontweight='bold')\n",
368+
" ax.legend(fontsize=8, loc='best')\n",
369+
"\n",
370+
"plt.tight_layout()\n",
371+
"plt.show()\n"
372+
]
373+
},
265374
{
266375
"cell_type": "markdown",
267376
"metadata": {},

‎report/sections/6_conclusion.tex‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,12 @@
11
\section{Conclusion}
22

3-
\subsection*{RQ1: Effectiveness and Convergence Rates against Baseline}
3+
This study evaluated metaheuristic optimizers (GA, PSO) against a Randomized Search baseline under a strict budget of 50 evaluations. No significant performance difference was found between methods across Decision Tree, KNN, or CNN models ($p > 0.05$), with all reaching similar fitness plateaus.
44

5-
This study evaluated metaheuristic optimizers (GA, PSO) against a Randomized Search baseline under a strict budget of 50 evaluations.
6-
7-
\subsection*{RQ2: Difference in Stability}
8-
No significant performance difference was found between searching methods across Decision Tree, KNN, or CNN models ($p > 0.05$), with all reaching similar fitness plateaus.
9-
10-
\subsection*{Further Observations}
115
The result is explained by initialization overhead: with a population of 30, GA used 60\% of its budget on initial sampling, leaving too few evaluations for evolutionary operators to yield improvement. In such micro-budget regimes (budget $< 2 \times$ population), population-based methods behave similarly to Random Search.
126

13-
\paragraph{Insufficient Budget} Therefore, for hyperparameter optimization with very limited evaluations, the added complexity of metaheuristics like GA and PSO is not justified. Random Search proved equally effective baseline under these constraints.
7+
Therefore, for hyperparameter optimization with very limited evaluations, the added complexity of metaheuristics like GA and PSO is not justified. Random Search proved equally effective baseline under these constraints.
148

15-
\subsection*{Future Work}
16-
\vspace{1em}
17-
\noindent Further contribution should:
9+
Future work should:
1810
\begin{itemize}
1911
\item Increase evaluation budgets to allow amortization of initialization costs.
2012
\item Explore adaptive or budget-aware variants of GA and PSO.

‎scripts/train_cnn.py‎

Lines changed: 0 additions & 170 deletions
This file was deleted.

0 commit comments

Comments
 (0)