Skip to content

Commit f7e03cc

Browse files
magicbugmagicbug
committed
Optimize grid 'new' checks with historical data caching
Replaced repeated database queries for grid 'new' checks with pre-built in-memory caches, significantly improving performance by reducing hundreds of queries to a few. Added build_historical_caches() to load all relevant historical grid data at once and refactored is_grid_new, is_grid_new_for_prop, and is_grid_new_for_hf to use these caches for instant lookups.
1 parent 36ae819 commit f7e03cc

1 file changed

Lines changed: 66 additions & 106 deletions

File tree

‎application/models/Monthlyreport_model.php‎

Lines changed: 66 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ public function generate_report($logbooks_locations_array, $year, $month) {
1919
// Calculate previous month end date for "new" calculations
2020
$prev_month_end = date('Y-m-d 23:59:59', strtotime($start_date . ' -1 day'));
2121

22+
// PERFORMANCE OPTIMIZATION: Pre-load all historical data for "new" checks
23+
// This reduces hundreds of queries to just a few
24+
$this->build_historical_caches($logbooks_locations_array, $start_date);
25+
2226
$report = array(
2327
'total_qsos' => $this->get_total_qsos($logbooks_locations_array, $start_date, $end_date),
2428
'unique_callsigns' => $this->get_unique_callsigns($logbooks_locations_array, $start_date, $end_date),
@@ -55,6 +59,62 @@ public function generate_report($logbooks_locations_array, $year, $month) {
5559
return $report;
5660
}
5761

62+
// Cache variables for historical data lookups
63+
private $historical_grids = array();
64+
private $historical_grids_by_prop = array();
65+
private $historical_grids_hf = array();
66+
67+
/**
68+
* Build historical caches for fast "new" checks
69+
* This prevents hundreds of individual queries
70+
*/
71+
private function build_historical_caches($locations, $start_date) {
72+
// Get ALL historical gridsquares (before this month)
73+
$this->db->select('COL_GRIDSQUARE, COL_VUCC_GRIDS, COL_PROP_MODE');
74+
$this->db->from($this->config->item('table_name'));
75+
$this->db->where_in('station_id', $locations);
76+
$this->db->where('COL_TIME_ON <', $start_date);
77+
$this->db->where("(COL_GRIDSQUARE IS NOT NULL AND COL_GRIDSQUARE != '') OR (COL_VUCC_GRIDS IS NOT NULL AND COL_VUCC_GRIDS != '')");
78+
79+
$query = $this->db->get();
80+
81+
foreach ($query->result() as $row) {
82+
$prop_mode = !empty($row->COL_PROP_MODE) ? $row->COL_PROP_MODE : '';
83+
84+
// Process main gridsquare
85+
if (!empty($row->COL_GRIDSQUARE) && strlen($row->COL_GRIDSQUARE) >= 4) {
86+
$grid_4char = strtoupper(substr($row->COL_GRIDSQUARE, 0, 4));
87+
$this->historical_grids[$grid_4char] = true;
88+
89+
// Track by prop mode
90+
if ($prop_mode === 'SAT' || $prop_mode === 'EME') {
91+
$this->historical_grids_by_prop[$prop_mode][$grid_4char] = true;
92+
} else {
93+
$this->historical_grids_hf[$grid_4char] = true;
94+
}
95+
}
96+
97+
// Process VUCC grids
98+
if (!empty($row->COL_VUCC_GRIDS)) {
99+
$vucc_grids = explode(',', $row->COL_VUCC_GRIDS);
100+
foreach ($vucc_grids as $grid) {
101+
$grid = trim($grid);
102+
if (strlen($grid) >= 4) {
103+
$grid_4char = strtoupper(substr($grid, 0, 4));
104+
$this->historical_grids[$grid_4char] = true;
105+
106+
// Track by prop mode
107+
if ($prop_mode === 'SAT' || $prop_mode === 'EME') {
108+
$this->historical_grids_by_prop[$prop_mode][$grid_4char] = true;
109+
} else {
110+
$this->historical_grids_hf[$grid_4char] = true;
111+
}
112+
}
113+
}
114+
}
115+
}
116+
}
117+
58118
/**
59119
* Sort bands in amateur radio order (160m, 80m, 40m, 20m, etc.)
60120
*/
@@ -477,124 +537,24 @@ private function get_new_gridsquares_hf($locations, $start_date, $end_date, $pre
477537
* Check if a grid is new (never worked before)
478538
*/
479539
private function is_grid_new($locations, $grid, $start_date) {
480-
// Check if grid exists in COL_GRIDSQUARE (first 4 chars match)
481-
$this->db->select('COUNT(*) as count');
482-
$this->db->from($this->config->item('table_name'));
483-
$this->db->where_in('station_id', $locations);
484-
$this->db->where('COL_TIME_ON <', $start_date);
485-
$this->db->where('UPPER(SUBSTR(COL_GRIDSQUARE, 1, 4)) =', strtoupper($grid));
486-
487-
$query = $this->db->get();
488-
if ($query->row()->count > 0) {
489-
return false;
490-
}
491-
492-
// Check if grid exists in COL_VUCC_GRIDS
493-
$this->db->select('COL_VUCC_GRIDS');
494-
$this->db->from($this->config->item('table_name'));
495-
$this->db->where_in('station_id', $locations);
496-
$this->db->where('COL_TIME_ON <', $start_date);
497-
$this->db->where('COL_VUCC_GRIDS IS NOT NULL');
498-
$this->db->where('COL_VUCC_GRIDS !=', '');
499-
500-
$query = $this->db->get();
501-
foreach ($query->result() as $row) {
502-
if (!empty($row->COL_VUCC_GRIDS)) {
503-
$vucc_grids = explode(',', $row->COL_VUCC_GRIDS);
504-
foreach ($vucc_grids as $vucc_grid) {
505-
$vucc_grid_4char = strtoupper(substr(trim($vucc_grid), 0, 4));
506-
if ($vucc_grid_4char === strtoupper($grid)) {
507-
return false;
508-
}
509-
}
510-
}
511-
}
512-
513-
return true;
540+
// Use pre-built cache for instant lookup
541+
return !isset($this->historical_grids[strtoupper($grid)]);
514542
}
515543

516544
/**
517545
* Check if a grid is new for a specific propagation mode
518546
*/
519547
private function is_grid_new_for_prop($locations, $grid, $start_date, $prop_mode) {
520-
// Check if grid exists in COL_GRIDSQUARE (first 4 chars match) with prop mode
521-
$this->db->select('COUNT(*) as count');
522-
$this->db->from($this->config->item('table_name'));
523-
$this->db->where_in('station_id', $locations);
524-
$this->db->where('COL_TIME_ON <', $start_date);
525-
$this->db->where('COL_PROP_MODE', $prop_mode);
526-
$this->db->where('UPPER(SUBSTR(COL_GRIDSQUARE, 1, 4)) =', strtoupper($grid));
527-
528-
$query = $this->db->get();
529-
if ($query->row()->count > 0) {
530-
return false;
531-
}
532-
533-
// Check if grid exists in COL_VUCC_GRIDS with prop mode
534-
$this->db->select('COL_VUCC_GRIDS');
535-
$this->db->from($this->config->item('table_name'));
536-
$this->db->where_in('station_id', $locations);
537-
$this->db->where('COL_TIME_ON <', $start_date);
538-
$this->db->where('COL_PROP_MODE', $prop_mode);
539-
$this->db->where('COL_VUCC_GRIDS IS NOT NULL');
540-
$this->db->where('COL_VUCC_GRIDS !=', '');
541-
542-
$query = $this->db->get();
543-
foreach ($query->result() as $row) {
544-
if (!empty($row->COL_VUCC_GRIDS)) {
545-
$vucc_grids = explode(',', $row->COL_VUCC_GRIDS);
546-
foreach ($vucc_grids as $vucc_grid) {
547-
$vucc_grid_4char = strtoupper(substr(trim($vucc_grid), 0, 4));
548-
if ($vucc_grid_4char === strtoupper($grid)) {
549-
return false;
550-
}
551-
}
552-
}
553-
}
554-
555-
return true;
548+
// Use pre-built cache for instant lookup
549+
return !isset($this->historical_grids_by_prop[$prop_mode][strtoupper($grid)]);
556550
}
557551

558552
/**
559553
* Check if a grid is new for HF (non-SAT, non-EME)
560554
*/
561555
private function is_grid_new_for_hf($locations, $grid, $start_date) {
562-
// Check if grid exists in COL_GRIDSQUARE (first 4 chars match) for HF
563-
$this->db->select('COUNT(*) as count');
564-
$this->db->from($this->config->item('table_name'));
565-
$this->db->where_in('station_id', $locations);
566-
$this->db->where('COL_TIME_ON <', $start_date);
567-
$this->db->where("(COL_PROP_MODE IS NULL OR COL_PROP_MODE = '' OR COL_PROP_MODE NOT IN ('SAT','EME'))");
568-
$this->db->where('UPPER(SUBSTR(COL_GRIDSQUARE, 1, 4)) =', strtoupper($grid));
569-
570-
$query = $this->db->get();
571-
if ($query->row()->count > 0) {
572-
return false;
573-
}
574-
575-
// Check if grid exists in COL_VUCC_GRIDS for HF
576-
$this->db->select('COL_VUCC_GRIDS');
577-
$this->db->from($this->config->item('table_name'));
578-
$this->db->where_in('station_id', $locations);
579-
$this->db->where('COL_TIME_ON <', $start_date);
580-
$this->db->where("(COL_PROP_MODE IS NULL OR COL_PROP_MODE = '' OR COL_PROP_MODE NOT IN ('SAT','EME'))");
581-
$this->db->where('COL_VUCC_GRIDS IS NOT NULL');
582-
$this->db->where('COL_VUCC_GRIDS !=', '');
583-
584-
$query = $this->db->get();
585-
foreach ($query->result() as $row) {
586-
if (!empty($row->COL_VUCC_GRIDS)) {
587-
$vucc_grids = explode(',', $row->COL_VUCC_GRIDS);
588-
foreach ($vucc_grids as $vucc_grid) {
589-
$vucc_grid_4char = strtoupper(substr(trim($vucc_grid), 0, 4));
590-
if ($vucc_grid_4char === strtoupper($grid)) {
591-
return false;
592-
}
593-
}
594-
}
595-
}
596-
597-
return true;
556+
// Use pre-built cache for instant lookup
557+
return !isset($this->historical_grids_hf[strtoupper($grid)]);
598558
}
599559

600560
/**

0 commit comments

Comments
 (0)