From 25fe67f698ea6abb94710375ba71d3bed27fa11a Mon Sep 17 00:00:00 2001 From: Michael Schink Date: Thu, 10 Jul 2025 12:17:24 +0200 Subject: [PATCH 1/4] ELEARN-1070 * Split db query for calculating triggers to courses in multiple chunks to prevent error, if more than 65535 courses are calculated * Not the best solution, but should work in most cases * Testing only --- classes/local/intersectedRecordset.php | 131 +++++++++++++++++++++++++ classes/processor.php | 54 +++++++--- 2 files changed, 169 insertions(+), 16 deletions(-) create mode 100644 classes/local/intersectedRecordset.php diff --git a/classes/local/intersectedRecordset.php b/classes/local/intersectedRecordset.php new file mode 100644 index 00000000..459ec0d9 --- /dev/null +++ b/classes/local/intersectedRecordset.php @@ -0,0 +1,131 @@ +. + +/** + * Helper class which intersects multiple moodle record sets. + * + * @package tool_lifecycle + * @copyright 2025 Michael Schink JKU + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +namespace tool_lifecycle\local; + +defined('MOODLE_INTERNAL') || die(); + +class intersectedRecordset implements \Iterator, \Countable { + private $records = []; + private $position = 0; + + /** + * Constructor: Inits class & intersects passed recordsets. + * + * @param moodle_recordset|array|null $recordsets + * @param string $key + */ + public function __construct($recordsets = null, string $key = 'id') { + if($recordsets !== null) { + if(is_array($recordsets)) { + foreach($recordsets as $recordset) { $this->add($recordset, $key); } + } else { $this->add($recordsets, $key); } + } + } + + /** + * Adds recordset & saves intersection of all recordsets. + * + * @param moodle_recordset $recordset + * @param string $key + */ + public function add($recordset, string $key = 'id'): void { + $newRecords = []; + foreach($recordset as $record) { $newRecords[] = $record; } + //$recordset->close(); + + $newRecordsByKey = []; + foreach($newRecords as $record) { + if(isset($record->$key)) { $newRecordsByKey[$record->$key] = $record; } + } + + if(empty($this->records)) { + $this->records = array_values($newRecordsByKey); + + return; + } + + $existingRecordsByKey = []; + foreach($this->records as $record) { + if(isset($record->$key)) { $existingRecordsByKey[$record->$key] = $record; } + } + + $intersectionKeys = array_intersect_key($existingRecordsByKey, $newRecordsByKey); + + $this->records = []; + foreach($intersectionKeys as $keyValue => $record) { + $this->records[] = $existingRecordsByKey[$keyValue]; + } + } + + /** + * Returns current recordset. + * + * @return mixed + */ + public function current(): mixed { + return $this->records[$this->position]; + } + + /** + * Returns current key (index). + * + * @return int + */ + public function key(): int { + return $this->position; + } + + /** + * Moves internal pointer to next recordset. + */ + public function next(): void { + $this->position++; + } + + /** + * Returns internal pointer to start. + */ + public function rewind(): void { + $this->position = 0; + } + + /** + * Checks if current pointer points to a valid recordset. + * + * @return bool + */ + public function valid(): bool { + return isset($this->records[$this->position]); + } + + /** + * Returns the amount of all recordsets. + * + * @return int + */ + public function count(): int { + return count($this->records); + } +} \ No newline at end of file diff --git a/classes/processor.php b/classes/processor.php index 37be60a5..fe3ea6c5 100644 --- a/classes/processor.php +++ b/classes/processor.php @@ -195,26 +195,31 @@ public function process_course_interactive($processid) { public function get_course_recordset($triggers, $exclude, $forcounting = false) { global $DB; - $where = 'true'; + // Mike + $where = []; $whereparams = []; + //$chunks = array_chunk($whereparams, 65535, true); + $recordsets = []; foreach ($triggers as $trigger) { $lib = lib_manager::get_automatic_trigger_lib($trigger->subpluginname); [$sql, $params] = $lib->get_course_recordset_where($trigger->id); if (!empty($sql)) { - $where .= ' AND ' . $sql; - $whereparams = array_merge($whereparams, $params); + $where[] = 'true AND ' . $sql; + $whereparams[] = $params; } } if (!empty($exclude)) { [$insql, $inparams] = $DB->get_in_or_equal($exclude, SQL_PARAMS_NAMED); - $where .= " AND NOT {course}.id {$insql}"; - $whereparams = array_merge($whereparams, $inparams); + $where[] = "true AND NOT {course}.id {$insql}"; + $whereparams[] = $inparams; } if ($forcounting) { - // Get course hasotherprocess and delay with the sql. - $sql = "SELECT {course}.id, + foreach ($where as $key => $where_tmp) { + $whereparams_tmp = $whereparams[$key]; + // Get course hasotherprocess and delay with the sql. + $sql = "SELECT {course}.id, COALESCE(p.courseid, pe.courseid, 0) as hasprocess, CASE WHEN COALESCE(p.workflowid, 0) > COALESCE(pe.workflowid, 0) THEN p.workflowid @@ -232,17 +237,32 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) LEFT JOIN {tool_lifecycle_proc_error} pe ON {course}.id = pe.courseid LEFT JOIN {tool_lifecycle_delayed} d ON {course}.id = d.courseid LEFT JOIN {tool_lifecycle_delayed_workf} dw ON {course}.id = dw.courseid - WHERE " . $where; + WHERE " . $where_tmp; + $recordsets[] = $DB->get_recordset_sql($sql, $whereparams_tmp); + } + + //use tool_lifecycle\local\intersectedRecordset; + $recordsets = new \tool_lifecycle\local\intersectedRecordset($recordsets); + mtrace('Intersected record sets (for counting): '.count($recordsets)); } else { - // Get only courses which are not part of an existing process. - $sql = 'SELECT {course}.id from {course} '. - 'LEFT JOIN {tool_lifecycle_process} '. - 'ON {course}.id = {tool_lifecycle_process}.courseid '. - 'LEFT JOIN {tool_lifecycle_proc_error} pe ON {course}.id = pe.courseid ' . - 'WHERE {tool_lifecycle_process}.courseid is null AND ' . - 'pe.courseid IS NULL AND '. $where; + foreach ($where as $key => $where_tmp) { + $whereparams_tmp = $whereparams[$key]; + // Get only courses which are not part of an existing process. + $sql = 'SELECT {course}.id from {course} '. + 'LEFT JOIN {tool_lifecycle_process} '. + 'ON {course}.id = {tool_lifecycle_process}.courseid '. + 'LEFT JOIN {tool_lifecycle_proc_error} pe ON {course}.id = pe.courseid ' . + 'WHERE {tool_lifecycle_process}.courseid is null AND ' . + 'pe.courseid IS NULL AND '. $where_tmp; + $recordsets[] = $DB->get_recordset_sql($sql, $whereparams_tmp); + } + + //use tool_lifecycle\local\intersectedRecordset; + $recordsets = new \tool_lifecycle\local\intersectedRecordset($recordsets); + mtrace('Intersected record sets: '.count($recordsets)); } - return $DB->get_recordset_sql($sql, $whereparams); + + return $recordsets; } /** @@ -435,6 +455,8 @@ public function get_count_of_courses_to_trigger_for_workflow($workflow) { $all->delayedcourses = $delayedcourses; // Delayed courses for workflow and globally. Excluded per default. $all->used = $usedcourses; $all->nextrun = $nextrun; + // Mike + mtrace('Courses triggered by workflow: '.count($all->triggered)); $amounts['all'] = $all; return $amounts; } From 5039a78c214d877b601ccf8a613b759bceb69f3c Mon Sep 17 00:00:00 2001 From: Michael Schink Date: Fri, 11 Jul 2025 17:03:45 +0200 Subject: [PATCH 2/4] ELEARN-1070 * Bugfix: class intersectedRecordset --- classes/local/intersectedRecordset.php | 31 +++++++++++++++----------- classes/processor.php | 4 ++-- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/classes/local/intersectedRecordset.php b/classes/local/intersectedRecordset.php index 459ec0d9..a8e4d860 100644 --- a/classes/local/intersectedRecordset.php +++ b/classes/local/intersectedRecordset.php @@ -29,6 +29,7 @@ class intersectedRecordset implements \Iterator, \Countable { private $records = []; private $position = 0; + private $wasFilled = false; /** * Constructor: Inits class & intersects passed recordsets. @@ -51,32 +52,36 @@ public function __construct($recordsets = null, string $key = 'id') { * @param string $key */ public function add($recordset, string $key = 'id'): void { + // Add new records to array with key $newRecords = []; - foreach($recordset as $record) { $newRecords[] = $record; } - //$recordset->close(); - - $newRecordsByKey = []; - foreach($newRecords as $record) { - if(isset($record->$key)) { $newRecordsByKey[$record->$key] = $record; } + foreach($recordset as $record) { + if(isset($record->$key)) { $newRecords[$record->$key] = $record; } } + //$recordset->close(); - if(empty($this->records)) { - $this->records = array_values($newRecordsByKey); + // Store new records without key, if no records were stored & return + if(empty($this->records) && !$this->wasFilled) { + $this->records = array_values($newRecords); + $this->wasFilled = true; return; } - $existingRecordsByKey = []; + // Add existing records to array with key + $existingRecords = []; foreach($this->records as $record) { - if(isset($record->$key)) { $existingRecordsByKey[$record->$key] = $record; } + if(isset($record->$key)) { $existingRecords[$record->$key] = $record; } } - $intersectionKeys = array_intersect_key($existingRecordsByKey, $newRecordsByKey); - + // Intersect existing & new records by keys + $intersectionKeys = array_intersect_key($existingRecords, $newRecords); + // Clear existing records $this->records = []; + // Store intersected records by keys foreach($intersectionKeys as $keyValue => $record) { - $this->records[] = $existingRecordsByKey[$keyValue]; + $this->records[] = $existingRecords[$keyValue]; } + //mtrace('Add - Intersected record sets: '.count($this->records)); } /** diff --git a/classes/processor.php b/classes/processor.php index fe3ea6c5..728f9856 100644 --- a/classes/processor.php +++ b/classes/processor.php @@ -243,7 +243,7 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) //use tool_lifecycle\local\intersectedRecordset; $recordsets = new \tool_lifecycle\local\intersectedRecordset($recordsets); - mtrace('Intersected record sets (for counting): '.count($recordsets)); + //mtrace('Intersected record sets (for counting): '.count($recordsets)); } else { foreach ($where as $key => $where_tmp) { $whereparams_tmp = $whereparams[$key]; @@ -259,7 +259,7 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) //use tool_lifecycle\local\intersectedRecordset; $recordsets = new \tool_lifecycle\local\intersectedRecordset($recordsets); - mtrace('Intersected record sets: '.count($recordsets)); + //mtrace('Intersected record sets: '.count($recordsets)); } return $recordsets; From a48a4087c95b24530e17fe04ba4f5fb0b6ddc567 Mon Sep 17 00:00:00 2001 From: Michael Schink Date: Tue, 9 Sep 2025 12:01:36 +0200 Subject: [PATCH 3/4] ELEARN-1070 * Extended last partial solution * Split db query for each trigger (= calculating trigger to courses) into "chunked sub queries", based on amount of params * Max params for each trigger query: 65535 * Should be final solution * Testing only --- classes/local/intersectedRecordset.php | 23 +++++- classes/processor.php | 97 +++++++++++++++++++++++--- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/classes/local/intersectedRecordset.php b/classes/local/intersectedRecordset.php index a8e4d860..b59cbc7e 100644 --- a/classes/local/intersectedRecordset.php +++ b/classes/local/intersectedRecordset.php @@ -40,7 +40,27 @@ class intersectedRecordset implements \Iterator, \Countable { public function __construct($recordsets = null, string $key = 'id') { if($recordsets !== null) { if(is_array($recordsets)) { - foreach($recordsets as $recordset) { $this->add($recordset, $key); } + // For multiple recordsets + foreach($recordsets as $recordset) { + // If recordset is a chunked recordset + if(is_array($recordset)) { + mtrace('Chunked recordset'); + // Create new array for chunked recordset + $chunkedRecords = []; + // For each chunked recordset + foreach($recordset as $chunk_recordset) { + // For each record in chunked recordset + foreach($chunk_recordset as $record) { + if(isset($record->$key)) { $chunkedRecords[$record->$key] = $record; } + } + } + // Add all records of chunked recordsets + $this->add($chunkedRecords, $key); + } else { + mtrace('Normal recordset'); + $this->add($recordset, $key); + } + } } else { $this->add($recordsets, $key); } } } @@ -57,6 +77,7 @@ public function add($recordset, string $key = 'id'): void { foreach($recordset as $record) { if(isset($record->$key)) { $newRecords[$record->$key] = $record; } } + mtrace(' Found '.count($newRecords).' records in recordset'); //$recordset->close(); // Store new records without key, if no records were stored & return diff --git a/classes/processor.php b/classes/processor.php index 728f9856..6327adb9 100644 --- a/classes/processor.php +++ b/classes/processor.php @@ -198,7 +198,6 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) // Mike $where = []; $whereparams = []; - //$chunks = array_chunk($whereparams, 65535, true); $recordsets = []; foreach ($triggers as $trigger) { $lib = lib_manager::get_automatic_trigger_lib($trigger->subpluginname); @@ -215,6 +214,55 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) $whereparams[] = $inparams; } + $maxparams = 65535; + //$maxparams = 4; + mtrace(''); + mtrace('Start - MAX params: '.$maxparams.', where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params '.count($whereparams)/*.' '.print_r($whereparams, true)*/); + foreach($whereparams as $key => $whereparam) { + if(count($whereparam) > $maxparams) { + mtrace('More than '.$maxparams.' params with key '.$key.': '.count($whereparam)); + // Get where part of params array + $wherepart = $where[$key]; + // Get first & last param + $first = ':'.array_key_first($whereparam); + $last = ':'.array_key_last($whereparam); + mtrace(' 1. Get first param '.$first.' & last param '.$last); + // Get where part before first param & after last param (to re-create where part) + $position = strpos($wherepart, $first); + $before = substr($wherepart, 0, $position); + $position = strpos($wherepart, $last); + $after = substr($wherepart, $position + strlen($last)); + mtrace(' 2. Re-create where part: '.$before.' '.$after); + // Remove original where part & params + //unset($where[$key]); + //unset($whereparams[$key]); + $where[$key] = []; + $whereparams[$key] = []; + mtrace(' 3. Remove where part & params with key: '.$key); + // Chunk params + $whereparam_chunks = array_chunk($whereparam, $maxparams, true); + mtrace(' 4. Chunk params: '.count($whereparam_chunks)/*.print_r($whereparam_chunks, true))*/.' ('.count($whereparam).'/'.$maxparams.')'); + // For each chunk of params + $counter = 0; + foreach($whereparam_chunks as $whereparam_chunk) { + $counter++; + // Create param string of chunk params + $whereparam_chunk_string = implode(',', array_map(function($value) { return ':' . $value; }, array_keys($whereparam_chunk))); + // Re-create where part for chunk + $where_chunk = $before.$whereparam_chunk_string.$after; + mtrace(' 5.'.$counter.' Add chunk query: '.$where_chunk.' & params: '.count($whereparam_chunk)/*.print_r($whereparam_chunk, true)*/); + // Add where part & params of chunk + if(count($where) && count($whereparams)) { + $where[$key][] = $where_chunk; + $whereparams[$key][] = $whereparam_chunk; + } else { mtrace('ERROR: Amount of where parts & params are not the same!'); } + } + } + } + mtrace('End - MAX params: '.$maxparams.', where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params '.count($whereparams)/*.' '.print_r($whereparams, true)*/); + mtrace(''); + //die(); + if ($forcounting) { foreach ($where as $key => $where_tmp) { $whereparams_tmp = $whereparams[$key]; @@ -237,8 +285,21 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) LEFT JOIN {tool_lifecycle_proc_error} pe ON {course}.id = pe.courseid LEFT JOIN {tool_lifecycle_delayed} d ON {course}.id = d.courseid LEFT JOIN {tool_lifecycle_delayed_workf} dw ON {course}.id = dw.courseid - WHERE " . $where_tmp; - $recordsets[] = $DB->get_recordset_sql($sql, $whereparams_tmp); + WHERE "; + + if(is_array($where_tmp)) { + //mtrace('Chunked recordset: '.count($where_tmp)/*.' '.print_r($where_tmp, true)*/.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/); + $tmp = []; + foreach($where_tmp as $chunk_key => $chunk_where_tmp) { + $sql_tmp = $sql.$chunk_where_tmp; + $tmp[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp[$chunk_key]); + } + $recordsets[] = $tmp; + } else { + //mtrace('Nomrmal recordset: '.$where_tmp.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/); + $sql_tmp = $sql.$where_tmp; + $recordsets[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp); + } } //use tool_lifecycle\local\intersectedRecordset; @@ -249,12 +310,25 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) $whereparams_tmp = $whereparams[$key]; // Get only courses which are not part of an existing process. $sql = 'SELECT {course}.id from {course} '. - 'LEFT JOIN {tool_lifecycle_process} '. - 'ON {course}.id = {tool_lifecycle_process}.courseid '. - 'LEFT JOIN {tool_lifecycle_proc_error} pe ON {course}.id = pe.courseid ' . - 'WHERE {tool_lifecycle_process}.courseid is null AND ' . - 'pe.courseid IS NULL AND '. $where_tmp; - $recordsets[] = $DB->get_recordset_sql($sql, $whereparams_tmp); + 'LEFT JOIN {tool_lifecycle_process} '. + 'ON {course}.id = {tool_lifecycle_process}.courseid '. + 'LEFT JOIN {tool_lifecycle_proc_error} pe ON {course}.id = pe.courseid ' . + 'WHERE {tool_lifecycle_process}.courseid is null AND ' . + 'pe.courseid IS NULL AND '; + + if(is_array($where_tmp)) { + //mtrace('Chunked recordset: '.count($where_tmp)/*.' '.print_r($where_tmp, true)*/.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/); + $tmp = []; + foreach($where_tmp as $chunk_key => $chunk_where_tmp) { + $sql_tmp = $sql.$chunk_where_tmp; + $tmp[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp[$chunk_key]); + } + $recordsets[] = $tmp; + } else { + //mtrace('Nomrmal recordset: '.$where_tmp.', params: '.count($whereparams_tmp)/*.' '.print_r($whereparams_tmp, true)*/); + $sql_tmp = $sql.$where_tmp; + $recordsets[] = $DB->get_recordset_sql($sql_tmp, $whereparams_tmp); + } } //use tool_lifecycle\local\intersectedRecordset; @@ -262,6 +336,11 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) //mtrace('Intersected record sets: '.count($recordsets)); } + mtrace(''); + mtrace('FINAL recordsets: '.$recordsets->count()/*.' '.print_r($recordsets, true)*/); + mtrace(''); + //die(); + return $recordsets; } From eb61fb518ab0b9195049e13024b9687ed1191fcc Mon Sep 17 00:00:00 2001 From: Michael Schink Date: Wed, 10 Sep 2025 16:37:08 +0200 Subject: [PATCH 4/4] ELEARN-1070 * Shorten debug output * Testing only --- classes/processor.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/classes/processor.php b/classes/processor.php index 6327adb9..b549a01e 100644 --- a/classes/processor.php +++ b/classes/processor.php @@ -215,9 +215,9 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) } $maxparams = 65535; - //$maxparams = 4; + //$maxparams = 1000; mtrace(''); - mtrace('Start - MAX params: '.$maxparams.', where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params '.count($whereparams)/*.' '.print_r($whereparams, true)*/); + mtrace('Start - MAX params: '.$maxparams.', trigger where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params: '.count($whereparams)/*.' '.print_r($whereparams, true)*/); foreach($whereparams as $key => $whereparam) { if(count($whereparam) > $maxparams) { mtrace('More than '.$maxparams.' params with key '.$key.': '.count($whereparam)); @@ -250,7 +250,8 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) $whereparam_chunk_string = implode(',', array_map(function($value) { return ':' . $value; }, array_keys($whereparam_chunk))); // Re-create where part for chunk $where_chunk = $before.$whereparam_chunk_string.$after; - mtrace(' 5.'.$counter.' Add chunk query: '.$where_chunk.' & params: '.count($whereparam_chunk)/*.print_r($whereparam_chunk, true)*/); + if(count($whereparam_chunks) > 10 && $counter == 10 ) { mtrace(' ...'); } + if($counter < 5 || $counter >= (count($whereparam_chunks) - 5)) { mtrace(' 5.'.$counter.' Add chunk query: '.(strlen($where_chunk) > 150 ? substr($where_chunk, 0, 150) . '...' : $where_chunk).' & params: '.count($whereparam_chunk)/*.print_r($whereparam_chunk, true)*/); } // Add where part & params of chunk if(count($where) && count($whereparams)) { $where[$key][] = $where_chunk; @@ -259,7 +260,7 @@ public function get_course_recordset($triggers, $exclude, $forcounting = false) } } } - mtrace('End - MAX params: '.$maxparams.', where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params '.count($whereparams)/*.' '.print_r($whereparams, true)*/); + mtrace('End - MAX params: '.$maxparams.', trigger where parts: '.count($where)/*.' '.print_r($where, true)*/.' & params '.count($whereparams)/*.' '.print_r($whereparams, true)*/); mtrace(''); //die();