Skip to content

Commit 39e49bd

Browse files
snake14snake14
authored
Merge pull request #260 from haristku/feature/worker_number_to_4096
increase worker number to 4096
2 parents 253df77 + bdd231a commit 39e49bd

5 files changed

Lines changed: 36 additions & 28 deletions

File tree

‎Queue/Manager.php‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -213,14 +213,17 @@ private function getVisitorIdFromRequest(Tracker\Request $request)
213213

214214
protected function getQueueIdForVisitor($visitorId)
215215
{
216-
$visitorId = strtolower(substr($visitorId, 0, 1));
217-
218-
if (isset($this->mappingLettersToNumeric[$visitorId])) {
219-
$id = $this->mappingLettersToNumeric[$visitorId];
220-
} else {
221-
$id = ord($visitorId);
216+
$visitorId = strtolower(substr($visitorId, 0, 3));
217+
if (ctype_xdigit($visitorId) === true) {
218+
$id = hexdec($visitorId);
222219
}
223-
220+
else {
221+
$pos1 = ord($visitorId);
222+
$pos2 = isset($visitorId[1]) ? ord($visitorId[1]) : $pos1;
223+
$pos3 = isset($visitorId[2]) ? ord($visitorId[2]) : $pos2;
224+
$id = $pos1 + $pos2 + $pos3;
225+
}
226+
224227
return $id % $this->numQueuesAvailable;
225228
}
226229

‎SystemSettings.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,9 +194,9 @@ private function createNumberOfQueueWorkerSetting()
194194
$field->title = Piwik::translate('QueuedTracking_NumberOfQueueWorkersFieldTitle');
195195
$field->uiControl = FieldConfig::UI_CONTROL_TEXT;
196196
$field->uiControlAttributes = array('size' => 5);
197-
$field->inlineHelp = Piwik::translate('QueuedTracking_NumberOfQueueWorkersFieldHelp') . '</br>';
197+
$field->inlineHelp = Piwik::translate('QueuedTracking_NumberOfQueueWorkersFieldHelpNew') . '</br>';
198198
$this->assignValueIsIntValidator($field);
199-
$field->validators[] = new NumberRange(1, 16);
199+
$field->validators[] = new NumberRange(1, 4096);
200200
});
201201

202202
$this->addSetting($numQueueWorkers);

‎lang/en.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
"RedisTimeoutFieldHelp": "Redis connection timeout in seconds. “0.0” meaning unlimited. Can be a float eg “2.5” for a connection timeout of 2.5 seconds.",
1212
"NumberOfQueueWorkersFieldTitle": "Number of queue workers",
1313
"NumberOfQueueWorkersFieldHelp": "Number of allowed maximum queue workers. Accepts a number between 1 and 16. Best practice is to set the number of CPUs you want to make available for queue processing. Be aware you need to make sure to start the workers manually. We recommend to not use 9-15 workers, rather use 8 or 16 as the queue might not be distributed evenly into different queues.",
14+
"NumberOfQueueWorkersFieldHelpNew": "Number of allowed maximum queue workers. Accepts a number between 1 and 4096. Best practice is to set the number of CPUs you want to make available for queue processing. Be aware you need to make sure to start the workers manually. We recommend to not use 9-15 workers, rather use 8 or 16 as the queue might not be distributed evenly into different queues.",
1415
"RedisPasswordFieldTitle": "Redis password",
1516
"RedisPasswordFieldHelp": "Password set on the Redis server, if any. Redis can be instructed to require a password before allowing clients to execute commands.",
1617
"RedisDatabaseFieldTitle": "Redis database",

‎tests/Integration/Queue/ManagerTest.php‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -147,12 +147,13 @@ public function test_getQueueIdForVisitor_shouldMoveVisitorIntoQueueBasedOnFirst
147147
$a = 10 % $numQueues; // 10, 11, 12 is a internal mapping see {Manager::$mappingLettersToNumeric}
148148
$b = 11 % $numQueues;
149149
$c = 12 % $numQueues;
150+
$d = 0;
150151

151152
$this->assertQueueIdForVisitorIdEquals($a, 'a');
152153
$this->assertQueueIdForVisitorIdEquals($b, 'b');
153154
$this->assertQueueIdForVisitorIdEquals($c, 'c');
154-
$this->assertQueueIdForVisitorIdEquals($a, 'abcdef');
155-
$this->assertQueueIdForVisitorIdEquals($b, 'bbcdef');
155+
$this->assertQueueIdForVisitorIdEquals($d, 'abcdef');
156+
$this->assertQueueIdForVisitorIdEquals($d, 'bbcdef');
156157
$this->assertQueueIdForVisitorIdEquals($c, 'cbcdef');
157158
}
158159

@@ -267,10 +268,10 @@ public function test_addRequestSetToQueues_getNumberOfRequestSetsInAllQueues_sho
267268
}
268269

269270
$this->assertSame(26, $this->manager->getNumberOfRequestSetsInAllQueues());
270-
$this->assertNumberOfRequestSetsInQueueEquals(5, $queueId = 0);
271+
$this->assertNumberOfRequestSetsInQueueEquals(3, $queueId = 0);
271272
$this->assertNumberOfRequestSetsInQueueEquals(9, $queueId = 1);
272-
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 2);
273-
$this->assertNumberOfRequestSetsInQueueEquals(11, $queueId = 3);
273+
$this->assertNumberOfRequestSetsInQueueEquals(7, $queueId = 2);
274+
$this->assertNumberOfRequestSetsInQueueEquals(7, $queueId = 3);
274275
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 4); // this queue is not available
275276
}
276277

@@ -287,12 +288,12 @@ public function test_addRequestSetToQueues_getNumberOfRequestSetsInAllQueues_sho
287288

288289
$this->assertSame(26, $this->manager->getNumberOfRequestSetsInAllQueues());
289290
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 0);
290-
$this->assertNumberOfRequestSetsInQueueEquals(26, $queueId = 1);
291-
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 2);
291+
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 1);
292+
$this->assertNumberOfRequestSetsInQueueEquals(26, $queueId = 2);
292293
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 3);
293294

294295
// verify all 26 written into queue
295-
$this->assertRequestSetsInQueueEquals($expectedRequestSets, 1);
296+
$this->assertRequestSetsInQueueEquals($expectedRequestSets, 2);
296297
}
297298

298299
public function test_addRequestSetToQueues_shouldMoveAllInSameQueue_IfAllHaveSameUidAndTheyAreInOneRequestSet()
@@ -303,12 +304,12 @@ public function test_addRequestSetToQueues_shouldMoveAllInSameQueue_IfAllHaveSam
303304

304305
$this->assertSame(1, $this->manager->getNumberOfRequestSetsInAllQueues());
305306
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 0);
306-
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 1);
307-
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 2);
307+
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 1);
308+
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 2);
308309
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 3);
309310

310311
// verify all 15 written into queue
311-
$this->assertRequestSetsInQueueEquals(array($requestSet), 1);
312+
$this->assertRequestSetsInQueueEquals(array($requestSet), 2);
312313
}
313314

314315
public function test_addRequestSetToQueues_shouldMoveIntoDifferentQueues_IfThereAreManyDifferentRequestsInOneSet()
@@ -328,21 +329,24 @@ public function test_addRequestSetToQueues_shouldMoveIntoDifferentQueues_IfThere
328329

329330
$this->manager->addRequestSetToQueues($req);
330331

331-
$this->assertSame(3, $this->manager->getNumberOfRequestSetsInAllQueues()); // 3 different uid
332+
$this->assertSame(4, $this->manager->getNumberOfRequestSetsInAllQueues()); // 4 different uid
332333

333-
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 0);
334+
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 0);
334335
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 1);
335336
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 2);
336337
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 3);
337338
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 4);
338339

339340
$req->setRequests(array($requests[1]));
341+
$this->assertRequestSetsInQueueEquals([$req], 0);
342+
343+
$req->setRequests(array($requests[2], $requests[5]));
340344
$this->assertRequestSetsInQueueEquals([$req], 1);
341345

342-
$req->setRequests(array($requests[3]));
346+
$req->setRequests(array($requests[0], $requests[4]));
343347
$this->assertRequestSetsInQueueEquals([$req], 2);
344348

345-
$req->setRequests(array($requests[0], $requests[2], $requests[4], $requests[5]));
349+
$req->setRequests(array($requests[3]));
346350
$this->assertRequestSetsInQueueEquals([$req], 3);
347351
}
348352

@@ -391,11 +395,11 @@ public function test_moveSomeQueuesIfNeeded_ShouldActuallyMoveQueues()
391395

392396
$this->manager->moveSomeQueuesIfNeeded($newNumWorkers, $oldNumWorkers);
393397

394-
$this->assertNumberOfRequestSetsInQueueEquals(6, $queueId = 0);
398+
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 0);
395399
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 1);
396400
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 2);
397401
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 3);
398-
$this->assertNumberOfRequestSetsInQueueEquals(1, $queueId = 4);
402+
$this->assertNumberOfRequestSetsInQueueEquals(6, $queueId = 4);
399403
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 5);
400404
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 6);
401405
$this->assertNumberOfRequestSetsInQueueEquals(0, $queueId = 7);

‎tests/Integration/SettingsTest.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,9 @@ public function test_numQueueWorkers_ShouldFail_IfIsNumericButFloat()
179179
public function test_numQueueWorkers_ShouldFail_IfTooHigh()
180180
{
181181
$this->expectException(\Exception::class);
182-
$this->expectExceptionMessage('The value should be at most 16');
182+
$this->expectExceptionMessage('The value should be at most 4096');
183183

184-
$this->settings->numQueueWorkers->setValue('17');
184+
$this->settings->numQueueWorkers->setValue('4097');
185185
}
186186

187187
public function test_numQueueWorkers_ShouldFail_IfTooLow()

0 commit comments

Comments
 (0)