From d70a20a46627b2edbc53f422bc9dd85dea8eea35 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Thu, 8 Feb 2018 18:39:48 +0000
Subject: [PATCH] We now incrementally calculate the queue delivery time
---
boot.php | 2 +-
database.sql | 17 ++++++++---------
src/Database/DBStructure.php | 9 ++++-----
src/Model/Queue.php | 17 ++++++++++++++++-
src/Worker/Queue.php | 23 +++--------------------
5 files changed, 32 insertions(+), 36 deletions(-)
diff --git a/boot.php b/boot.php
index 48c9b29444..feca503997 100644
--- a/boot.php
+++ b/boot.php
@@ -39,7 +39,7 @@ define('FRIENDICA_PLATFORM', 'Friendica');
define('FRIENDICA_CODENAME', 'Asparagus');
define('FRIENDICA_VERSION', '3.6-dev');
define('DFRN_PROTOCOL_VERSION', '2.23');
-define('DB_UPDATE_VERSION', 1250);
+define('DB_UPDATE_VERSION', 1251);
define('NEW_UPDATE_ROUTINE_VERSION', 1170);
/**
diff --git a/database.sql b/database.sql
index 7193b59d17..3dadc23961 100644
--- a/database.sql
+++ b/database.sql
@@ -853,19 +853,18 @@ CREATE TABLE IF NOT EXISTS `push_subscriber` (
--
CREATE TABLE IF NOT EXISTS `queue` (
`id` int NOT NULL auto_increment COMMENT '',
- `cid` int NOT NULL DEFAULT 0 COMMENT '',
- `network` varchar(32) NOT NULL DEFAULT '' COMMENT '',
- `guid` varchar(255) NOT NULL DEFAULT '' COMMENT '',
- `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
- `last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
+ `cid` int NOT NULL DEFAULT 0 COMMENT 'Message receiver',
+ `network` varchar(32) NOT NULL DEFAULT '' COMMENT 'Receiver\'s network',
+ `guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'Unique GUID of the message',
+ `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date, when the message was created',
+ `last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last trial',
+ `next` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Next retrial date',
+ `retrial` tinyint NOT NULL DEFAULT 0 COMMENT 'Retrial counter',
`content` mediumtext COMMENT '',
`batch` boolean NOT NULL DEFAULT '0' COMMENT '',
PRIMARY KEY(`id`),
- INDEX `cid` (`cid`),
- INDEX `created` (`created`),
INDEX `last` (`last`),
- INDEX `network` (`network`),
- INDEX `batch` (`batch`)
+ INDEX `next` (`next`)
) DEFAULT COLLATE utf8mb4_general_ci;
--
diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php
index 60859e53c3..9ab866de9d 100644
--- a/src/Database/DBStructure.php
+++ b/src/Database/DBStructure.php
@@ -1553,17 +1553,16 @@ class DBStructure
"network" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Receiver's network"],
"guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Unique GUID of the message"],
"created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date, when the message was created"],
- "last" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => ""],
+ "last" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date of last trial"],
+ "next" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Next retrial date"],
+ "retrial" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Retrial counter"],
"content" => ["type" => "mediumtext", "comment" => ""],
"batch" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
],
"indexes" => [
"PRIMARY" => ["id"],
- "cid" => ["cid"],
- "created" => ["created"],
"last" => ["last"],
- "network" => ["network"],
- "batch" => ["batch"],
+ "next" => ["next"],
]
];
$database["register"] = [
diff --git a/src/Model/Queue.php b/src/Model/Queue.php
index 2d7873a1e0..df8038c0db 100644
--- a/src/Model/Queue.php
+++ b/src/Model/Queue.php
@@ -19,7 +19,22 @@ class Queue
public static function updateTime($id)
{
logger('queue: requeue item ' . $id);
- dba::update('queue', ['last' => DateTimeFormat::utcNow()], ['id' => $id]);
+ $queue = dba::selectFirst('queue', ['retrial'], ['id' => $id]);
+ if (!DBM::is_result($queue)) {
+ return;
+ }
+
+ $retrial = $queue['retrial'];
+
+ if ($retrial > 14) {
+ self::removeItem($id);
+ }
+
+ // Calculate the delay until the next trial
+ $delay = (($retrial + 3) ** 4) + (rand(1, 30) * ($retrial + 1));
+ $next = DateTimeFormat::utc(date('c', time() + $delay));
+
+ dba::update('queue', ['last' => DateTimeFormat::utcNow(), 'retrial' => $retrial + 1, 'next' => $next], ['id' => $id]);
}
/**
diff --git a/src/Worker/Queue.php b/src/Worker/Queue.php
index d92be19fe3..29be22199f 100644
--- a/src/Worker/Queue.php
+++ b/src/Worker/Queue.php
@@ -32,30 +32,12 @@ class Queue
$no_dead_check = Config::get('system', 'queue_no_dead_check', false);
if (!$queue_id) {
- logger('queue: start');
+ logger('filling queue jobs - start');
// Handling the pubsubhubbub requests
Worker::add(['priority' => PRIORITY_HIGH, 'dont_fork' => true], 'PubSubPublish');
- $r = q(
- "SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue`
- INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
- WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY"
- );
-
- if (DBM::is_result($r)) {
- foreach ($r as $rr) {
- logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']);
- logger('Expired queue data: ' . $rr['content'], LOGGER_DATA);
- }
- q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
- }
-
- /*
- * For the first 12 hours we'll try to deliver every 15 minutes
- * After that, we'll only attempt delivery once per hour.
- */
- $r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`");
+ $r = dba::inArray(dba::p("SELECT `id` FROM `queue` WHERE `next` < UTC_TIMESTAMP()"));
Addon::callHooks('queue_predeliver', $r);
@@ -65,6 +47,7 @@ class Queue
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], "Queue", (int) $q_item['id']);
}
}
+ logger('filling queue jobs - end');
return;
}