From d70a20a46627b2edbc53f422bc9dd85dea8eea35 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 8 Feb 2018 18:39:48 +0000 Subject: [PATCH] We now incrementally calculate the queue delivery time --- boot.php | 2 +- database.sql | 17 ++++++++--------- src/Database/DBStructure.php | 9 ++++----- src/Model/Queue.php | 17 ++++++++++++++++- src/Worker/Queue.php | 23 +++-------------------- 5 files changed, 32 insertions(+), 36 deletions(-) diff --git a/boot.php b/boot.php index 48c9b29444..feca503997 100644 --- a/boot.php +++ b/boot.php @@ -39,7 +39,7 @@ define('FRIENDICA_PLATFORM', 'Friendica'); define('FRIENDICA_CODENAME', 'Asparagus'); define('FRIENDICA_VERSION', '3.6-dev'); define('DFRN_PROTOCOL_VERSION', '2.23'); -define('DB_UPDATE_VERSION', 1250); +define('DB_UPDATE_VERSION', 1251); define('NEW_UPDATE_ROUTINE_VERSION', 1170); /** diff --git a/database.sql b/database.sql index 7193b59d17..3dadc23961 100644 --- a/database.sql +++ b/database.sql @@ -853,19 +853,18 @@ CREATE TABLE IF NOT EXISTS `push_subscriber` ( -- CREATE TABLE IF NOT EXISTS `queue` ( `id` int NOT NULL auto_increment COMMENT '', - `cid` int NOT NULL DEFAULT 0 COMMENT '', - `network` varchar(32) NOT NULL DEFAULT '' COMMENT '', - `guid` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', - `last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', + `cid` int NOT NULL DEFAULT 0 COMMENT 'Message receiver', + `network` varchar(32) NOT NULL DEFAULT '' COMMENT 'Receiver\'s network', + `guid` varchar(255) NOT NULL DEFAULT '' COMMENT 'Unique GUID of the message', + `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date, when the message was created', + `last` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last trial', + `next` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Next retrial date', + `retrial` tinyint NOT NULL DEFAULT 0 COMMENT 'Retrial counter', `content` mediumtext COMMENT '', `batch` boolean NOT NULL DEFAULT '0' COMMENT '', PRIMARY KEY(`id`), - INDEX `cid` (`cid`), - INDEX `created` (`created`), INDEX `last` (`last`), - INDEX `network` (`network`), - INDEX `batch` (`batch`) + INDEX `next` (`next`) ) DEFAULT COLLATE utf8mb4_general_ci; -- diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php index 60859e53c3..9ab866de9d 100644 --- a/src/Database/DBStructure.php +++ b/src/Database/DBStructure.php @@ -1553,17 +1553,16 @@ class DBStructure "network" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Receiver's network"], "guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Unique GUID of the message"], "created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date, when the message was created"], - "last" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => ""], + "last" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date of last trial"], + "next" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Next retrial date"], + "retrial" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Retrial counter"], "content" => ["type" => "mediumtext", "comment" => ""], "batch" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], ], "indexes" => [ "PRIMARY" => ["id"], - "cid" => ["cid"], - "created" => ["created"], "last" => ["last"], - "network" => ["network"], - "batch" => ["batch"], + "next" => ["next"], ] ]; $database["register"] = [ diff --git a/src/Model/Queue.php b/src/Model/Queue.php index 2d7873a1e0..df8038c0db 100644 --- a/src/Model/Queue.php +++ b/src/Model/Queue.php @@ -19,7 +19,22 @@ class Queue public static function updateTime($id) { logger('queue: requeue item ' . $id); - dba::update('queue', ['last' => DateTimeFormat::utcNow()], ['id' => $id]); + $queue = dba::selectFirst('queue', ['retrial'], ['id' => $id]); + if (!DBM::is_result($queue)) { + return; + } + + $retrial = $queue['retrial']; + + if ($retrial > 14) { + self::removeItem($id); + } + + // Calculate the delay until the next trial + $delay = (($retrial + 3) ** 4) + (rand(1, 30) * ($retrial + 1)); + $next = DateTimeFormat::utc(date('c', time() + $delay)); + + dba::update('queue', ['last' => DateTimeFormat::utcNow(), 'retrial' => $retrial + 1, 'next' => $next], ['id' => $id]); } /** diff --git a/src/Worker/Queue.php b/src/Worker/Queue.php index d92be19fe3..29be22199f 100644 --- a/src/Worker/Queue.php +++ b/src/Worker/Queue.php @@ -32,30 +32,12 @@ class Queue $no_dead_check = Config::get('system', 'queue_no_dead_check', false); if (!$queue_id) { - logger('queue: start'); + logger('filling queue jobs - start'); // Handling the pubsubhubbub requests Worker::add(['priority' => PRIORITY_HIGH, 'dont_fork' => true], 'PubSubPublish'); - $r = q( - "SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue` - INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id` - WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY" - ); - - if (DBM::is_result($r)) { - foreach ($r as $rr) { - logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']); - logger('Expired queue data: ' . $rr['content'], LOGGER_DATA); - } - q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); - } - - /* - * For the first 12 hours we'll try to deliver every 15 minutes - * After that, we'll only attempt delivery once per hour. - */ - $r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`"); + $r = dba::inArray(dba::p("SELECT `id` FROM `queue` WHERE `next` < UTC_TIMESTAMP()")); Addon::callHooks('queue_predeliver', $r); @@ -65,6 +47,7 @@ class Queue Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], "Queue", (int) $q_item['id']); } } + logger('filling queue jobs - end'); return; }