From 2aa302c570ca75969ad0ed4697d4f38cf98741ab Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Tue, 1 Dec 2020 22:11:29 +0000
Subject: [PATCH 01/16] New class for handling delayed posts
---
src/Model/Post/Delayed.php | 112 ++++++++++++++++++++++++++++++++++
src/Protocol/Feed.php | 30 ++++++---
src/Worker/DelayedPublish.php | 17 +-----
static/dbstructure.config.php | 15 ++++-
4 files changed, 149 insertions(+), 25 deletions(-)
create mode 100644 src/Model/Post/Delayed.php
diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php
new file mode 100644
index 0000000000..ccb678d466
--- /dev/null
+++ b/src/Model/Post/Delayed.php
@@ -0,0 +1,112 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use Friendica\Core\Logger;
+use Friendica\Database\DBA;
+use Friendica\Core\Worker;
+use Friendica\Database\Database;
+use Friendica\Model\Item;
+use Friendica\Model\Tag;
+
+class Delayed
+{
+ /**
+ * Insert a new delayed post
+ *
+ * @param string $uri
+ * @param integer $uid
+ * @param string $delayed
+ * @param array $item
+ * @param integer $notify
+ * @param array $taglist
+ * @param array $attachments
+ * @return bool insert success
+ */
+ public static function add(string $uri, int $uid, string $delayed, array $item, int $notify = 0, array $taglist = [], array $attachments = [])
+ {
+ if (self::exists($uri)) {
+ return false;
+ }
+
+ Logger::notice('Adding post for delayed publishing', ['uid' => $uid, 'delayed' => $delayed, 'uri' => $uri]);
+
+ Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $delayed], 'DelayedPublish', $item, $notify, $taglist, $attachments);
+ return DBA::insert('delayed-post', ['uri' => $uri, 'uid' => $uid, 'delayed' => $delayed], Database::INSERT_IGNORE);
+ }
+
+ /**
+ * Delete a delayed post
+ *
+ * @param string $uri
+ *
+ * @return bool delete success
+ */
+ private static function delete(string $uri)
+ {
+ return DBA::delete('delayed-post', ['uri' => $uri]);
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param string $uri
+ *
+ * @return bool "true" if an entry with that URI exists
+ */
+ public static function exists(string $uri)
+ {
+ return DBA::exists('delayed-post', ['uri' => $uri]);
+ }
+
+ /**
+ * Publish a delayed post
+ *
+ * @param array $item
+ * @param integer $notify
+ * @param array $taglist
+ * @param array $attachments
+ * @return bool
+ */
+ public static function publish(array $item, int $notify = 0, array $taglist = [], array $attachments = [])
+ {
+ $id = Item::insert($item, $notify);
+
+ Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
+
+ if (!empty($id) && (!empty($taglist) || !empty($attachments))) {
+ $feeditem = Item::selectFirst(['uri-id', 'uri'], ['id' => $id]);
+ self::delete($feeditem['uri']);
+
+ foreach ($taglist as $tag) {
+ Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag);
+ }
+
+ foreach ($attachments as $attachment) {
+ $attachment['uri-id'] = $feeditem['uri-id'];
+ Media::insert($attachment);
+ }
+ }
+
+ return $id;
+ }
+}
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 6390fab658..f874b4b026 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -606,7 +606,7 @@ class Feed
// Additionally we have to avoid conflicts with identical URI between imported feeds and these items.
if ($notify) {
$item['guid'] = Item::guidFromUri($orig_plink, DI::baseUrl()->getHostname());
- unset($item['uri']);
+ $item['uri'] = Item::newURI($item['uid'], $item['guid']);
unset($item['thr-parent']);
unset($item['parent-uri']);
@@ -614,16 +614,21 @@ class Feed
$notify = PRIORITY_MEDIUM;
}
- $postings[] = ['item' => $item, 'notify' => $notify,
- 'taglist' => $taglist, 'attachments' => $attachments];
+ if (!Post\Delayed::exists($item["uri"])) {
+ $postings[] = ['item' => $item, 'notify' => $notify,
+ 'taglist' => $taglist, 'attachments' => $attachments];
+ } else {
+ Logger::info('Post already exists in the delayed posts queue', ['uri' => $item["uri"]]);
+ }
}
if (!empty($postings)) {
+ $min_posting = DI::config()->get('system', 'minimum_posting_interval', 0);
$total = count($postings);
if ($total > 1) {
// Posts shouldn't be delayed more than a day
$interval = min(1440, self::getPollInterval($contact));
- $delay = round(($interval * 60) / $total);
+ $delay = max(round(($interval * 60) / $total), 60 * $min_posting);
Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]);
} else {
$delay = 0;
@@ -633,15 +638,22 @@ class Feed
foreach ($postings as $posting) {
if ($delay > 0) {
- $publish_at = DateTimeFormat::utc('now + ' . $post_delay . ' second');
- Logger::notice('Got publishing date', ['delay' => $delay, 'publish_at' => $publish_at, 'cid' => $contact['id'], 'url' => $contact['url']]);
+ $publish_time = time() + $post_delay;
+ Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]);
$post_delay += $delay;
} else {
- $publish_at = DBA::NULL_DATETIME;
+ $publish_time = time();
}
- Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $publish_at],
- 'DelayedPublish', $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']);
+ $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true);
+ $next_publish = max($last_publish + (60 * $min_posting), time());
+ if ($publish_time < $next_publish) {
+ $publish_time = $next_publish;
+ }
+ $publish_at = date(DateTimeFormat::MYSQL, $publish_time);
+
+ Post\Delayed::add($item['uri'], $item['uid'], $publish_at, $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']);
+ DI::pConfig()->set($item['uid'], 'system', 'last_publish', $next_publish);
}
}
diff --git a/src/Worker/DelayedPublish.php b/src/Worker/DelayedPublish.php
index 3dad540481..beffb22e30 100644
--- a/src/Worker/DelayedPublish.php
+++ b/src/Worker/DelayedPublish.php
@@ -39,20 +39,7 @@ class DelayedPublish
*/
public static function execute(array $item, int $notify = 0, array $taglist = [], array $attachments = [])
{
- $id = Item::insert($item, $notify);
-
- Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
-
- if (!empty($id) && (!empty($taglist) || !empty($attachments))) {
- $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]);
- foreach ($taglist as $tag) {
- Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag);
- }
- foreach ($attachments as $attachment) {
- $attachment['uri-id'] = $feeditem['uri-id'];
- Post\Media::insert($attachment);
- }
- }
-
+ $id = Post\Delayed::publish($item, $notify, $taglist, $attachments);
+ Logger::notice('Post published', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
}
}
diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php
index 9a125cc15b..65b4fecf08 100644
--- a/static/dbstructure.config.php
+++ b/static/dbstructure.config.php
@@ -55,7 +55,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
- define('DB_UPDATE_VERSION', 1381);
+ define('DB_UPDATE_VERSION', 1382);
}
return [
@@ -527,6 +527,19 @@ return [
"received" => ["received"],
]
],
+ "delayed-post" => [
+ "comment" => "Posts that are about to be posted at a later time",
+ "fields" => [
+ "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"],
+ "uri" => ["type" => "varchar(255)", "comment" => "URI of the post that will be posted later"],
+ "uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Owner User id"],
+ "delayed" => ["type" => "datetime", "comment" => "delay time"],
+ ],
+ "indexes" => [
+ "PRIMARY" => ["id"],
+ "url" => ["UNIQUE", "url"],
+ ]
+ ],
"diaspora-interaction" => [
"comment" => "Signed Diaspora Interaction",
"fields" => [
From 61ecdf789c648cc5789ade2da3172bb7cc7a734d Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Tue, 1 Dec 2020 22:21:44 +0000
Subject: [PATCH 02/16] Fix table structure
---
database.sql | 32 +++++++++++++++++++++++---------
static/dbstructure.config.php | 2 +-
2 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/database.sql b/database.sql
index 8564a8ba02..520c5e9b05 100644
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2020.12-dev (Red Hot Poker)
--- DB_UPDATE_VERSION 1381
+-- DB_UPDATE_VERSION 1382
-- ------------------------------------------
@@ -461,6 +461,19 @@ CREATE TABLE IF NOT EXISTS `conversation` (
INDEX `received` (`received`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages';
+--
+-- TABLE delayed-post
+--
+CREATE TABLE IF NOT EXISTS `delayed-post` (
+ `id` int unsigned NOT NULL auto_increment,
+ `uri` varchar(255) COMMENT 'URI of the post that will be posted later',
+ `uid` mediumint unsigned COMMENT 'Owner User id',
+ `delayed` datetime COMMENT 'delay time',
+ PRIMARY KEY(`id`),
+ UNIQUE INDEX `uri` (`uri`(190)),
+ FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be posted at a later time';
+
--
-- TABLE diaspora-interaction
--
@@ -1472,7 +1485,7 @@ CREATE TABLE IF NOT EXISTS `workerqueue` (
--
-- VIEW category-view
--
-DROP VIEW IF EXISTS `category-view`;
+DROP VIEW `category-view`;
CREATE VIEW `category-view` AS SELECT
`post-category`.`uri-id` AS `uri-id`,
`post-category`.`uid` AS `uid`,
@@ -1489,7 +1502,7 @@ CREATE VIEW `category-view` AS SELECT
--
-- VIEW tag-view
--
-DROP VIEW IF EXISTS `tag-view`;
+DROP VIEW `tag-view`;
CREATE VIEW `tag-view` AS SELECT
`post-tag`.`uri-id` AS `uri-id`,
`item-uri`.`uri` AS `uri`,
@@ -1507,7 +1520,7 @@ CREATE VIEW `tag-view` AS SELECT
--
-- VIEW network-item-view
--
-DROP VIEW IF EXISTS `network-item-view`;
+DROP VIEW `network-item-view`;
CREATE VIEW `network-item-view` AS SELECT
`item`.`parent-uri-id` AS `uri-id`,
`item`.`parent-uri` AS `uri`,
@@ -1539,7 +1552,7 @@ CREATE VIEW `network-item-view` AS SELECT
--
-- VIEW network-thread-view
--
-DROP VIEW IF EXISTS `network-thread-view`;
+DROP VIEW `network-thread-view`;
CREATE VIEW `network-thread-view` AS SELECT
`item`.`uri-id` AS `uri-id`,
`item`.`uri` AS `uri`,
@@ -1570,7 +1583,7 @@ CREATE VIEW `network-thread-view` AS SELECT
--
-- VIEW owner-view
--
-DROP VIEW IF EXISTS `owner-view`;
+DROP VIEW `owner-view`;
CREATE VIEW `owner-view` AS SELECT
`contact`.`id` AS `id`,
`contact`.`uid` AS `uid`,
@@ -1705,7 +1718,7 @@ CREATE VIEW `owner-view` AS SELECT
--
-- VIEW pending-view
--
-DROP VIEW IF EXISTS `pending-view`;
+DROP VIEW `pending-view`;
CREATE VIEW `pending-view` AS SELECT
`register`.`id` AS `id`,
`register`.`hash` AS `hash`,
@@ -1727,7 +1740,7 @@ CREATE VIEW `pending-view` AS SELECT
--
-- VIEW tag-search-view
--
-DROP VIEW IF EXISTS `tag-search-view`;
+DROP VIEW `tag-search-view`;
CREATE VIEW `tag-search-view` AS SELECT
`post-tag`.`uri-id` AS `uri-id`,
`item`.`id` AS `iid`,
@@ -1748,7 +1761,7 @@ CREATE VIEW `tag-search-view` AS SELECT
--
-- VIEW workerqueue-view
--
-DROP VIEW IF EXISTS `workerqueue-view`;
+DROP VIEW `workerqueue-view`;
CREATE VIEW `workerqueue-view` AS SELECT
`process`.`pid` AS `pid`,
`workerqueue`.`priority` AS `priority`
@@ -1756,3 +1769,4 @@ CREATE VIEW `workerqueue-view` AS SELECT
INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid`
WHERE NOT `workerqueue`.`done`;
+
diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php
index 65b4fecf08..9af9c46257 100644
--- a/static/dbstructure.config.php
+++ b/static/dbstructure.config.php
@@ -537,7 +537,7 @@ return [
],
"indexes" => [
"PRIMARY" => ["id"],
- "url" => ["UNIQUE", "url"],
+ "uri" => ["UNIQUE", "uri(190)"],
]
],
"diaspora-interaction" => [
From b3f71d174c2f64d4bb527a6343cc4ba88f410a42 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Tue, 1 Dec 2020 23:54:49 +0100
Subject: [PATCH 03/16] Added logging
---
src/Model/Post/Delayed.php | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php
index ccb678d466..14f5d7198e 100644
--- a/src/Model/Post/Delayed.php
+++ b/src/Model/Post/Delayed.php
@@ -67,7 +67,7 @@ class Delayed
}
/**
- * Undocumented function
+ * Check if an entry exists
*
* @param string $uri
*
@@ -95,7 +95,8 @@ class Delayed
if (!empty($id) && (!empty($taglist) || !empty($attachments))) {
$feeditem = Item::selectFirst(['uri-id', 'uri'], ['id' => $id]);
- self::delete($feeditem['uri']);
+ $result = self::delete($feeditem['uri']);
+ Logger::notice('Delayed post entry deleted', ['result' => $result, 'item' => $feeditem]);
foreach ($taglist as $tag) {
Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag);
From a283b1183b4ec79ac2e35c0757391c4605bf30d4 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Tue, 1 Dec 2020 23:04:04 +0000
Subject: [PATCH 04/16] Move deletion
---
src/Model/Post/Delayed.php | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php
index 14f5d7198e..357a3e8aa9 100644
--- a/src/Model/Post/Delayed.php
+++ b/src/Model/Post/Delayed.php
@@ -93,10 +93,14 @@ class Delayed
Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
+ // It should always contain an URI since this is needed to create a delayed post entry
+ if (!empty($item['uri'])) {
+ $result = self::delete($item['uri']);
+ Logger::notice('Delayed post entry deleted', ['result' => $result, 'uri' => $item['uri']]);
+ }
+
if (!empty($id) && (!empty($taglist) || !empty($attachments))) {
- $feeditem = Item::selectFirst(['uri-id', 'uri'], ['id' => $id]);
- $result = self::delete($feeditem['uri']);
- Logger::notice('Delayed post entry deleted', ['result' => $result, 'item' => $feeditem]);
+ $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]);
foreach ($taglist as $tag) {
Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag);
From b047d31a4f681ec3038607cb1938f47a8f81ce88 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Tue, 1 Dec 2020 23:46:50 +0000
Subject: [PATCH 05/16] Added logging and default config value
---
src/Protocol/Feed.php | 4 ++++
static/defaults.config.php | 4 ++++
2 files changed, 8 insertions(+)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index f874b4b026..2d5fbff4f7 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -648,6 +648,10 @@ class Feed
$last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true);
$next_publish = max($last_publish + (60 * $min_posting), time());
if ($publish_time < $next_publish) {
+ Logger::notice('Adapting publish time',
+ ['last' => date(DateTimeFormat::MYSQL, $last_publish),
+ 'next' => date(DateTimeFormat::MYSQL, $next_publish),
+ 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]);
$publish_time = $next_publish;
}
$publish_at = date(DateTimeFormat::MYSQL, $publish_time);
diff --git a/static/defaults.config.php b/static/defaults.config.php
index f70bf8c8f6..e7d7985aa4 100644
--- a/static/defaults.config.php
+++ b/static/defaults.config.php
@@ -383,6 +383,10 @@ return [
// minimal distance in minutes between two polls for a contact. Reasonable values are between 1 and 59.
'min_poll_interval' => 15,
+ // minimum_posting_interval (Integer)
+ // Minimum interval between two feed posts per user
+ 'minimum_posting_interval' => 0,
+
// no_count (Boolean)
// Don't do count calculations (currently only when showing photo albums).
'no_count' => false,
From 4fbd0a46b58cae1f04059b1ab9b0df654940cba7 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 00:05:03 +0000
Subject: [PATCH 06/16] Reducing parameter list
---
src/Model/Post/Delayed.php | 10 ++++------
src/Protocol/Feed.php | 2 +-
2 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php
index 357a3e8aa9..963122b5b8 100644
--- a/src/Model/Post/Delayed.php
+++ b/src/Model/Post/Delayed.php
@@ -33,8 +33,6 @@ class Delayed
/**
* Insert a new delayed post
*
- * @param string $uri
- * @param integer $uid
* @param string $delayed
* @param array $item
* @param integer $notify
@@ -42,16 +40,16 @@ class Delayed
* @param array $attachments
* @return bool insert success
*/
- public static function add(string $uri, int $uid, string $delayed, array $item, int $notify = 0, array $taglist = [], array $attachments = [])
+ public static function add(string $delayed, array $item, int $notify = 0, array $taglist = [], array $attachments = [])
{
- if (self::exists($uri)) {
+ if (empty($item['uri']) || empty($item['uid']) || self::exists($item['uri'])) {
return false;
}
- Logger::notice('Adding post for delayed publishing', ['uid' => $uid, 'delayed' => $delayed, 'uri' => $uri]);
+ Logger::notice('Adding post for delayed publishing', ['uid' => $item['uid'], 'delayed' => $delayed, 'uri' => $item['uri']]);
Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $delayed], 'DelayedPublish', $item, $notify, $taglist, $attachments);
- return DBA::insert('delayed-post', ['uri' => $uri, 'uid' => $uid, 'delayed' => $delayed], Database::INSERT_IGNORE);
+ return DBA::insert('delayed-post', ['uri' => $item['uri'], 'uid' => $item['uid'], 'delayed' => $delayed], Database::INSERT_IGNORE);
}
/**
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 2d5fbff4f7..821de8c335 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -656,7 +656,7 @@ class Feed
}
$publish_at = date(DateTimeFormat::MYSQL, $publish_time);
- Post\Delayed::add($item['uri'], $item['uid'], $publish_at, $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']);
+ Post\Delayed::add($publish_at, $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']);
DI::pConfig()->set($item['uid'], 'system', 'last_publish', $next_publish);
}
}
From c302b7c347c43b3f7d1e997bf3447b7d48d7f9b1 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 00:20:07 +0000
Subject: [PATCH 07/16] Check for existing post
---
src/Protocol/Feed.php | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 821de8c335..d3d0d5ef46 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -614,7 +614,8 @@ class Feed
$notify = PRIORITY_MEDIUM;
}
- if (!Post\Delayed::exists($item["uri"])) {
+ $condition = ['uid' => $item['uid'], 'uri' => $item['uri'], 'network' => Protocol::FEED];
+ if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"])) {
$postings[] = ['item' => $item, 'notify' => $notify,
'taglist' => $taglist, 'attachments' => $attachments];
} else {
From 425440ec246c6caed22bf99d6d6763c6daf50e3d Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 00:29:57 +0000
Subject: [PATCH 08/16] The network doesn't matter
---
src/Protocol/Feed.php | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index d3d0d5ef46..48677a91b7 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -614,7 +614,7 @@ class Feed
$notify = PRIORITY_MEDIUM;
}
- $condition = ['uid' => $item['uid'], 'uri' => $item['uri'], 'network' => Protocol::FEED];
+ $condition = ['uid' => $item['uid'], 'uri' => $item['uri']];
if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"])) {
$postings[] = ['item' => $item, 'notify' => $notify,
'taglist' => $taglist, 'attachments' => $attachments];
From b2dadf0f92a2caaa52bb673736183d212bb85ba3 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 00:34:10 +0000
Subject: [PATCH 09/16] Use a combined unique index
---
database.sql | 2 +-
src/Model/Post/Delayed.php | 12 ++++++------
src/Protocol/Feed.php | 2 +-
static/dbstructure.config.php | 2 +-
4 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/database.sql b/database.sql
index 520c5e9b05..99053884d8 100644
--- a/database.sql
+++ b/database.sql
@@ -470,7 +470,7 @@ CREATE TABLE IF NOT EXISTS `delayed-post` (
`uid` mediumint unsigned COMMENT 'Owner User id',
`delayed` datetime COMMENT 'delay time',
PRIMARY KEY(`id`),
- UNIQUE INDEX `uri` (`uri`(190)),
+ UNIQUE INDEX `uid_uri` (`uid`,`uri`(190)),
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be posted at a later time';
diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php
index 963122b5b8..a29d05422a 100644
--- a/src/Model/Post/Delayed.php
+++ b/src/Model/Post/Delayed.php
@@ -42,7 +42,7 @@ class Delayed
*/
public static function add(string $delayed, array $item, int $notify = 0, array $taglist = [], array $attachments = [])
{
- if (empty($item['uri']) || empty($item['uid']) || self::exists($item['uri'])) {
+ if (empty($item['uri']) || empty($item['uid']) || self::exists($item['uri'], $item['uid'])) {
return false;
}
@@ -59,9 +59,9 @@ class Delayed
*
* @return bool delete success
*/
- private static function delete(string $uri)
+ private static function delete(string $uri, int $uid)
{
- return DBA::delete('delayed-post', ['uri' => $uri]);
+ return DBA::delete('delayed-post', ['uri' => $uri, 'uid' => $uid]);
}
/**
@@ -71,9 +71,9 @@ class Delayed
*
* @return bool "true" if an entry with that URI exists
*/
- public static function exists(string $uri)
+ public static function exists(string $uri, int $uid)
{
- return DBA::exists('delayed-post', ['uri' => $uri]);
+ return DBA::exists('delayed-post', ['uri' => $uri, 'uid' => $uid]);
}
/**
@@ -93,7 +93,7 @@ class Delayed
// It should always contain an URI since this is needed to create a delayed post entry
if (!empty($item['uri'])) {
- $result = self::delete($item['uri']);
+ $result = self::delete($item['uri'], $item['uid']);
Logger::notice('Delayed post entry deleted', ['result' => $result, 'uri' => $item['uri']]);
}
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 48677a91b7..ed24defc2e 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -615,7 +615,7 @@ class Feed
}
$condition = ['uid' => $item['uid'], 'uri' => $item['uri']];
- if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"])) {
+ if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"], $item['uid'])) {
$postings[] = ['item' => $item, 'notify' => $notify,
'taglist' => $taglist, 'attachments' => $attachments];
} else {
diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php
index 9af9c46257..106ae4e3f0 100644
--- a/static/dbstructure.config.php
+++ b/static/dbstructure.config.php
@@ -537,7 +537,7 @@ return [
],
"indexes" => [
"PRIMARY" => ["id"],
- "uri" => ["UNIQUE", "uri(190)"],
+ "uid_uri" => ["UNIQUE", "uid", "uri(190)"],
]
],
"diaspora-interaction" => [
From e9fecf1541bb233b1a702ee7b00af4f539e39a5d Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 04:53:54 +0000
Subject: [PATCH 10/16] Only delay mirrored feeds
---
src/Protocol/Feed.php | 32 +++++++++++++++++---------------
1 file changed, 17 insertions(+), 15 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index ed24defc2e..1cecee1e23 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -638,22 +638,24 @@ class Feed
$post_delay = 0;
foreach ($postings as $posting) {
- if ($delay > 0) {
- $publish_time = time() + $post_delay;
- Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]);
- $post_delay += $delay;
- } else {
- $publish_time = time();
- }
+ $publish_time = time();
- $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true);
- $next_publish = max($last_publish + (60 * $min_posting), time());
- if ($publish_time < $next_publish) {
- Logger::notice('Adapting publish time',
- ['last' => date(DateTimeFormat::MYSQL, $last_publish),
- 'next' => date(DateTimeFormat::MYSQL, $next_publish),
- 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]);
- $publish_time = $next_publish;
+ if ($posting['notify']) {
+ if ($delay > 0) {
+ $publish_time = time() + $post_delay;
+ Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]);
+ $post_delay += $delay;
+ }
+
+ $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true);
+ $next_publish = max($last_publish + (60 * $min_posting), time());
+ if ($publish_time < $next_publish) {
+ Logger::notice('Adapting publish time',
+ ['last' => date(DateTimeFormat::MYSQL, $last_publish),
+ 'next' => date(DateTimeFormat::MYSQL, $next_publish),
+ 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]);
+ $publish_time = $next_publish;
+ }
}
$publish_at = date(DateTimeFormat::MYSQL, $publish_time);
From 775075c5db8a06ffa0d0808fb5c45e9908cc9074 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 05:07:44 +0000
Subject: [PATCH 11/16] Only spool mirrored posts
---
src/Model/Post/Delayed.php | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php
index a29d05422a..87e5fe8235 100644
--- a/src/Model/Post/Delayed.php
+++ b/src/Model/Post/Delayed.php
@@ -92,7 +92,7 @@ class Delayed
Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
// It should always contain an URI since this is needed to create a delayed post entry
- if (!empty($item['uri'])) {
+ if (!empty($item['uri']) && self::exists($item['uri'], $item['uid'])) {
$result = self::delete($item['uri'], $item['uid']);
Logger::notice('Delayed post entry deleted', ['result' => $result, 'uri' => $item['uri']]);
}
From b54465ba9c3c3fcea52f2d2907496c056dc0e152 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 05:08:39 +0000
Subject: [PATCH 12/16] Directly post non mirrored feed items
---
src/Protocol/Feed.php | 35 +++++++++++++++++++----------------
1 file changed, 19 insertions(+), 16 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 1cecee1e23..dbfb72977d 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -638,24 +638,27 @@ class Feed
$post_delay = 0;
foreach ($postings as $posting) {
- $publish_time = time();
+ if (!$posting['notify']) {
+ Post\Delayed::publish($posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']);
+ continue;
+ }
- if ($posting['notify']) {
- if ($delay > 0) {
- $publish_time = time() + $post_delay;
- Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]);
- $post_delay += $delay;
- }
+ if ($delay > 0) {
+ $publish_time = time() + $post_delay;
+ Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]);
+ $post_delay += $delay;
+ } else {
+ $publish_time = time();
+ }
- $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true);
- $next_publish = max($last_publish + (60 * $min_posting), time());
- if ($publish_time < $next_publish) {
- Logger::notice('Adapting publish time',
- ['last' => date(DateTimeFormat::MYSQL, $last_publish),
- 'next' => date(DateTimeFormat::MYSQL, $next_publish),
- 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]);
- $publish_time = $next_publish;
- }
+ $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true);
+ $next_publish = max($last_publish + (60 * $min_posting), time());
+ if ($publish_time < $next_publish) {
+ Logger::notice('Adapting publish time',
+ ['last' => date(DateTimeFormat::MYSQL, $last_publish),
+ 'next' => date(DateTimeFormat::MYSQL, $next_publish),
+ 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]);
+ $publish_time = $next_publish;
}
$publish_at = date(DateTimeFormat::MYSQL, $publish_time);
From 6c3b1ac4bd1a27237209e63a6ee9bc0d590563a8 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 05:41:33 +0000
Subject: [PATCH 13/16] Moved direct storage
---
src/Protocol/Feed.php | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index dbfb72977d..c5234c8cd4 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -616,10 +616,14 @@ class Feed
$condition = ['uid' => $item['uid'], 'uri' => $item['uri']];
if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"], $item['uid'])) {
- $postings[] = ['item' => $item, 'notify' => $notify,
- 'taglist' => $taglist, 'attachments' => $attachments];
+ if (!$notify) {
+ Post\Delayed::publish($item, $notify, $taglist, $attachments);
+ } else {
+ $postings[] = ['item' => $item, 'notify' => $notify,
+ 'taglist' => $taglist, 'attachments' => $attachments];
+ }
} else {
- Logger::info('Post already exists in the delayed posts queue', ['uri' => $item["uri"]]);
+ Logger::info('Post already crated or exists in the delayed posts queue', ['uri' => $item["uri"]]);
}
}
@@ -638,11 +642,6 @@ class Feed
$post_delay = 0;
foreach ($postings as $posting) {
- if (!$posting['notify']) {
- Post\Delayed::publish($posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']);
- continue;
- }
-
if ($delay > 0) {
$publish_time = time() + $post_delay;
Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]);
@@ -657,7 +656,9 @@ class Feed
Logger::notice('Adapting publish time',
['last' => date(DateTimeFormat::MYSQL, $last_publish),
'next' => date(DateTimeFormat::MYSQL, $next_publish),
- 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]);
+ 'publish' => date(DateTimeFormat::MYSQL, $publish_time),
+ 'uid' => $posting['item']['uid'], 'cid' => $posting['item']['contact-id'],
+ 'uri' => $posting['item']["uri"]]);
$publish_time = $next_publish;
}
$publish_at = date(DateTimeFormat::MYSQL, $publish_time);
From c1d1b189f7356aa221f4b9d7138e34ba5de51b17 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 08:36:23 +0000
Subject: [PATCH 14/16] Cleaned code
---
src/Model/Post/Delayed.php | 4 +---
src/Protocol/Feed.php | 11 ++---------
2 files changed, 3 insertions(+), 12 deletions(-)
diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php
index 87e5fe8235..bfab406282 100644
--- a/src/Model/Post/Delayed.php
+++ b/src/Model/Post/Delayed.php
@@ -91,10 +91,8 @@ class Delayed
Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
- // It should always contain an URI since this is needed to create a delayed post entry
if (!empty($item['uri']) && self::exists($item['uri'], $item['uid'])) {
- $result = self::delete($item['uri'], $item['uid']);
- Logger::notice('Delayed post entry deleted', ['result' => $result, 'uri' => $item['uri']]);
+ self::delete($item['uri'], $item['uid']);
}
if (!empty($id) && (!empty($taglist) || !empty($attachments))) {
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index c5234c8cd4..fa568b4c3c 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -623,7 +623,7 @@ class Feed
'taglist' => $taglist, 'attachments' => $attachments];
}
} else {
- Logger::info('Post already crated or exists in the delayed posts queue', ['uri' => $item["uri"]]);
+ Logger::info('Post already crated or exists in the delayed posts queue', ['uid' => $item['uid'], 'uri' => $item["uri"]]);
}
}
@@ -634,7 +634,7 @@ class Feed
// Posts shouldn't be delayed more than a day
$interval = min(1440, self::getPollInterval($contact));
$delay = max(round(($interval * 60) / $total), 60 * $min_posting);
- Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]);
+ Logger::info('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]);
} else {
$delay = 0;
}
@@ -644,7 +644,6 @@ class Feed
foreach ($postings as $posting) {
if ($delay > 0) {
$publish_time = time() + $post_delay;
- Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]);
$post_delay += $delay;
} else {
$publish_time = time();
@@ -653,12 +652,6 @@ class Feed
$last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true);
$next_publish = max($last_publish + (60 * $min_posting), time());
if ($publish_time < $next_publish) {
- Logger::notice('Adapting publish time',
- ['last' => date(DateTimeFormat::MYSQL, $last_publish),
- 'next' => date(DateTimeFormat::MYSQL, $next_publish),
- 'publish' => date(DateTimeFormat::MYSQL, $publish_time),
- 'uid' => $posting['item']['uid'], 'cid' => $posting['item']['contact-id'],
- 'uri' => $posting['item']["uri"]]);
$publish_time = $next_publish;
}
$publish_at = date(DateTimeFormat::MYSQL, $publish_time);
From bd4dc10588d2c1b5d3c2b4ba2476d87700b0d4a6 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 08:50:45 +0000
Subject: [PATCH 15/16] Ensure that the database.sql contains "if exists"
---
database.sql | 16 ++++++++--------
src/Database/View.php | 4 ++--
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/database.sql b/database.sql
index 99053884d8..e5e1ca2393 100644
--- a/database.sql
+++ b/database.sql
@@ -1485,7 +1485,7 @@ CREATE TABLE IF NOT EXISTS `workerqueue` (
--
-- VIEW category-view
--
-DROP VIEW `category-view`;
+DROP VIEW IF EXISTS `category-view`;
CREATE VIEW `category-view` AS SELECT
`post-category`.`uri-id` AS `uri-id`,
`post-category`.`uid` AS `uid`,
@@ -1502,7 +1502,7 @@ CREATE VIEW `category-view` AS SELECT
--
-- VIEW tag-view
--
-DROP VIEW `tag-view`;
+DROP VIEW IF EXISTS `tag-view`;
CREATE VIEW `tag-view` AS SELECT
`post-tag`.`uri-id` AS `uri-id`,
`item-uri`.`uri` AS `uri`,
@@ -1520,7 +1520,7 @@ CREATE VIEW `tag-view` AS SELECT
--
-- VIEW network-item-view
--
-DROP VIEW `network-item-view`;
+DROP VIEW IF EXISTS `network-item-view`;
CREATE VIEW `network-item-view` AS SELECT
`item`.`parent-uri-id` AS `uri-id`,
`item`.`parent-uri` AS `uri`,
@@ -1552,7 +1552,7 @@ CREATE VIEW `network-item-view` AS SELECT
--
-- VIEW network-thread-view
--
-DROP VIEW `network-thread-view`;
+DROP VIEW IF EXISTS `network-thread-view`;
CREATE VIEW `network-thread-view` AS SELECT
`item`.`uri-id` AS `uri-id`,
`item`.`uri` AS `uri`,
@@ -1583,7 +1583,7 @@ CREATE VIEW `network-thread-view` AS SELECT
--
-- VIEW owner-view
--
-DROP VIEW `owner-view`;
+DROP VIEW IF EXISTS `owner-view`;
CREATE VIEW `owner-view` AS SELECT
`contact`.`id` AS `id`,
`contact`.`uid` AS `uid`,
@@ -1718,7 +1718,7 @@ CREATE VIEW `owner-view` AS SELECT
--
-- VIEW pending-view
--
-DROP VIEW `pending-view`;
+DROP VIEW IF EXISTS `pending-view`;
CREATE VIEW `pending-view` AS SELECT
`register`.`id` AS `id`,
`register`.`hash` AS `hash`,
@@ -1740,7 +1740,7 @@ CREATE VIEW `pending-view` AS SELECT
--
-- VIEW tag-search-view
--
-DROP VIEW `tag-search-view`;
+DROP VIEW IF EXISTS `tag-search-view`;
CREATE VIEW `tag-search-view` AS SELECT
`post-tag`.`uri-id` AS `uri-id`,
`item`.`id` AS `iid`,
@@ -1761,7 +1761,7 @@ CREATE VIEW `tag-search-view` AS SELECT
--
-- VIEW workerqueue-view
--
-DROP VIEW `workerqueue-view`;
+DROP VIEW IF EXISTS `workerqueue-view`;
CREATE VIEW `workerqueue-view` AS SELECT
`process`.`pid` AS `pid`,
`workerqueue`.`priority` AS `priority`
diff --git a/src/Database/View.php b/src/Database/View.php
index 51d051786d..fd0a8ce1a5 100644
--- a/src/Database/View.php
+++ b/src/Database/View.php
@@ -112,9 +112,9 @@ class View
}
if (self::isView($name)) {
- $sql = sprintf("DROP VIEW `%s`", DBA::escape($name));
+ $sql = sprintf("DROP VIEW IF EXISTS `%s`", DBA::escape($name));
} elseif (self::isTable($name)) {
- $sql = sprintf("DROP TABLE `%s`", DBA::escape($name));
+ $sql = sprintf("DROP TABLE IF EXISTS `%s`", DBA::escape($name));
}
if ($verbose) {
From 90349d14e1e76e6a60506789299cd309602934f3 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 2 Dec 2020 14:55:57 +0000
Subject: [PATCH 16/16] Chaning wording
---
database.sql | 4 ++--
static/dbstructure.config.php | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/database.sql b/database.sql
index e5e1ca2393..b04b088d8d 100644
--- a/database.sql
+++ b/database.sql
@@ -466,13 +466,13 @@ CREATE TABLE IF NOT EXISTS `conversation` (
--
CREATE TABLE IF NOT EXISTS `delayed-post` (
`id` int unsigned NOT NULL auto_increment,
- `uri` varchar(255) COMMENT 'URI of the post that will be posted later',
+ `uri` varchar(255) COMMENT 'URI of the post that will be distributed later',
`uid` mediumint unsigned COMMENT 'Owner User id',
`delayed` datetime COMMENT 'delay time',
PRIMARY KEY(`id`),
UNIQUE INDEX `uid_uri` (`uid`,`uri`(190)),
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be posted at a later time';
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be distributed at a later time';
--
-- TABLE diaspora-interaction
diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php
index 106ae4e3f0..bc07c67245 100644
--- a/static/dbstructure.config.php
+++ b/static/dbstructure.config.php
@@ -528,10 +528,10 @@ return [
]
],
"delayed-post" => [
- "comment" => "Posts that are about to be posted at a later time",
+ "comment" => "Posts that are about to be distributed at a later time",
"fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"],
- "uri" => ["type" => "varchar(255)", "comment" => "URI of the post that will be posted later"],
+ "uri" => ["type" => "varchar(255)", "comment" => "URI of the post that will be distributed later"],
"uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Owner User id"],
"delayed" => ["type" => "datetime", "comment" => "delay time"],
],