Merge remote-tracking branch 'upstream/develop' into worker-command

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2020.12-dev (Red Hot Poker)
--- DB_UPDATE_VERSION 1381
+-- DB_UPDATE_VERSION 1382
 -- ------------------------------------------
 
 
@@ -461,6 +461,19 @@ CREATE TABLE IF NOT EXISTS `conversation` (
 	 INDEX `received` (`received`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages';
 
+--
+-- TABLE delayed-post
+--
+CREATE TABLE IF NOT EXISTS `delayed-post` (
+	`id` int unsigned NOT NULL auto_increment,
+	`uri` varchar(255) COMMENT 'URI of the post that will be distributed later',
+	`uid` mediumint unsigned COMMENT 'Owner User id',
+	`delayed` datetime COMMENT 'delay time',
+	 PRIMARY KEY(`id`),
+	 UNIQUE INDEX `uid_uri` (`uid`,`uri`(190)),
+	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be distributed at a later time';
+
 --
 -- TABLE diaspora-interaction
 --
@@ -1756,3 +1769,4 @@ CREATE VIEW `workerqueue-view` AS SELECT
 			INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid`
 			WHERE NOT `workerqueue`.`done`;
 
+

src/Database/View.php

@@ -112,9 +112,9 @@ class View
 		}
 
 		if (self::isView($name)) {
-			$sql = sprintf("DROP VIEW `%s`", DBA::escape($name));
+			$sql = sprintf("DROP VIEW IF EXISTS `%s`", DBA::escape($name));
 		} elseif (self::isTable($name)) {
-			$sql = sprintf("DROP TABLE `%s`", DBA::escape($name));
+			$sql = sprintf("DROP TABLE IF EXISTS `%s`", DBA::escape($name));
 		}
 
 		if ($verbose) {

src/Model/Post/Delayed.php

@@ -0,0 +1,113 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use Friendica\Core\Logger;
+use Friendica\Database\DBA;
+use Friendica\Core\Worker;
+use Friendica\Database\Database;
+use Friendica\Model\Item;
+use Friendica\Model\Tag;
+
+class Delayed
+{
+	/**
+	 * Insert a new delayed post
+	 *
+	 * @param string $delayed
+	 * @param array $item
+	 * @param integer $notify
+	 * @param array $taglist
+	 * @param array $attachments
+	 * @return bool insert success
+	 */ 
+	public static function add(string $delayed, array $item, int $notify = 0, array $taglist = [], array $attachments = [])
+	{
+		if (empty($item['uri']) || empty($item['uid']) || self::exists($item['uri'], $item['uid'])) {
+			return false;
+		}
+
+		Logger::notice('Adding post for delayed publishing', ['uid' => $item['uid'], 'delayed' => $delayed, 'uri' => $item['uri']]);
+
+		Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $delayed], 'DelayedPublish', $item, $notify, $taglist, $attachments);
+		return DBA::insert('delayed-post', ['uri' => $item['uri'], 'uid' => $item['uid'], 'delayed' => $delayed], Database::INSERT_IGNORE);
+	}
+
+	/**
+	 * Delete a delayed post
+	 *
+	 * @param string $uri
+	 *
+	 * @return bool delete success
+	 */
+	private static function delete(string $uri, int $uid)
+	{
+		return DBA::delete('delayed-post', ['uri' => $uri, 'uid' => $uid]);
+	}
+
+	/**
+	 * Check if an entry exists
+	 *
+	 * @param string $uri
+	 *
+	 * @return bool "true" if an entry with that URI exists
+	 */
+	public static function exists(string $uri, int $uid)
+	{
+		return DBA::exists('delayed-post', ['uri' => $uri, 'uid' => $uid]);
+	}
+
+	/**
+	 * Publish a delayed post
+	 *
+	 * @param array $item
+	 * @param integer $notify
+	 * @param array $taglist
+	 * @param array $attachments
+	 * @return bool
+	 */
+	public static function publish(array $item, int $notify = 0, array $taglist = [], array $attachments = [])
+	{
+		$id = Item::insert($item, $notify);
+
+		Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
+
+		if (!empty($item['uri']) && self::exists($item['uri'], $item['uid'])) {
+			self::delete($item['uri'], $item['uid']);
+		}
+
+		if (!empty($id) && (!empty($taglist) || !empty($attachments))) {
+			$feeditem = Item::selectFirst(['uri-id'], ['id' => $id]);
+
+			foreach ($taglist as $tag) {
+				Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag);
+			}
+
+			foreach ($attachments as $attachment) {
+				$attachment['uri-id'] = $feeditem['uri-id'];
+				Media::insert($attachment);
+			}
+		}
+
+		return $id;
+	}
+}

src/Protocol/Feed.php

@@ -606,7 +606,7 @@ class Feed
 			// Additionally we have to avoid conflicts with identical URI between imported feeds and these items.
 			if ($notify) {
 				$item['guid'] = Item::guidFromUri($orig_plink, DI::baseUrl()->getHostname());
-				unset($item['uri']);
+				$item['uri'] = Item::newURI($item['uid'], $item['guid']);
 				unset($item['thr-parent']);
 				unset($item['parent-uri']);
 
@@ -614,17 +614,27 @@ class Feed
 				$notify = PRIORITY_MEDIUM;
 			}
 
+			$condition = ['uid' => $item['uid'], 'uri' => $item['uri']];
+			if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"], $item['uid'])) {
+				if (!$notify) {
+					Post\Delayed::publish($item, $notify, $taglist, $attachments);
+				} else {
 					$postings[] = ['item' => $item, 'notify' => $notify,
 						'taglist' => $taglist, 'attachments' => $attachments];
 				}
+			} else {
+				Logger::info('Post already crated or exists in the delayed posts queue', ['uid' => $item['uid'], 'uri' => $item["uri"]]);
+			}
+		}
 
 		if (!empty($postings)) {
+			$min_posting = DI::config()->get('system', 'minimum_posting_interval', 0);
 			$total = count($postings);
 			if ($total > 1) {
 				// Posts shouldn't be delayed more than a day
 				$interval = min(1440, self::getPollInterval($contact));
-				$delay = round(($interval * 60) / $total);
+				$delay = max(round(($interval * 60) / $total), 60 * $min_posting);
-				Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]);
+				Logger::info('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]);
 			} else {
 				$delay = 0;
 			}
@@ -633,15 +643,21 @@ class Feed
 
 			foreach ($postings as $posting) {
 				if ($delay > 0) {
-					$publish_at = DateTimeFormat::utc('now + ' . $post_delay . ' second');
+					$publish_time = time() + $post_delay;
-					Logger::notice('Got publishing date', ['delay' => $delay, 'publish_at' => $publish_at, 'cid' => $contact['id'], 'url' => $contact['url']]);
 					$post_delay += $delay;
 				} else {
-					$publish_at = DBA::NULL_DATETIME;
+					$publish_time = time();
 				}
 
-				Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $publish_at],
+				$last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true);
-					'DelayedPublish', $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']);
+				$next_publish = max($last_publish + (60 * $min_posting), time());
+				if ($publish_time < $next_publish) {
+					$publish_time = $next_publish;
+				}
+				$publish_at = date(DateTimeFormat::MYSQL, $publish_time);
+
+				Post\Delayed::add($publish_at, $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']);
+				DI::pConfig()->set($item['uid'], 'system', 'last_publish', $next_publish);
 			}
 		}
 

src/Worker/DelayedPublish.php

@@ -39,20 +39,7 @@ class DelayedPublish
 	  */
 	public static function execute(array $item, int $notify = 0, array $taglist = [], array $attachments = [])
 	{
-		$id = Item::insert($item, $notify);
+		$id = Post\Delayed::publish($item, $notify, $taglist, $attachments);
-
+		Logger::notice('Post published', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
-		Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
-
-		if (!empty($id) && (!empty($taglist) || !empty($attachments))) {
-			$feeditem = Item::selectFirst(['uri-id'], ['id' => $id]);
-			foreach ($taglist as $tag) {
-				Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag);
-			}
-			foreach ($attachments as $attachment) {
-				$attachment['uri-id'] = $feeditem['uri-id'];
-				Post\Media::insert($attachment);
-			}
-		}
-
 	}
 }

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1381);
+	define('DB_UPDATE_VERSION', 1382);
 }
 
 return [
@@ -529,6 +529,19 @@ return [
 			"received" => ["received"],
 		]
 	],
+	"delayed-post" => [
+		"comment" => "Posts that are about to be distributed at a later time",
+		"fields" => [
+			"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"],
+			"uri" => ["type" => "varchar(255)", "comment" => "URI of the post that will be distributed later"],
+			"uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Owner User id"],
+			"delayed" => ["type" => "datetime", "comment" => "delay time"],
+		],
+		"indexes" => [
+			"PRIMARY" => ["id"],
+			"uid_uri" => ["UNIQUE", "uid", "uri(190)"],
+		]
+	],
 	"diaspora-interaction" => [
 		"comment" => "Signed Diaspora Interaction",
 		"fields" => [

static/defaults.config.php

@@ -383,6 +383,10 @@ return [
 		// minimal distance in minutes between two polls for a contact. Reasonable values are between 1 and 59.
 		'min_poll_interval' => 15,
 
+		// minimum_posting_interval (Integer)
+		// Minimum interval between two feed posts per user
+		'minimum_posting_interval' => 0,
+
 		// no_count (Boolean)
 		// Don't do count calculations (currently only when showing photo albums).
 		'no_count' => false,