From 2f462ffa1674c9f79acba4c317eeb07ffb105227 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 18 Jul 2022 03:31:00 +0000 Subject: [PATCH 01/26] Store incoming posts in a queue --- src/Protocol/ActivityPub/Processor.php | 3 ++ src/Protocol/ActivityPub/Receiver.php | 51 ++++++++++++++++++++------ static/dbstructure.config.php | 32 +++++++++++++++- 3 files changed, 74 insertions(+), 12 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index c6420436e8..c49a2da686 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -223,6 +223,8 @@ class Processor Post\History::add($item['uri-id'], $item); Item::update($item, ['uri' => $activity['id']]); + DBA::delete('inbox-queue', ['url' => $item['uri']]); + if ($activity['object_type'] == 'as:Event') { $posts = Post::select(['event-id', 'uid'], ["`uri` = ? AND `event-id` > ?", $activity['id'], 0]); while ($post = DBA::fetch($posts)) { @@ -890,6 +892,7 @@ class Processor $item_id = Item::insert($item); if ($item_id) { Logger::info('Item insertion successful', ['user' => $item['uid'], 'item_id' => $item_id]); + DBA::delete('inbox-queue', ['url' => $item['uri']]); } else { Logger::notice('Item insertion aborted', ['user' => $item['uid']]); } diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 2a56e9e5e7..b55636a30e 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -28,6 +28,7 @@ use Friendica\Content\Text\Markdown; use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Core\System; +use Friendica\Database\Database; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\APContact; @@ -36,6 +37,7 @@ use Friendica\Model\Post; use Friendica\Model\User; use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; +use Friendica\Util\DateTimeFormat; use Friendica\Util\HTTPSignature; use Friendica\Util\JsonLD; use Friendica\Util\LDSignature; @@ -96,7 +98,21 @@ class Receiver $ldactivity = JsonLD::compact($activity); + $http_signer = HTTPSignature::getSigner($body, $header); + if ($http_signer === false) { + Logger::warning('Invalid HTTP signature, message will be discarded.'); + return; + } elseif (empty($http_signer)) { + Logger::info('Signer is a tombstone. The message will be discarded, the signer account is deleted.'); + return; + } else { + Logger::info('Valid HTTP signature', ['signer' => $http_signer]); + } + + self::enqueuePost($ldactivity, $http_signer, $uid); + $actor = JsonLD::fetchElement($ldactivity, 'as:actor', '@id') ?? ''; + $apcontact = APContact::getByURL($actor); if (empty($apcontact)) { @@ -109,17 +125,6 @@ class Receiver APContact::unmarkForArchival($apcontact); } - $http_signer = HTTPSignature::getSigner($body, $header); - if ($http_signer === false) { - Logger::warning('Invalid HTTP signature, message will be discarded.'); - return; - } elseif (empty($http_signer)) { - Logger::info('Signer is a tombstone. The message will be discarded, the signer account is deleted.'); - return; - } else { - Logger::info('Valid HTTP signature', ['signer' => $http_signer]); - } - $signer = [$http_signer]; Logger::info('Message for user ' . $uid . ' is from actor ' . $actor); @@ -157,6 +162,30 @@ class Receiver $fetchQueue->process(); } + private static function enqueuePost(array $ldactivity = [], string $signer, int $uid) + { + if (empty($ldactivity['as:object'])) { + return; + } + + $url = JsonLD::fetchElement($ldactivity, 'as:object', '@id'); + $fields = [ + 'url' => $url, + 'in-reply-to-url' => JsonLD::fetchElement($ldactivity['as:object'], 'as:inReplyTo', '@id'), + 'signer' => $signer, + 'type' => JsonLD::fetchElement($ldactivity, '@type'), + 'object-type' => JsonLD::fetchElement($ldactivity['as:object'], '@type'), + 'activity' => json_encode($ldactivity, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT), + 'received' => DateTimeFormat::utcNow(), + ]; + DBA::insert('inbox-queue', $fields, Database::INSERT_IGNORE); + + $queue = DBA::selectFirst('inbox-queue', ['id'], ['url' => $url]); + if (!empty($queue['id'])) { + DBA::insert('inbox-queue-receiver', ['queue-id' => $queue['id'], 'uid' => $uid], Database::INSERT_IGNORE); + } + } + /** * Process incoming posts from relays * diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 758c33d0da..1d8b21b02c 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1473); + define('DB_UPDATE_VERSION', 1474); } return [ @@ -784,6 +784,36 @@ return [ "hook_file_function" => ["UNIQUE", "hook", "file", "function"], ] ], + "inbox-queue" => [ + "comment" => "Incoming activity", + "fields" => [ + "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"], + "url" => ["type" => "varbinary(255)", "comment" => "id of the incoming activity"], + "in-reply-to-url" => ["type" => "varbinary(255)", "comment" => "related id of the incoming activity"], + "signer" => ["type" => "varbinary(255)", "comment" => "Signer of the incoming activity"], + "activity" => ["type" => "mediumtext", "comment" => "The JSON activity"], + "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Receiving date"], + "type" => ["type" => "varchar(64)", "not null" => "1", "default" => "", "comment" => "Type of the activity"], + "object-type" => ["type" => "varchar(64)", "not null" => "1", "default" => "", "comment" => "Type of the object activity"], + ], + "indexes" => [ + "PRIMARY" => ["id"], + "url" => ["UNIQUE", "url"], + "in-reply-to-url" => ["in-reply-to-url"], + "received" => ["received"], + ] + ], + "inbox-queue-receiver" => [ + "comment" => "Receiver for the incoming activity", + "fields" => [ + "queue-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["inbox-queue" => "id"], "comment" => ""], + "uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "foreign" => ["user" => "uid"], "comment" => "User id"], + ], + "indexes" => [ + "PRIMARY" => ["queue-id", "uid"], + "uid" => ["uid"], + ] + ], "inbox-status" => [ "comment" => "Status of ActivityPub inboxes", "fields" => [ From ce38e63399aa41b87836dea85f13e3c0491ef5e3 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 18 Jul 2022 14:03:49 +0000 Subject: [PATCH 02/26] Add and delete entries --- src/Protocol/ActivityPub/Processor.php | 7 ++- src/Protocol/ActivityPub/Receiver.php | 73 +++++++++++++++----------- static/dbstructure.config.php | 25 ++++----- 3 files changed, 61 insertions(+), 44 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index c49a2da686..6e81cb767f 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -223,7 +223,7 @@ class Processor Post\History::add($item['uri-id'], $item); Item::update($item, ['uri' => $activity['id']]); - DBA::delete('inbox-queue', ['url' => $item['uri']]); + Receiver::removeFromQueue($activity); if ($activity['object_type'] == 'as:Event') { $posts = Post::select(['event-id', 'uid'], ["`uri` = ? AND `event-id` > ?", $activity['id'], 0]); @@ -428,6 +428,7 @@ class Processor Logger::info('Deleting item', ['object' => $activity['object_id'], 'owner' => $owner]); Item::markForDeletion(['uri' => $activity['object_id'], 'owner-id' => $owner]); + Receiver::removeFromQueue($activity); } /** @@ -892,7 +893,7 @@ class Processor $item_id = Item::insert($item); if ($item_id) { Logger::info('Item insertion successful', ['user' => $item['uid'], 'item_id' => $item_id]); - DBA::delete('inbox-queue', ['url' => $item['uri']]); + Receiver::removeFromQueue($activity); } else { Logger::notice('Item insertion aborted', ['user' => $item['uid']]); } @@ -1351,6 +1352,7 @@ class Processor Logger::info('Updating profile', ['object' => $activity['object_id']]); Contact::updateFromProbeByURL($activity['object_id']); + Receiver::removeFromQueue($activity); } /** @@ -1379,6 +1381,7 @@ class Processor DBA::close($contacts); Logger::info('Deleted contact', ['object' => $activity['object_id']]); + Receiver::removeFromQueue($activity); } /** diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index b55636a30e..11e7151e1e 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -98,19 +98,6 @@ class Receiver $ldactivity = JsonLD::compact($activity); - $http_signer = HTTPSignature::getSigner($body, $header); - if ($http_signer === false) { - Logger::warning('Invalid HTTP signature, message will be discarded.'); - return; - } elseif (empty($http_signer)) { - Logger::info('Signer is a tombstone. The message will be discarded, the signer account is deleted.'); - return; - } else { - Logger::info('Valid HTTP signature', ['signer' => $http_signer]); - } - - self::enqueuePost($ldactivity, $http_signer, $uid); - $actor = JsonLD::fetchElement($ldactivity, 'as:actor', '@id') ?? ''; $apcontact = APContact::getByURL($actor); @@ -125,6 +112,17 @@ class Receiver APContact::unmarkForArchival($apcontact); } + $http_signer = HTTPSignature::getSigner($body, $header); + if ($http_signer === false) { + Logger::warning('Invalid HTTP signature, message will be discarded.'); + return; + } elseif (empty($http_signer)) { + Logger::info('Signer is a tombstone. The message will be discarded, the signer account is deleted.'); + return; + } else { + Logger::info('Valid HTTP signature', ['signer' => $http_signer]); + } + $signer = [$http_signer]; Logger::info('Message for user ' . $uid . ' is from actor ' . $actor); @@ -158,32 +156,45 @@ class Receiver } $fetchQueue = new FetchQueue(); - self::processActivity($fetchQueue, $ldactivity, $body, $uid, $trust_source, true, $signer); + self::processActivity($fetchQueue, $ldactivity, $body, $uid, $trust_source, true, $signer, $http_signer); $fetchQueue->process(); } - private static function enqueuePost(array $ldactivity = [], string $signer, int $uid) + private static function enqueuePost(array $ldactivity = [], string $type, int $uid, string $http_signer): array { - if (empty($ldactivity['as:object'])) { - return; - } - - $url = JsonLD::fetchElement($ldactivity, 'as:object', '@id'); $fields = [ - 'url' => $url, - 'in-reply-to-url' => JsonLD::fetchElement($ldactivity['as:object'], 'as:inReplyTo', '@id'), - 'signer' => $signer, - 'type' => JsonLD::fetchElement($ldactivity, '@type'), - 'object-type' => JsonLD::fetchElement($ldactivity['as:object'], '@type'), + 'activity-id' => $ldactivity['id'], + 'object-id' => $ldactivity['object_id'], + 'type' => $type, + 'object-type' => $ldactivity['object_type'], 'activity' => json_encode($ldactivity, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT), 'received' => DateTimeFormat::utcNow(), ]; - DBA::insert('inbox-queue', $fields, Database::INSERT_IGNORE); - $queue = DBA::selectFirst('inbox-queue', ['id'], ['url' => $url]); - if (!empty($queue['id'])) { - DBA::insert('inbox-queue-receiver', ['queue-id' => $queue['id'], 'uid' => $uid], Database::INSERT_IGNORE); + if (!empty($ldactivity['object_object_type'])) { + $fields['object-object-type'] = $ldactivity['object_object_type']; } + + if (!empty($http_signer)) { + $fields['signer'] = $http_signer; + } + + DBA::insert('inbox-entry', $fields, Database::INSERT_IGNORE); + + $queue = DBA::selectFirst('inbox-entry', ['id'], ['activity-id' => $ldactivity['id']]); + if (!empty($queue['id'])) { + $ldactivity['entry-id'] = $queue['id']; + DBA::insert('inbox-entry-receiver', ['queue-id' => $queue['id'], 'uid' => $uid], Database::INSERT_IGNORE); + } + return $ldactivity; + } + + public static function removeFromQueue(array $activity = []) + { + if (empty($activity['entry-id'])) { + return; + } + DBA::delete('inbox-entry', ['id' => $activity['entry-id']]); } /** @@ -517,7 +528,7 @@ class Receiver * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function processActivity(FetchQueue $fetchQueue, array $activity, string $body = '', int $uid = null, bool $trust_source = false, bool $push = false, array $signer = []) + public static function processActivity(FetchQueue $fetchQueue, array $activity, string $body = '', int $uid = null, bool $trust_source = false, bool $push = false, array $signer = [], string $http_signer = '') { $type = JsonLD::fetchElement($activity, '@type'); if (!$type) { @@ -591,6 +602,8 @@ class Receiver $object_data['from-relay'] = $activity['from-relay']; } + $object_data = self::enqueuePost($object_data, $type, $uid, $http_signer); + if (in_array('as:Question', [$object_data['object_type'] ?? '', $object_data['object_object_type'] ?? ''])) { self::storeUnhandledActivity(false, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 1d8b21b02c..c1f1cfcbc1 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -784,30 +784,31 @@ return [ "hook_file_function" => ["UNIQUE", "hook", "file", "function"], ] ], - "inbox-queue" => [ + "inbox-entry" => [ "comment" => "Incoming activity", "fields" => [ "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"], - "url" => ["type" => "varbinary(255)", "comment" => "id of the incoming activity"], - "in-reply-to-url" => ["type" => "varbinary(255)", "comment" => "related id of the incoming activity"], - "signer" => ["type" => "varbinary(255)", "comment" => "Signer of the incoming activity"], + "activity-id" => ["type" => "varbinary(255)", "comment" => "id of the incoming activity"], + "object-id" => ["type" => "varbinary(255)", "comment" => ""], + "type" => ["type" => "varchar(64)", "comment" => "Type of the activity"], + "object-type" => ["type" => "varchar(64)", "comment" => "Type of the object activity"], + "object-object-type" => ["type" => "varchar(64)", "comment" => "Type of the object's object activity"], + "received" => ["type" => "datetime", "comment" => "Receiving date"], "activity" => ["type" => "mediumtext", "comment" => "The JSON activity"], - "received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Receiving date"], - "type" => ["type" => "varchar(64)", "not null" => "1", "default" => "", "comment" => "Type of the activity"], - "object-type" => ["type" => "varchar(64)", "not null" => "1", "default" => "", "comment" => "Type of the object activity"], + "signer" => ["type" => "varchar(255)", "comment" => ""], ], "indexes" => [ "PRIMARY" => ["id"], - "url" => ["UNIQUE", "url"], - "in-reply-to-url" => ["in-reply-to-url"], + "activity-id" => ["UNIQUE", "activity-id"], + "object-id" => ["object-id"], "received" => ["received"], ] ], - "inbox-queue-receiver" => [ + "inbox-entry-receiver" => [ "comment" => "Receiver for the incoming activity", "fields" => [ - "queue-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["inbox-queue" => "id"], "comment" => ""], - "uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "foreign" => ["user" => "uid"], "comment" => "User id"], + "queue-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["inbox-entry" => "id"], "comment" => ""], + "uid" => ["type" => "mediumint unsigned", "not null" => "1", "primary" => "1", "foreign" => ["user" => "uid"], "comment" => "User id"], ], "indexes" => [ "PRIMARY" => ["queue-id", "uid"], From 7dcd02938de26ba6cb612b88d3988c868d09a82b Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 20 Jul 2022 05:37:40 +0000 Subject: [PATCH 03/26] Remove entries from queue / relay detection --- src/Model/APContact.php | 18 +++++++++++++++++- src/Protocol/ActivityPub/Processor.php | 6 ++++++ src/Protocol/ActivityPub/Receiver.php | 24 +++++++++++++++--------- src/Util/XML.php | 4 ++-- 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/Model/APContact.php b/src/Model/APContact.php index baa364e940..ba2e3c9734 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -26,7 +26,6 @@ use Friendica\Core\Cache\Enum\Duration; use Friendica\Core\Logger; use Friendica\Core\System; use Friendica\Database\DBA; -use Friendica\Database\DBStructure; use Friendica\DI; use Friendica\Network\HTTPClient\Client\HttpClientAccept; use Friendica\Network\HTTPException; @@ -539,4 +538,21 @@ class APContact HTTPSignature::setInboxStatus($url, true, $shared); } + + public static function isRelay(array $apcontact): bool + { + if ($apcontact['nick'] != 'relay') { + return false; + } + + if ($apcontact['type'] == 'Application') { + return true; + } + + if (in_array($apcontact['type'], ['Group', 'Service']) && ($apcontact['nick'] == 'relay') && is_null($apcontact['outbox'])) { + return true; + } + + return false; + } } diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 6e81cb767f..e6ff43f06a 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -546,6 +546,7 @@ class Processor Logger::debug('Add post to featured collection', ['uri-id' => $uriid]); Post\Collection::add($uriid, Post\Collection::FEATURED); + Receiver::removeFromQueue($activity); } /** @@ -563,6 +564,7 @@ class Processor Logger::debug('Remove post from featured collection', ['uri-id' => $uriid]); Post\Collection::remove($uriid, Post\Collection::FEATURED); + Receiver::removeFromQueue($activity); } /** @@ -1464,6 +1466,7 @@ class Processor $condition = ['id' => $cid]; Contact::update($fields, $condition); Logger::info('Accept contact request', ['contact' => $cid, 'user' => $uid]); + Receiver::removeFromQueue($activity); } /** @@ -1497,6 +1500,7 @@ class Processor } else { Logger::info('Rejected contact request', ['contact' => $cid, 'user' => $uid]); } + Receiver::removeFromQueue($activity); } /** @@ -1522,6 +1526,7 @@ class Processor } Item::markForDeletion(['uri' => $activity['object_id'], 'author-id' => $author_id, 'gravity' => GRAVITY_ACTIVITY]); + Receiver::removeFromQueue($activity); } /** @@ -1558,6 +1563,7 @@ class Processor Contact::removeFollower($contact); Logger::info('Undo following request', ['contact' => $cid, 'user' => $uid]); + Receiver::removeFromQueue($activity); } /** diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 11e7151e1e..4d73090016 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -105,7 +105,7 @@ class Receiver if (empty($apcontact)) { Logger::notice('Unable to retrieve AP contact for actor - message is discarded', ['actor' => $actor]); return; - } elseif ($apcontact['type'] == 'Application' && $apcontact['nick'] == 'relay') { + } elseif (APContact::isRelay($apcontact)) { self::processRelayPost($ldactivity, $actor); return; } else { @@ -208,18 +208,18 @@ class Receiver { $type = JsonLD::fetchElement($activity, '@type'); if (!$type) { - Logger::info('Empty type', ['activity' => $activity]); + Logger::info('Empty type', ['activity' => $activity, 'actor' => $actor]); return; } if ($type != 'as:Announce') { - Logger::info('Not an announcement', ['activity' => $activity]); + Logger::info('Not an announcement', ['activity' => $activity, 'actor' => $actor]); return; } $object_id = JsonLD::fetchElement($activity, 'as:object', '@id'); if (empty($object_id)) { - Logger::info('No object id found', ['activity' => $activity]); + Logger::info('No object id found', ['activity' => $activity, 'actor' => $actor]); return; } @@ -234,11 +234,11 @@ class Receiver return; } - Logger::info('Got relayed message id', ['id' => $object_id]); + Logger::info('Got relayed message id', ['id' => $object_id, 'actor' => $actor]); $item_id = Item::searchByLink($object_id); if ($item_id) { - Logger::info('Relayed message already exists', ['id' => $object_id, 'item' => $item_id]); + Logger::info('Relayed message already exists', ['id' => $object_id, 'item' => $item_id, 'actor' => $actor]); return; } @@ -246,7 +246,7 @@ class Receiver $id = Processor::fetchMissingActivity($fetchQueue, $object_id, [], $actor, self::COMPLETION_RELAY); if (empty($id)) { - Logger::notice('Relayed message had not been fetched', ['id' => $object_id]); + Logger::notice('Relayed message had not been fetched', ['id' => $object_id, 'actor' => $actor]); return; } @@ -254,9 +254,9 @@ class Receiver $item_id = Item::searchByLink($object_id); if ($item_id) { - Logger::info('Relayed message had been fetched and stored', ['id' => $object_id, 'item' => $item_id]); + Logger::info('Relayed message had been fetched and stored', ['id' => $object_id, 'item' => $item_id, 'actor' => $actor]); } else { - Logger::notice('Relayed message had not been stored', ['id' => $object_id]); + Logger::notice('Relayed message had not been stored', ['id' => $object_id, 'actor' => $actor]); } } @@ -615,6 +615,7 @@ class Receiver ActivityPub\Processor::postItem($object_data, $item); } elseif (in_array($object_data['object_type'], ['pt:CacheFile'])) { // Unhandled Peertube activity + self::removeFromQueue($object_data); } else { self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); } @@ -706,6 +707,7 @@ class Receiver ActivityPub\Processor::updatePerson($object_data); } elseif (in_array($object_data['object_type'], ['pt:CacheFile'])) { // Unhandled Peertube activity + self::removeFromQueue($object_data); } else { self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); } @@ -791,9 +793,11 @@ class Receiver } elseif (in_array($object_data['object_type'], array_merge(self::ACTIVITY_TYPES, ['as:Announce', 'as:Create', ''])) && empty($object_data['object_object_type'])) { // We cannot detect the target object. So we can ignore it. + self::removeFromQueue($object_data); } elseif (in_array($object_data['object_type'], ['as:Create']) && in_array($object_data['object_object_type'], ['pt:CacheFile'])) { // Unhandled Peertube activity + self::removeFromQueue($object_data); } else { self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); } @@ -804,6 +808,7 @@ class Receiver ActivityPub\Processor::createActivity($fetchQueue, $object_data, Activity::VIEW); } elseif ($object_data['object_type'] == '') { // The object type couldn't be determined. Most likely we don't have it here. We ignore this activity. + self::removeFromQueue($object_data); } else { self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); } @@ -843,6 +848,7 @@ class Receiver private static function storeUnhandledActivity(bool $unknown, string $type, array $object_data, array $activity, string $body = '', int $uid = null, bool $trust_source = false, bool $push = false, array $signer = []) { if (!DI::config()->get('debug', 'ap_log_unknown')) { + self::removeFromQueue($activity); return; } diff --git a/src/Util/XML.php b/src/Util/XML.php index e371ea3cd6..a6e6cd5bab 100644 --- a/src/Util/XML.php +++ b/src/Util/XML.php @@ -141,9 +141,9 @@ class XML * @param string $elementname Name of the XML element of the target * @return void */ - public static function copy(&$source, &$target, string $elementname) + public static function copy(&$source, &$target, $elementname) { - if (is_string($source)) { + if (count($source->children()) == 0) { $target->addChild($elementname, self::escape($source)); } else { $child = $target->addChild($elementname); From 1d13574225ef977fa4f9d9a5e96ba7e81e7d22d3 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 05:16:14 +0000 Subject: [PATCH 04/26] Fetching of missing posts is reworked --- src/Model/Item.php | 4 +- src/Module/Debug/ActivityPubConversion.php | 2 +- src/Protocol/ActivityPub.php | 7 +- src/Protocol/ActivityPub/FetchQueue.php | 57 ------ src/Protocol/ActivityPub/FetchQueueItem.php | 62 ------- src/Protocol/ActivityPub/Processor.php | 69 ++++---- src/Protocol/ActivityPub/Queue.php | 114 ++++++++++++ src/Protocol/ActivityPub/Receiver.php | 182 +++++++++----------- src/Worker/FetchMissingActivity.php | 40 +++++ static/dbstructure.config.php | 2 + 10 files changed, 275 insertions(+), 264 deletions(-) delete mode 100644 src/Protocol/ActivityPub/FetchQueue.php delete mode 100644 src/Protocol/ActivityPub/FetchQueueItem.php create mode 100644 src/Protocol/ActivityPub/Queue.php create mode 100644 src/Worker/FetchMissingActivity.php diff --git a/src/Model/Item.php b/src/Model/Item.php index 01ea942c86..7c3eea9eb1 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -3410,9 +3410,7 @@ class Item return is_numeric($hookData['item_id']) ? $hookData['item_id'] : 0; } - $fetchQueue = new ActivityPub\FetchQueue(); - $fetched_uri = ActivityPub\Processor::fetchMissingActivity($fetchQueue, $uri); - $fetchQueue->process(); + $fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri); if ($fetched_uri) { $item_id = self::searchByLink($fetched_uri, $uid); diff --git a/src/Module/Debug/ActivityPubConversion.php b/src/Module/Debug/ActivityPubConversion.php index 5fa9a8b409..ec7fee3f46 100644 --- a/src/Module/Debug/ActivityPubConversion.php +++ b/src/Module/Debug/ActivityPubConversion.php @@ -123,7 +123,7 @@ class ActivityPubConversion extends BaseModule 'content' => visible_whitespace(var_export($object_data, true)) ]; - $item = ActivityPub\Processor::createItem(new ActivityPub\FetchQueue(), $object_data); + $item = ActivityPub\Processor::createItem($object_data); $results[] = [ 'title' => DI::l10n()->t('Result Item'), diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index 858f837e8f..93204e81d3 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -25,7 +25,6 @@ use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Model\APContact; use Friendica\Model\User; -use Friendica\Protocol\ActivityPub\FetchQueue; use Friendica\Util\HTTPSignature; use Friendica\Util\JsonLD; @@ -224,14 +223,10 @@ class ActivityPub $items = []; } - $fetchQueue = new FetchQueue(); - foreach ($items as $activity) { $ldactivity = JsonLD::compact($activity); - ActivityPub\Receiver::processActivity($fetchQueue, $ldactivity, '', $uid, true); + ActivityPub\Receiver::processActivity($ldactivity, '', $uid, true); } - - $fetchQueue->process(); } /** diff --git a/src/Protocol/ActivityPub/FetchQueue.php b/src/Protocol/ActivityPub/FetchQueue.php deleted file mode 100644 index dfaa338361..0000000000 --- a/src/Protocol/ActivityPub/FetchQueue.php +++ /dev/null @@ -1,57 +0,0 @@ -. - * - */ - -namespace Friendica\Protocol\ActivityPub; - -/** - * This class prevents maximum function nesting errors by flattening recursive calls to Processor::fetchMissingActivity - */ -class FetchQueue -{ - /** @var FetchQueueItem[] */ - protected $queue = []; - - public function push(FetchQueueItem $item) - { - array_push($this->queue, $item); - } - - /** - * Processes missing activities one by one. It is possible that a processing call will add additional missing - * activities, they will be processed in subsequent iterations of the loop. - * - * Since this process is self-contained, it isn't suitable to retrieve the URI of a single activity. - * - * The simplest way to get the URI of the first activity and ensures all the parents are fetched is this way: - * - * $fetchQueue = new ActivityPub\FetchQueue(); - * $fetchedUri = ActivityPub\Processor::fetchMissingActivity($fetchQueue, $activityUri); - * $fetchQueue->process(); - */ - public function process() - { - while (count($this->queue)) { - $fetchQueueItem = array_pop($this->queue); - - call_user_func_array([Processor::class, 'fetchMissingActivity'], array_merge([$this], $fetchQueueItem->toParameters())); - } - } -} diff --git a/src/Protocol/ActivityPub/FetchQueueItem.php b/src/Protocol/ActivityPub/FetchQueueItem.php deleted file mode 100644 index 716c231c99..0000000000 --- a/src/Protocol/ActivityPub/FetchQueueItem.php +++ /dev/null @@ -1,62 +0,0 @@ -. - * - */ - -namespace Friendica\Protocol\ActivityPub; - -class FetchQueueItem -{ - /** @var string */ - private $url; - /** @var array */ - private $child; - /** @var string */ - private $relay_actor; - /** @var int */ - private $completion; - - /** - * This constructor matches the signature of Processor::fetchMissingActivity except for the default $completion value - * - * @param string $url - * @param array $child - * @param string $relay_actor - * @param int $completion - */ - public function __construct(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_AUTO) - { - $this->url = $url; - $this->child = $child; - $this->relay_actor = $relay_actor; - $this->completion = $completion; - } - - /** - * Array meant to be used in call_user_function_array([Processor::class, 'fetchMissingActivity']). Caller needs to - * provide an instance of a FetchQueue that isn't included in these parameters. - * - * @see FetchQueue::process() - * @return array - */ - public function toParameters(): array - { - return [$this->url, $this->child, $this->relay_actor, $this->completion]; - } -} diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index e6ff43f06a..60793f8026 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -189,17 +189,16 @@ class Processor /** * Updates a message * - * @param FetchQueue $fetchQueue * @param array $activity Activity array * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function updateItem(FetchQueue $fetchQueue, array $activity) + public static function updateItem(array $activity) { $item = Post::selectFirst(['uri', 'uri-id', 'thr-parent', 'gravity', 'post-type'], ['uri' => $activity['id']]); if (!DBA::isResult($item)) { Logger::warning('No existing item, item will be created', ['uri' => $activity['id']]); - $item = self::createItem($fetchQueue, $activity); + $item = self::createItem($activity); if (empty($item)) { return; } @@ -223,7 +222,7 @@ class Processor Post\History::add($item['uri-id'], $item); Item::update($item, ['uri' => $activity['id']]); - Receiver::removeFromQueue($activity); + Queue::remove($activity); if ($activity['object_type'] == 'as:Event') { $posts = Post::select(['event-id', 'uid'], ["`uri` = ? AND `event-id` > ?", $activity['id'], 0]); @@ -262,13 +261,12 @@ class Processor /** * Prepares data for a message * - * @param FetchQueue $fetchQueue * @param array $activity Activity array * @return array Internal item * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function createItem(FetchQueue $fetchQueue, array $activity): array + public static function createItem(array $activity): array { $item = []; $item['verb'] = Activity::POST; @@ -283,13 +281,15 @@ class Processor } if (empty($activity['directmessage']) && ($activity['id'] != $activity['reply-to-id']) && !Post::exists(['uri' => $activity['reply-to-id']])) { - Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id']]); - /** - * Instead of calling recursively self::fetchMissingActivity which can hit PHP's default function nesting - * limit of 256 recursive calls, we push the parent activity fetch parameters in this queue. The initial - * caller is responsible for processing the remaining queue once the original activity has been processed. - */ - $fetchQueue->push(new FetchQueueItem($activity['reply-to-id'], $activity)); + $recursion_depth = $activity['recursion-depth'] ?? 0; + Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); + if ($recursion_depth < 10) { + self::fetchMissingActivity($activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); + } else { + Logger::notice('Recursion level is too high, fetching is done by worker.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); + Worker::add(PRIORITY_HIGH, 'FetchMissingActivity', $activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); + return []; + } } $item['diaspora_signed_text'] = $activity['diaspora:comment'] ?? ''; @@ -428,7 +428,7 @@ class Processor Logger::info('Deleting item', ['object' => $activity['object_id'], 'owner' => $owner]); Item::markForDeletion(['uri' => $activity['object_id'], 'owner-id' => $owner]); - Receiver::removeFromQueue($activity); + Queue::remove($activity); } /** @@ -464,15 +464,14 @@ class Processor /** * Prepare the item array for an activity * - * @param FetchQueue $fetchQueue * @param array $activity Activity array * @param string $verb Activity verb * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function createActivity(FetchQueue $fetchQueue, array $activity, string $verb) + public static function createActivity(array $activity, string $verb) { - $item = self::createItem($fetchQueue, $activity); + $item = self::createItem($activity); if (empty($item)) { return; } @@ -546,7 +545,7 @@ class Processor Logger::debug('Add post to featured collection', ['uri-id' => $uriid]); Post\Collection::add($uriid, Post\Collection::FEATURED); - Receiver::removeFromQueue($activity); + Queue::remove($activity); } /** @@ -564,7 +563,7 @@ class Processor Logger::debug('Remove post from featured collection', ['uri-id' => $uriid]); Post\Collection::remove($uriid, Post\Collection::FEATURED); - Receiver::removeFromQueue($activity); + Queue::remove($activity); } /** @@ -652,13 +651,12 @@ class Processor $item['body'] = Item::improveSharedDataInBody($item); } else { if (empty($activity['directmessage']) && ($item['thr-parent'] != $item['uri']) && ($item['gravity'] == GRAVITY_COMMENT)) { - $item_private = !in_array(0, $activity['item_receiver']); $parent = Post::selectFirst(['id', 'uri-id', 'private', 'author-link', 'alias'], ['uri' => $item['thr-parent']]); if (!DBA::isResult($parent)) { Logger::warning('Unknown parent item.', ['uri' => $item['thr-parent']]); return false; } - if ($item_private && ($parent['private'] != Item::PRIVATE)) { + if (($parent['private'] == Item::PRIVATE) && ($parent['private'] != Item::PRIVATE)) { Logger::warning('Item is private but the parent is not. Dropping.', ['item-uri' => $item['uri'], 'thr-parent' => $item['thr-parent']]); return false; } @@ -783,6 +781,7 @@ class Processor } $stored = false; + $success = false; ksort($activity['receiver']); if (!self::isSolicitedMessage($activity, $item)) { @@ -895,7 +894,7 @@ class Processor $item_id = Item::insert($item); if ($item_id) { Logger::info('Item insertion successful', ['user' => $item['uid'], 'item_id' => $item_id]); - Receiver::removeFromQueue($activity); + $success = true; } else { Logger::notice('Item insertion aborted', ['user' => $item['uid']]); } @@ -905,6 +904,11 @@ class Processor } } + if ($success) { + Queue::remove($activity); + Queue::processReplyByUri($item['uri']); + } + // Store send a follow request for every reshare - but only when the item had been stored if ($stored && ($item['private'] != Item::PRIVATE) && ($item['gravity'] == GRAVITY_PARENT) && !empty($item['author-link']) && ($item['author-link'] != $item['owner-link'])) { $author = APContact::getByURL($item['owner-link'], false); @@ -1121,7 +1125,6 @@ class Processor /** * Fetches missing posts * - * @param FetchQueue $fetchQueue * @param string $url message URL * @param array $child activity array with the child of this message * @param string $relay_actor Relay actor @@ -1130,7 +1133,7 @@ class Processor * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function fetchMissingActivity(FetchQueue $fetchQueue, string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL): string + public static function fetchMissingActivity(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL): string { if (!empty($child['receiver'])) { $uid = ActivityPub\Receiver::getFirstUserFromReceivers($child['receiver']); @@ -1140,7 +1143,7 @@ class Processor $object = ActivityPub::fetchContent($url, $uid); if (empty($object)) { - Logger::notice('Activity was not fetchable, aborting.', ['url' => $url]); + Logger::notice('Activity was not fetchable, aborting.', ['url' => $url, 'uid' => $uid]); return ''; } @@ -1192,6 +1195,8 @@ class Processor $ldactivity = JsonLD::compact($activity); + $ldactivity['recursion-depth'] = !empty($child['recursion-depth']) ? $child['recursion-depth'] + 1 : 1; + if (!empty($relay_actor)) { $ldactivity['thread-completion'] = $ldactivity['from-relay'] = Contact::getIdForURL($relay_actor); $ldactivity['completion-mode'] = Receiver::COMPLETION_RELAY; @@ -1211,7 +1216,7 @@ class Processor return ''; } - ActivityPub\Receiver::processActivity($fetchQueue, $ldactivity, json_encode($activity), $uid, true, false, $signer); + ActivityPub\Receiver::processActivity($ldactivity, json_encode($activity), $uid, true, false, $signer); Logger::notice('Activity had been fetched and processed.', ['url' => $url, 'object' => $activity['id']]); @@ -1354,7 +1359,7 @@ class Processor Logger::info('Updating profile', ['object' => $activity['object_id']]); Contact::updateFromProbeByURL($activity['object_id']); - Receiver::removeFromQueue($activity); + Queue::remove($activity); } /** @@ -1383,7 +1388,7 @@ class Processor DBA::close($contacts); Logger::info('Deleted contact', ['object' => $activity['object_id']]); - Receiver::removeFromQueue($activity); + Queue::remove($activity); } /** @@ -1466,7 +1471,7 @@ class Processor $condition = ['id' => $cid]; Contact::update($fields, $condition); Logger::info('Accept contact request', ['contact' => $cid, 'user' => $uid]); - Receiver::removeFromQueue($activity); + Queue::remove($activity); } /** @@ -1500,7 +1505,7 @@ class Processor } else { Logger::info('Rejected contact request', ['contact' => $cid, 'user' => $uid]); } - Receiver::removeFromQueue($activity); + Queue::remove($activity); } /** @@ -1526,7 +1531,7 @@ class Processor } Item::markForDeletion(['uri' => $activity['object_id'], 'author-id' => $author_id, 'gravity' => GRAVITY_ACTIVITY]); - Receiver::removeFromQueue($activity); + Queue::remove($activity); } /** @@ -1563,7 +1568,7 @@ class Processor Contact::removeFollower($contact); Logger::info('Undo following request', ['contact' => $cid, 'user' => $uid]); - Receiver::removeFromQueue($activity); + Queue::remove($activity); } /** diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php new file mode 100644 index 0000000000..0cb84727fc --- /dev/null +++ b/src/Protocol/ActivityPub/Queue.php @@ -0,0 +1,114 @@ +. + * + */ + +namespace Friendica\Protocol\ActivityPub; + +use Friendica\Core\Logger; +use Friendica\Database\Database; +use Friendica\Database\DBA; +use Friendica\Util\DateTimeFormat; + +/** + * This class handles the processing of incoming posts + */ +class Queue +{ + public static function add(array $activity, string $type, int $uid, string $http_signer, bool $push): array + { + $fields = [ + 'activity-id' => $activity['id'], + 'object-id' => $activity['object_id'], + 'type' => $type, + 'object-type' => $activity['object_type'], + 'activity' => json_encode($activity, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT), + 'received' => DateTimeFormat::utcNow(), + 'push' => $push, + ]; + + if (!empty($activity['reply-to-id'])) { + $fields['in-reply-to-id'] = $activity['reply-to-id']; + } + + if (!empty($activity['object_object_type'])) { + $fields['object-object-type'] = $activity['object_object_type']; + } + + if (!empty($http_signer)) { + $fields['signer'] = $http_signer; + } + + DBA::insert('inbox-entry', $fields, Database::INSERT_IGNORE); + + $queue = DBA::selectFirst('inbox-entry', ['id'], ['activity-id' => $activity['id']]); + if (!empty($queue['id'])) { + $activity['entry-id'] = $queue['id']; + DBA::insert('inbox-entry-receiver', ['queue-id' => $queue['id'], 'uid' => $uid], Database::INSERT_IGNORE); + } + return $activity; + } + + public static function remove(array $activity = []) + { + if (empty($activity['entry-id'])) { + return; + } + DBA::delete('inbox-entry', ['id' => $activity['entry-id']]); + //echo "Delete ".$activity['entry-id']."\n"; + + } + + public static function process(int $id) + { + $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]); + if (empty($entry)) { + return; + } + + Logger::debug('Processing queue entry', ['id' => $entry['id'], 'type' => $entry['type'], 'object-type' => $entry['object-type'], 'uri' => $entry['object-id'], 'in-reply-to' => $entry['in-reply-to-id']]); + + $activity = json_decode($entry['activity'], true); + $type = $entry['type']; + $push = $entry['push']; + + $activity['entry-id'] = $entry['id']; + + if (!Receiver::routeActivities($activity, $type, $push)) { + self::remove($activity); + } + } + + public static function processAll() + { + $entries = DBA::select('inbox-entry', ['id', 'type', 'object-type'], [], ['order' => ['id' => true]]); + while ($entry = DBA::fetch($entries)) { + echo $entry['id'] . "\t" . $entry['type'] . "\t" . $entry['object-type'] . "\n"; + self::process($entry['id']); + } + } + + public static function processReplyByUri(string $uri) + { + $entries = DBA::select('inbox-entry', ['id'], ['in-reply-to-id' => $uri], ['order' => ['id' => true]]); + while ($entry = DBA::fetch($entries)) { + self::process($entry['id']); + } + } +} diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 4d73090016..cd583132e6 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -28,7 +28,6 @@ use Friendica\Content\Text\Markdown; use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Core\System; -use Friendica\Database\Database; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\APContact; @@ -37,7 +36,6 @@ use Friendica\Model\Post; use Friendica\Model\User; use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; -use Friendica\Util\DateTimeFormat; use Friendica\Util\HTTPSignature; use Friendica\Util\JsonLD; use Friendica\Util\LDSignature; @@ -155,46 +153,7 @@ class Receiver $trust_source = false; } - $fetchQueue = new FetchQueue(); - self::processActivity($fetchQueue, $ldactivity, $body, $uid, $trust_source, true, $signer, $http_signer); - $fetchQueue->process(); - } - - private static function enqueuePost(array $ldactivity = [], string $type, int $uid, string $http_signer): array - { - $fields = [ - 'activity-id' => $ldactivity['id'], - 'object-id' => $ldactivity['object_id'], - 'type' => $type, - 'object-type' => $ldactivity['object_type'], - 'activity' => json_encode($ldactivity, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT), - 'received' => DateTimeFormat::utcNow(), - ]; - - if (!empty($ldactivity['object_object_type'])) { - $fields['object-object-type'] = $ldactivity['object_object_type']; - } - - if (!empty($http_signer)) { - $fields['signer'] = $http_signer; - } - - DBA::insert('inbox-entry', $fields, Database::INSERT_IGNORE); - - $queue = DBA::selectFirst('inbox-entry', ['id'], ['activity-id' => $ldactivity['id']]); - if (!empty($queue['id'])) { - $ldactivity['entry-id'] = $queue['id']; - DBA::insert('inbox-entry-receiver', ['queue-id' => $queue['id'], 'uid' => $uid], Database::INSERT_IGNORE); - } - return $ldactivity; - } - - public static function removeFromQueue(array $activity = []) - { - if (empty($activity['entry-id'])) { - return; - } - DBA::delete('inbox-entry', ['id' => $activity['entry-id']]); + self::processActivity($ldactivity, $body, $uid, $trust_source, true, $signer, $http_signer); } /** @@ -242,16 +201,12 @@ class Receiver return; } - $fetchQueue = new FetchQueue(); - - $id = Processor::fetchMissingActivity($fetchQueue, $object_id, [], $actor, self::COMPLETION_RELAY); + $id = Processor::fetchMissingActivity($object_id, [], $actor, self::COMPLETION_RELAY); if (empty($id)) { Logger::notice('Relayed message had not been fetched', ['id' => $object_id, 'actor' => $actor]); return; } - $fetchQueue->process(); - $item_id = Item::searchByLink($object_id); if ($item_id) { Logger::info('Relayed message had been fetched and stored', ['id' => $object_id, 'item' => $item_id, 'actor' => $actor]); @@ -518,7 +473,6 @@ class Receiver /** * Processes the activity object * - * @param FetchQueue $fetchQueue * @param array $activity Array with activity data * @param string $body The unprocessed body * @param int|null $uid User ID @@ -528,7 +482,7 @@ class Receiver * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function processActivity(FetchQueue $fetchQueue, array $activity, string $body = '', int $uid = null, bool $trust_source = false, bool $push = false, array $signer = [], string $http_signer = '') + public static function processActivity(array $activity, string $body = '', int $uid = null, bool $trust_source = false, bool $push = false, array $signer = [], string $http_signer = '') { $type = JsonLD::fetchElement($activity, '@type'); if (!$type) { @@ -588,7 +542,7 @@ class Receiver if (!empty($activity['thread-completion'])) { $object_data['thread-completion'] = $activity['thread-completion']; } - + if (!empty($activity['completion-mode'])) { $object_data['completion-mode'] = $activity['completion-mode']; } @@ -597,36 +551,56 @@ class Receiver $object_data['thread-children-type'] = $activity['thread-children-type']; } + if (!empty($activity['recursion-depth'])) { + $object_data['recursion-depth'] = $activity['recursion-depth']; + } + // Internal flag for posts that arrived via relay if (!empty($activity['from-relay'])) { $object_data['from-relay'] = $activity['from-relay']; } - $object_data = self::enqueuePost($object_data, $type, $uid, $http_signer); + if ($type == 'as:Announce') { + $object_data['object_activity'] = $activity; + } + + $object_data = Queue::add($object_data, $type, $uid, $http_signer, $push); if (in_array('as:Question', [$object_data['object_type'] ?? '', $object_data['object_object_type'] ?? ''])) { self::storeUnhandledActivity(false, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); } + if (!self::routeActivities($object_data, $type, $push)) { + self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + //if (!DI::config()->get('debug', 'ap_log_unknown')) { + // Queue::remove($object_data); + //} + } + } + + public static function routeActivities($object_data, $type, $push) + { + $activity = $object_data['object_activity'] ?? []; + switch ($type) { case 'as:Create': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - $item = ActivityPub\Processor::createItem($fetchQueue, $object_data); + $item = ActivityPub\Processor::createItem($object_data); ActivityPub\Processor::postItem($object_data, $item); } elseif (in_array($object_data['object_type'], ['pt:CacheFile'])) { // Unhandled Peertube activity - self::removeFromQueue($object_data); + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; case 'as:Invite': if (in_array($object_data['object_type'], ['as:Event'])) { - $item = ActivityPub\Processor::createItem($fetchQueue, $object_data); + $item = ActivityPub\Processor::createItem($object_data); ActivityPub\Processor::postItem($object_data, $item); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; @@ -637,17 +611,19 @@ class Receiver ActivityPub\Processor::addToFeaturedCollection($object_data); } elseif ($object_data['object_type'] == '') { // The object type couldn't be determined. We don't have it and we can't fetch it. We ignore this activity. + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; case 'as:Announce': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { + $actor = JsonLD::fetchElement($activity, 'as:actor', '@id'); $object_data['thread-completion'] = Contact::getIdForURL($actor); $object_data['completion-mode'] = self::COMPLETION_ANNOUCE; - $item = ActivityPub\Processor::createItem($fetchQueue, $object_data); + $item = ActivityPub\Processor::createItem($object_data); if (empty($item)) { return; } @@ -655,61 +631,64 @@ class Receiver $item['post-reason'] = Item::PR_ANNOUNCEMENT; ActivityPub\Processor::postItem($object_data, $item); - $announce_object_data = self::processObject($activity); - $announce_object_data['name'] = $type; - $announce_object_data['author'] = JsonLD::fetchElement($activity, 'as:actor', '@id'); - $announce_object_data['object_id'] = $object_data['object_id']; - $announce_object_data['object_type'] = $object_data['object_type']; - $announce_object_data['push'] = $push; + if (!empty($activity)) { + $announce_object_data = self::processObject($activity); + $announce_object_data['name'] = $type; + $announce_object_data['author'] = $actor; + $announce_object_data['object_id'] = $object_data['object_id']; + $announce_object_data['object_type'] = $object_data['object_type']; + $announce_object_data['push'] = $push; - if (!empty($body)) { - $announce_object_data['raw'] = $body; - } - - ActivityPub\Processor::createActivity($fetchQueue, $announce_object_data, Activity::ANNOUNCE); + if (!empty($object_data['raw'])) { + $announce_object_data['raw'] = $object_data['raw']; + } + ActivityPub\Processor::createActivity($announce_object_data, Activity::ANNOUNCE); + } else echo "\n***************************\n"; } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; case 'as:Like': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - ActivityPub\Processor::createActivity($fetchQueue, $object_data, Activity::LIKE); + ActivityPub\Processor::createActivity($object_data, Activity::LIKE); } elseif ($object_data['object_type'] == '') { // The object type couldn't be determined. We don't have it and we can't fetch it. We ignore this activity. + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; case 'as:Dislike': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - ActivityPub\Processor::createActivity($fetchQueue, $object_data, Activity::DISLIKE); + ActivityPub\Processor::createActivity($object_data, Activity::DISLIKE); } elseif ($object_data['object_type'] == '') { // The object type couldn't be determined. We don't have it and we can't fetch it. We ignore this activity. + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; case 'as:TentativeAccept': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - ActivityPub\Processor::createActivity($fetchQueue, $object_data, Activity::ATTENDMAYBE); + ActivityPub\Processor::createActivity($object_data, Activity::ATTENDMAYBE); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; case 'as:Update': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - ActivityPub\Processor::updateItem($fetchQueue, $object_data); + ActivityPub\Processor::updateItem($object_data); } elseif (in_array($object_data['object_type'], self::ACCOUNT_TYPES)) { ActivityPub\Processor::updatePerson($object_data); } elseif (in_array($object_data['object_type'], ['pt:CacheFile'])) { // Unhandled Peertube activity - self::removeFromQueue($object_data); + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; @@ -720,8 +699,9 @@ class Receiver ActivityPub\Processor::deletePerson($object_data); } elseif ($object_data['object_type'] == '') { // The object type couldn't be determined. Most likely we don't have it here. We ignore this activity. + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; @@ -729,7 +709,7 @@ class Receiver if (in_array($object_data['object_type'], self::ACCOUNT_TYPES)) { ActivityPub\Processor::blockAccount($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; @@ -738,8 +718,9 @@ class Receiver ActivityPub\Processor::removeFromFeaturedCollection($object_data); } elseif ($object_data['object_type'] == '') { // The object type couldn't be determined. We don't have it and we can't fetch it. We ignore this activity. + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; @@ -748,9 +729,9 @@ class Receiver ActivityPub\Processor::followUser($object_data); } elseif (in_array($object_data['object_type'], self::CONTENT_TYPES)) { $object_data['reply-to-id'] = $object_data['object_id']; - ActivityPub\Processor::createActivity($fetchQueue, $object_data, Activity::FOLLOW); + ActivityPub\Processor::createActivity($object_data, Activity::FOLLOW); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; @@ -758,9 +739,9 @@ class Receiver if ($object_data['object_type'] == 'as:Follow') { ActivityPub\Processor::acceptFollowUser($object_data); } elseif (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - ActivityPub\Processor::createActivity($fetchQueue, $object_data, Activity::ATTEND); + ActivityPub\Processor::createActivity($object_data, Activity::ATTEND); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; @@ -768,9 +749,9 @@ class Receiver if ($object_data['object_type'] == 'as:Follow') { ActivityPub\Processor::rejectFollowUser($object_data); } elseif (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - ActivityPub\Processor::createActivity($fetchQueue, $object_data, Activity::ATTENDNO); + ActivityPub\Processor::createActivity($object_data, Activity::ATTENDNO); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; @@ -793,42 +774,42 @@ class Receiver } elseif (in_array($object_data['object_type'], array_merge(self::ACTIVITY_TYPES, ['as:Announce', 'as:Create', ''])) && empty($object_data['object_object_type'])) { // We cannot detect the target object. So we can ignore it. - self::removeFromQueue($object_data); } elseif (in_array($object_data['object_type'], ['as:Create']) && in_array($object_data['object_object_type'], ['pt:CacheFile'])) { // Unhandled Peertube activity - self::removeFromQueue($object_data); + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; case 'as:View': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - ActivityPub\Processor::createActivity($fetchQueue, $object_data, Activity::VIEW); + ActivityPub\Processor::createActivity($object_data, Activity::VIEW); } elseif ($object_data['object_type'] == '') { // The object type couldn't be determined. Most likely we don't have it here. We ignore this activity. - self::removeFromQueue($object_data); + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; case 'litepub:EmojiReact': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - ActivityPub\Processor::createActivity($fetchQueue, $object_data, Activity::EMOJIREACT); + ActivityPub\Processor::createActivity($object_data, Activity::EMOJIREACT); } elseif ($object_data['object_type'] == '') { // The object type couldn't be determined. We don't have it and we can't fetch it. We ignore this activity. + Queue::remove($object_data); } else { - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); + return false; } break; default: Logger::info('Unknown activity: ' . $type . ' ' . $object_data['object_type']); - self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); - break; + return false; } + return true; } /** @@ -847,11 +828,6 @@ class Receiver */ private static function storeUnhandledActivity(bool $unknown, string $type, array $object_data, array $activity, string $body = '', int $uid = null, bool $trust_source = false, bool $push = false, array $signer = []) { - if (!DI::config()->get('debug', 'ap_log_unknown')) { - self::removeFromQueue($activity); - return; - } - $file = ($unknown ? 'unknown-' : 'unhandled-') . str_replace(':', '-', $type) . '-'; if (!empty($object_data['object_type'])) { diff --git a/src/Worker/FetchMissingActivity.php b/src/Worker/FetchMissingActivity.php new file mode 100644 index 0000000000..5aabe89ced --- /dev/null +++ b/src/Worker/FetchMissingActivity.php @@ -0,0 +1,40 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Core\Logger; +use Friendica\Protocol\ActivityPub; +use Friendica\Protocol\ActivityPub\Receiver; + +class FetchMissingActivity +{ + /** + * Fetch missing activities + * @param string $url Contact URL + */ + public static function execute(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL) + { + Logger::info('Start fetching missing activity', ['url' => $url]); + $result = ActivityPub\Processor::fetchMissingActivity($url, $child, $relay_actor, $completion); + Logger::info('Finished fetching missing activity', ['url' => $url, 'result' => $result]); + } +} diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index c1f1cfcbc1..0db5681871 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -790,12 +790,14 @@ return [ "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"], "activity-id" => ["type" => "varbinary(255)", "comment" => "id of the incoming activity"], "object-id" => ["type" => "varbinary(255)", "comment" => ""], + "in-reply-to-id" => ["type" => "varbinary(255)", "comment" => ""], "type" => ["type" => "varchar(64)", "comment" => "Type of the activity"], "object-type" => ["type" => "varchar(64)", "comment" => "Type of the object activity"], "object-object-type" => ["type" => "varchar(64)", "comment" => "Type of the object's object activity"], "received" => ["type" => "datetime", "comment" => "Receiving date"], "activity" => ["type" => "mediumtext", "comment" => "The JSON activity"], "signer" => ["type" => "varchar(255)", "comment" => ""], + "push" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], ], "indexes" => [ "PRIMARY" => ["id"], From 4a1fb1da12ba357a7695c4dcf9dbae7e67fedd7e Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 05:33:01 +0000 Subject: [PATCH 05/26] Hourly process pending queue entries --- src/Protocol/ActivityPub/Queue.php | 40 +++++++++++++++++++++++++++--- src/Worker/Cron.php | 4 +++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index 0cb84727fc..12bf7e268d 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -31,6 +31,16 @@ use Friendica\Util\DateTimeFormat; */ class Queue { + /** + * Add activity to the queue + * + * @param array $activity + * @param string $type + * @param integer $uid + * @param string $http_signer + * @param boolean $push + * @return array + */ public static function add(array $activity, string $type, int $uid, string $http_signer, bool $push): array { $fields = [ @@ -65,23 +75,33 @@ class Queue return $activity; } + /** + * Remove activity from the queue + * + * @param array $activity + * @return void + */ public static function remove(array $activity = []) { if (empty($activity['entry-id'])) { return; } DBA::delete('inbox-entry', ['id' => $activity['entry-id']]); - //echo "Delete ".$activity['entry-id']."\n"; - } + /** + * Process the activity with the given id + * + * @param integer $id + * @return void + */ public static function process(int $id) { $entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]); if (empty($entry)) { return; } - + Logger::debug('Processing queue entry', ['id' => $entry['id'], 'type' => $entry['type'], 'object-type' => $entry['object-type'], 'uri' => $entry['object-id'], 'in-reply-to' => $entry['in-reply-to-id']]); $activity = json_decode($entry['activity'], true); @@ -95,15 +115,27 @@ class Queue } } + /** + * Process all activities + * + * @return void + */ public static function processAll() { $entries = DBA::select('inbox-entry', ['id', 'type', 'object-type'], [], ['order' => ['id' => true]]); while ($entry = DBA::fetch($entries)) { - echo $entry['id'] . "\t" . $entry['type'] . "\t" . $entry['object-type'] . "\n"; self::process($entry['id']); } + + DBA::delete('inbox-entry', ["`received` < ?", DateTimeFormat::utc('now - 1 days')]); } + /** + * Process all activities that are children of a given post url + * + * @param string $uri + * @return void + */ public static function processReplyByUri(string $uri) { $entries = DBA::select('inbox-entry', ['id'], ['in-reply-to-id' => $uri], ['order' => ['id' => true]]); diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 30efc61dba..ee3fafe4bb 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -27,6 +27,7 @@ use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Tag; +use Friendica\Protocol\ActivityPub\Queue; use Friendica\Protocol\Relay; class Cron @@ -88,6 +89,9 @@ class Cron Tag::setLocalTrendingHashtags(24, 20); Tag::setGlobalTrendingHashtags(24, 20); + // Process pending posts in the queue + Queue::processAll(); + // Search for new contacts in the directory if (DI::config()->get('system', 'synchronize_directory')) { Worker::add(PRIORITY_LOW, 'PullDirectory'); From d4a536137d368fd039e8cdeb44761e074234d8af Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 05:42:53 +0000 Subject: [PATCH 06/26] database.sql updated, standards fixed --- database.sql | 35 ++++++++++++++++++++++++- doc/database.md | 2 ++ doc/database/db_inbox-entry-receiver.md | 30 +++++++++++++++++++++ doc/database/db_inbox-entry.md | 34 ++++++++++++++++++++++++ src/Protocol/ActivityPub/Queue.php | 8 +++--- 5 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 doc/database/db_inbox-entry-receiver.md create mode 100644 doc/database/db_inbox-entry.md diff --git a/database.sql b/database.sql index 01bd84b00b..d8b26cebeb 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2022.09-dev (Giant Rhubarb) --- DB_UPDATE_VERSION 1473 +-- DB_UPDATE_VERSION 1474 -- ------------------------------------------ @@ -724,6 +724,39 @@ CREATE TABLE IF NOT EXISTS `hook` ( UNIQUE INDEX `hook_file_function` (`hook`,`file`,`function`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='addon hook registry'; +-- +-- TABLE inbox-entry +-- +CREATE TABLE IF NOT EXISTS `inbox-entry` ( + `id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID', + `activity-id` varbinary(255) COMMENT 'id of the incoming activity', + `object-id` varbinary(255) COMMENT '', + `in-reply-to-id` varbinary(255) COMMENT '', + `type` varchar(64) COMMENT 'Type of the activity', + `object-type` varchar(64) COMMENT 'Type of the object activity', + `object-object-type` varchar(64) COMMENT 'Type of the object\'s object activity', + `received` datetime COMMENT 'Receiving date', + `activity` mediumtext COMMENT 'The JSON activity', + `signer` varchar(255) COMMENT '', + `push` boolean NOT NULL DEFAULT '0' COMMENT '', + PRIMARY KEY(`id`), + UNIQUE INDEX `activity-id` (`activity-id`), + INDEX `object-id` (`object-id`), + INDEX `received` (`received`) +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Incoming activity'; + +-- +-- TABLE inbox-entry-receiver +-- +CREATE TABLE IF NOT EXISTS `inbox-entry-receiver` ( + `queue-id` int unsigned NOT NULL COMMENT '', + `uid` mediumint unsigned NOT NULL COMMENT 'User id', + PRIMARY KEY(`queue-id`,`uid`), + INDEX `uid` (`uid`), + FOREIGN KEY (`queue-id`) REFERENCES `inbox-entry` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Receiver for the incoming activity'; + -- -- TABLE inbox-status -- diff --git a/doc/database.md b/doc/database.md index fcff590e13..0effb1754a 100644 --- a/doc/database.md +++ b/doc/database.md @@ -31,6 +31,8 @@ Database Tables | [gserver](help/database/db_gserver) | Global servers | | [gserver-tag](help/database/db_gserver-tag) | Tags that the server has subscribed | | [hook](help/database/db_hook) | addon hook registry | +| [inbox-entry](help/database/db_inbox-entry) | Incoming activity | +| [inbox-entry-receiver](help/database/db_inbox-entry-receiver) | Receiver for the incoming activity | | [inbox-status](help/database/db_inbox-status) | Status of ActivityPub inboxes | | [intro](help/database/db_intro) | | | [item-uri](help/database/db_item-uri) | URI and GUID for items | diff --git a/doc/database/db_inbox-entry-receiver.md b/doc/database/db_inbox-entry-receiver.md new file mode 100644 index 0000000000..f905289d27 --- /dev/null +++ b/doc/database/db_inbox-entry-receiver.md @@ -0,0 +1,30 @@ +Table inbox-entry-receiver +=========== + +Receiver for the incoming activity + +Fields +------ + +| Field | Description | Type | Null | Key | Default | Extra | +| -------- | ----------- | ------------------ | ---- | --- | ------- | ----- | +| queue-id | | int unsigned | NO | PRI | NULL | | +| uid | User id | mediumint unsigned | NO | PRI | NULL | | + +Indexes +------------ + +| Name | Fields | +| ------- | ------------- | +| PRIMARY | queue-id, uid | +| uid | uid | + +Foreign Keys +------------ + +| Field | Target Table | Target Field | +|-------|--------------|--------------| +| queue-id | [inbox-entry](help/database/db_inbox-entry) | id | +| uid | [user](help/database/db_user) | uid | + +Return to [database documentation](help/database) diff --git a/doc/database/db_inbox-entry.md b/doc/database/db_inbox-entry.md new file mode 100644 index 0000000000..56dee2c620 --- /dev/null +++ b/doc/database/db_inbox-entry.md @@ -0,0 +1,34 @@ +Table inbox-entry +=========== + +Incoming activity + +Fields +------ + +| Field | Description | Type | Null | Key | Default | Extra | +| ------------------ | ------------------------------------ | -------------- | ---- | --- | ------- | -------------- | +| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment | +| activity-id | id of the incoming activity | varbinary(255) | YES | | NULL | | +| object-id | | varbinary(255) | YES | | NULL | | +| in-reply-to-id | | varbinary(255) | YES | | NULL | | +| type | Type of the activity | varchar(64) | YES | | NULL | | +| object-type | Type of the object activity | varchar(64) | YES | | NULL | | +| object-object-type | Type of the object's object activity | varchar(64) | YES | | NULL | | +| received | Receiving date | datetime | YES | | NULL | | +| activity | The JSON activity | mediumtext | YES | | NULL | | +| signer | | varchar(255) | YES | | NULL | | +| push | | boolean | NO | | 0 | | + +Indexes +------------ + +| Name | Fields | +| ----------- | ------------------- | +| PRIMARY | id | +| activity-id | UNIQUE, activity-id | +| object-id | object-id | +| received | received | + + +Return to [database documentation](help/database) diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index 12bf7e268d..c3605ebb64 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -44,7 +44,7 @@ class Queue public static function add(array $activity, string $type, int $uid, string $http_signer, bool $push): array { $fields = [ - 'activity-id' => $activity['id'], + 'activity-id' => $activity['id'], 'object-id' => $activity['object_id'], 'type' => $type, 'object-type' => $activity['object_type'], @@ -104,9 +104,9 @@ class Queue Logger::debug('Processing queue entry', ['id' => $entry['id'], 'type' => $entry['type'], 'object-type' => $entry['object-type'], 'uri' => $entry['object-id'], 'in-reply-to' => $entry['in-reply-to-id']]); - $activity = json_decode($entry['activity'], true); - $type = $entry['type']; - $push = $entry['push']; + $activity = json_decode($entry['activity'], true); + $type = $entry['type']; + $push = $entry['push']; $activity['entry-id'] = $entry['id']; From c7758331173ec318d5755e943c9abf9eb76a7bcf Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 06:23:55 +0000 Subject: [PATCH 07/26] Use deferred tasks --- src/Protocol/ActivityPub/Processor.php | 11 +++++++++-- src/Protocol/ActivityPub/Queue.php | 10 +++++++++- src/Worker/Cron.php | 4 ++-- src/Worker/FetchMissingActivity.php | 10 +++++++++- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 60793f8026..fe7a4c5967 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -284,9 +284,16 @@ class Processor $recursion_depth = $activity['recursion-depth'] ?? 0; Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); if ($recursion_depth < 10) { - self::fetchMissingActivity($activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); + $result = self::fetchMissingActivity($activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); + $fetch_by_worker = empty($result); } else { - Logger::notice('Recursion level is too high, fetching is done by worker.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); + Logger::notice('Recursion level is too high.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); + $activity['recursion-depth'] = 0; + $fetch_by_worker = true; + } + + if ($fetch_by_worker) { + Logger::notice('Fetching is done by worker.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); Worker::add(PRIORITY_HIGH, 'FetchMissingActivity', $activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); return []; } diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index c3605ebb64..c5cfd522ae 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -126,8 +126,16 @@ class Queue while ($entry = DBA::fetch($entries)) { self::process($entry['id']); } + } - DBA::delete('inbox-entry', ["`received` < ?", DateTimeFormat::utc('now - 1 days')]); + /** + * Process all activities + * + * @return void + */ + public static function clear() + { + DBA::delete('inbox-entry', ["`received` < ?", DateTimeFormat::utc('now - 2 days')]); } /** diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index ee3fafe4bb..68ad218014 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -89,8 +89,8 @@ class Cron Tag::setLocalTrendingHashtags(24, 20); Tag::setGlobalTrendingHashtags(24, 20); - // Process pending posts in the queue - Queue::processAll(); + // Remove old pending posts from the queue + Queue::clear(); // Search for new contacts in the directory if (DI::config()->get('system', 'synchronize_directory')) { diff --git a/src/Worker/FetchMissingActivity.php b/src/Worker/FetchMissingActivity.php index 5aabe89ced..ceab8b7338 100644 --- a/src/Worker/FetchMissingActivity.php +++ b/src/Worker/FetchMissingActivity.php @@ -22,6 +22,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; +use Friendica\Core\Worker; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\ActivityPub\Receiver; @@ -35,6 +36,13 @@ class FetchMissingActivity { Logger::info('Start fetching missing activity', ['url' => $url]); $result = ActivityPub\Processor::fetchMissingActivity($url, $child, $relay_actor, $completion); - Logger::info('Finished fetching missing activity', ['url' => $url, 'result' => $result]); + if ($result) { + Logger::info('Successfully fetched missing activity', ['url' => $url]); + } elseif (!Worker::defer()) { + Logger::info('Activity could not be fetched', ['url' => $url]); + // Possibly we should recursively remove child activities at this point. + } else { + Logger::info('Fetching deferred', ['url' => $url]); + } } } From 28fb0224250b63d1330b0e96993a1533147c6ce2 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 06:26:31 +0000 Subject: [PATCH 08/26] Reset the recursion depth at a different place --- src/Protocol/ActivityPub/Processor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index fe7a4c5967..7973f0c0aa 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -288,12 +288,12 @@ class Processor $fetch_by_worker = empty($result); } else { Logger::notice('Recursion level is too high.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); - $activity['recursion-depth'] = 0; $fetch_by_worker = true; } if ($fetch_by_worker) { Logger::notice('Fetching is done by worker.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); + $activity['recursion-depth'] = 0; Worker::add(PRIORITY_HIGH, 'FetchMissingActivity', $activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); return []; } From 4930e77eb394a023e7d336ebcdad1209f69fea7c Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 07:05:38 +0000 Subject: [PATCH 09/26] We are now setting the corresponding worker id --- database.sql | 5 ++++- doc/database/db_inbox-entry.md | 8 ++++++++ src/Protocol/ActivityPub/Processor.php | 3 ++- src/Protocol/ActivityPub/Queue.php | 17 ++++++++++++++++- static/dbstructure.config.php | 3 ++- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/database.sql b/database.sql index d8b26cebeb..a16bf24cf3 100644 --- a/database.sql +++ b/database.sql @@ -739,10 +739,13 @@ CREATE TABLE IF NOT EXISTS `inbox-entry` ( `activity` mediumtext COMMENT 'The JSON activity', `signer` varchar(255) COMMENT '', `push` boolean NOT NULL DEFAULT '0' COMMENT '', + `wid` int unsigned COMMENT 'Workerqueue id', PRIMARY KEY(`id`), UNIQUE INDEX `activity-id` (`activity-id`), INDEX `object-id` (`object-id`), - INDEX `received` (`received`) + INDEX `received` (`received`), + INDEX `wid` (`wid`), + FOREIGN KEY (`wid`) REFERENCES `workerqueue` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Incoming activity'; -- diff --git a/doc/database/db_inbox-entry.md b/doc/database/db_inbox-entry.md index 56dee2c620..91126919e9 100644 --- a/doc/database/db_inbox-entry.md +++ b/doc/database/db_inbox-entry.md @@ -19,6 +19,7 @@ Fields | activity | The JSON activity | mediumtext | YES | | NULL | | | signer | | varchar(255) | YES | | NULL | | | push | | boolean | NO | | 0 | | +| wid | Workerqueue id | int unsigned | YES | | NULL | | Indexes ------------ @@ -29,6 +30,13 @@ Indexes | activity-id | UNIQUE, activity-id | | object-id | object-id | | received | received | +| wid | wid | +Foreign Keys +------------ + +| Field | Target Table | Target Field | +|-------|--------------|--------------| +| wid | [workerqueue](help/database/db_workerqueue) | id | Return to [database documentation](help/database) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 7973f0c0aa..03c68eac00 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -294,7 +294,8 @@ class Processor if ($fetch_by_worker) { Logger::notice('Fetching is done by worker.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); $activity['recursion-depth'] = 0; - Worker::add(PRIORITY_HIGH, 'FetchMissingActivity', $activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); + $wid = Worker::add(PRIORITY_HIGH, 'FetchMissingActivity', $activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); + Queue::setWorkerId($activity, $wid); return []; } } diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index c5cfd522ae..f8f5666d13 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -89,6 +89,21 @@ class Queue DBA::delete('inbox-entry', ['id' => $activity['entry-id']]); } + /** + * Set the worker id for the queue entry + * + * @param array $activity + * @param int $wid + * @return void + */ + public static function setWorkerId(array $activity, int $wid) + { + if (empty($activity['entry-id']) || empty($wid)) { + return; + } + DBA::update('inbox-entry', ['wid' => $wid], ['id' => $activity['entry-id']]); + } + /** * Process the activity with the given id * @@ -129,7 +144,7 @@ class Queue } /** - * Process all activities + * Clear old activities * * @return void */ diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 0db5681871..abaddc8d0a 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -798,12 +798,13 @@ return [ "activity" => ["type" => "mediumtext", "comment" => "The JSON activity"], "signer" => ["type" => "varchar(255)", "comment" => ""], "push" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], - ], + "wid" => ["type" => "int unsigned", "foreign" => ["workerqueue" => "id"], "comment" => "Workerqueue id"], ], "indexes" => [ "PRIMARY" => ["id"], "activity-id" => ["UNIQUE", "activity-id"], "object-id" => ["object-id"], "received" => ["received"], + "wid" => ["wid"], ] ], "inbox-entry-receiver" => [ From 2e39cebcb9e07ee7d3b3a82427cf8b81827f75fd Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 07:08:17 +0000 Subject: [PATCH 10/26] Process the replies --- src/Worker/FetchMissingActivity.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Worker/FetchMissingActivity.php b/src/Worker/FetchMissingActivity.php index ceab8b7338..116f098db0 100644 --- a/src/Worker/FetchMissingActivity.php +++ b/src/Worker/FetchMissingActivity.php @@ -24,6 +24,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Core\Worker; use Friendica\Protocol\ActivityPub; +use Friendica\Protocol\ActivityPub\Queue; use Friendica\Protocol\ActivityPub\Receiver; class FetchMissingActivity @@ -38,6 +39,7 @@ class FetchMissingActivity $result = ActivityPub\Processor::fetchMissingActivity($url, $child, $relay_actor, $completion); if ($result) { Logger::info('Successfully fetched missing activity', ['url' => $url]); + Queue::processReplyByUri($url); } elseif (!Worker::defer()) { Logger::info('Activity could not be fetched', ['url' => $url]); // Possibly we should recursively remove child activities at this point. From 3966bff8a186a9d5d0d983bd30e1fbb6bc6376c6 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 08:14:47 +0000 Subject: [PATCH 11/26] Fix standards --- src/Worker/FetchMissingActivity.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Worker/FetchMissingActivity.php b/src/Worker/FetchMissingActivity.php index 116f098db0..1a2818c3cc 100644 --- a/src/Worker/FetchMissingActivity.php +++ b/src/Worker/FetchMissingActivity.php @@ -42,7 +42,6 @@ class FetchMissingActivity Queue::processReplyByUri($url); } elseif (!Worker::defer()) { Logger::info('Activity could not be fetched', ['url' => $url]); - // Possibly we should recursively remove child activities at this point. } else { Logger::info('Fetching deferred', ['url' => $url]); } From 99f7270e18e3e9cff3f8222329c2718287ff4361 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 08:55:45 +0000 Subject: [PATCH 12/26] Improved workerqueue handling --- src/Protocol/ActivityPub/Processor.php | 5 +++++ src/Protocol/ActivityPub/Queue.php | 19 +++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 03c68eac00..b896dfa147 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -281,6 +281,11 @@ class Processor } if (empty($activity['directmessage']) && ($activity['id'] != $activity['reply-to-id']) && !Post::exists(['uri' => $activity['reply-to-id']])) { + if (Queue::hasWorker($activity)) { + Logger::notice('There is already a worker task to dfetch the post.', ['parent' => $activity['reply-to-id']]); + return []; + } + $recursion_depth = $activity['recursion-depth'] ?? 0; Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); if ($recursion_depth < 10) { diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index f8f5666d13..d5a308b109 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -104,6 +104,20 @@ class Queue DBA::update('inbox-entry', ['wid' => $wid], ['id' => $activity['entry-id']]); } + /** + * Check if there is an assigned worker task + * + * @param array $activity + * @return bool + */ + public static function hasWorker(array $activity = []): bool + { + if (empty($activity['worker-id'])) { + return false; + } + return DBA::exists('workerqueue', ['id' => $activity['worker-id'], 'done' => false]); + } + /** * Process the activity with the given id * @@ -123,7 +137,8 @@ class Queue $type = $entry['type']; $push = $entry['push']; - $activity['entry-id'] = $entry['id']; + $activity['entry-id'] = $entry['id']; + $activity['worker-id'] = $entry['wid']; if (!Receiver::routeActivities($activity, $type, $push)) { self::remove($activity); @@ -150,7 +165,7 @@ class Queue */ public static function clear() { - DBA::delete('inbox-entry', ["`received` < ?", DateTimeFormat::utc('now - 2 days')]); + DBA::delete('inbox-entry', ["`wid` IS NULL AND `received` < ?", DateTimeFormat::utc('now - 4 hours')]); } /** From 4c5e9f206ebd24cd7188b897d9c01a08ef081ef1 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 11:47:23 +0000 Subject: [PATCH 13/26] Remove ignored undo --- src/Protocol/ActivityPub/Receiver.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index cd583132e6..5eae464e05 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -774,6 +774,7 @@ class Receiver } elseif (in_array($object_data['object_type'], array_merge(self::ACTIVITY_TYPES, ['as:Announce', 'as:Create', ''])) && empty($object_data['object_object_type'])) { // We cannot detect the target object. So we can ignore it. + Queue::remove($object_data); } elseif (in_array($object_data['object_type'], ['as:Create']) && in_array($object_data['object_object_type'], ['pt:CacheFile'])) { // Unhandled Peertube activity From 54cebf5a886e591ab591d5dcaf3bebab07b299d9 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 21 Jul 2022 12:42:26 +0000 Subject: [PATCH 14/26] Remove test code --- src/Protocol/ActivityPub/Receiver.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 5eae464e05..045248e629 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -542,7 +542,7 @@ class Receiver if (!empty($activity['thread-completion'])) { $object_data['thread-completion'] = $activity['thread-completion']; } - + if (!empty($activity['completion-mode'])) { $object_data['completion-mode'] = $activity['completion-mode']; } @@ -572,9 +572,7 @@ class Receiver if (!self::routeActivities($object_data, $type, $push)) { self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); - //if (!DI::config()->get('debug', 'ap_log_unknown')) { - // Queue::remove($object_data); - //} + Queue::remove($object_data); } } @@ -643,7 +641,7 @@ class Receiver $announce_object_data['raw'] = $object_data['raw']; } ActivityPub\Processor::createActivity($announce_object_data, Activity::ANNOUNCE); - } else echo "\n***************************\n"; + } } else { return false; } @@ -715,7 +713,7 @@ class Receiver case 'as:Remove': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - ActivityPub\Processor::removeFromFeaturedCollection($object_data); + ActivityPub\Processor::removeFromFeaturedCollection($object_data); } elseif ($object_data['object_type'] == '') { // The object type couldn't be determined. We don't have it and we can't fetch it. We ignore this activity. Queue::remove($object_data); @@ -805,7 +803,7 @@ class Receiver return false; } break; - + default: Logger::info('Unknown activity: ' . $type . ' ' . $object_data['object_type']); return false; @@ -829,8 +827,12 @@ class Receiver */ private static function storeUnhandledActivity(bool $unknown, string $type, array $object_data, array $activity, string $body = '', int $uid = null, bool $trust_source = false, bool $push = false, array $signer = []) { + if (!DI::config()->get('debug', 'ap_log_unknown')) { + return; + } + $file = ($unknown ? 'unknown-' : 'unhandled-') . str_replace(':', '-', $type) . '-'; - + if (!empty($object_data['object_type'])) { $file .= str_replace(':', '-', $object_data['object_type']) . '-'; } From 7eb410bed722a0df72b21231275cab5ce3db298f Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 23 Jul 2022 06:52:43 +0000 Subject: [PATCH 15/26] Conversation/context is now stored --- database.sql | 18 ++++++++ doc/database/db_post-thread-user.md | 53 ++++++++++++----------- doc/database/db_post-thread.md | 41 +++++++++--------- src/Model/Item.php | 28 +++++++++--- src/Protocol/ActivityPub/Processor.php | 54 ++++++++++++++++++------ src/Protocol/ActivityPub/Queue.php | 8 ++++ src/Protocol/ActivityPub/Receiver.php | 4 ++ src/Protocol/ActivityPub/Transmitter.php | 22 +--------- src/Worker/ExpirePosts.php | 1 + static/dbstructure.config.php | 4 ++ static/dbview.config.php | 12 ++++++ 11 files changed, 161 insertions(+), 84 deletions(-) diff --git a/database.sql b/database.sql index a16bf24cf3..eb97ec3074 100644 --- a/database.sql +++ b/database.sql @@ -1323,6 +1323,7 @@ CREATE TABLE IF NOT EXISTS `post-tag` ( -- CREATE TABLE IF NOT EXISTS `post-thread` ( `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `conversation-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the conversation uri', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item author', `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', @@ -1332,12 +1333,14 @@ CREATE TABLE IF NOT EXISTS `post-thread` ( `changed` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date that something in the conversation changed, indicating clients should fetch the conversation again', `commented` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', PRIMARY KEY(`uri-id`), + INDEX `conversation-id` (`conversation-id`), INDEX `owner-id` (`owner-id`), INDEX `author-id` (`author-id`), INDEX `causer-id` (`causer-id`), INDEX `received` (`received`), INDEX `commented` (`commented`), FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`conversation-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`owner-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, FOREIGN KEY (`author-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, FOREIGN KEY (`causer-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT @@ -1416,6 +1419,7 @@ CREATE TABLE IF NOT EXISTS `post-user` ( -- CREATE TABLE IF NOT EXISTS `post-thread-user` ( `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `conversation-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the conversation uri', `owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner', `author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item author', `causer-id` int unsigned COMMENT 'Link to the contact table with uid=0 of the contact that caused the item creation', @@ -1440,6 +1444,7 @@ CREATE TABLE IF NOT EXISTS `post-thread-user` ( `post-user-id` int unsigned COMMENT 'Id of the post-user table', PRIMARY KEY(`uid`,`uri-id`), INDEX `uri-id` (`uri-id`), + INDEX `conversation-id` (`conversation-id`), INDEX `owner-id` (`owner-id`), INDEX `author-id` (`author-id`), INDEX `causer-id` (`causer-id`), @@ -1454,6 +1459,7 @@ CREATE TABLE IF NOT EXISTS `post-thread-user` ( INDEX `uid_starred` (`uid`,`starred`), INDEX `uid_mention` (`uid`,`mention`), FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`conversation-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`owner-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, FOREIGN KEY (`author-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, FOREIGN KEY (`causer-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT, @@ -1745,6 +1751,8 @@ CREATE VIEW `post-user-view` AS SELECT `post-user`.`parent-uri-id` AS `parent-uri-id`, `thr-parent-item-uri`.`uri` AS `thr-parent`, `post-user`.`thr-parent-id` AS `thr-parent-id`, + `conversation-item-uri`.`uri` AS `conversation`, + `post-thread-user`.`conversation-id` AS `conversation-id`, `item-uri`.`guid` AS `guid`, `post-user`.`wall` AS `wall`, `post-user`.`gravity` AS `gravity`, @@ -1893,6 +1901,7 @@ CREATE VIEW `post-user-view` AS SELECT LEFT JOIN `item-uri` ON `item-uri`.`id` = `post-user`.`uri-id` LEFT JOIN `item-uri` AS `thr-parent-item-uri` ON `thr-parent-item-uri`.`id` = `post-user`.`thr-parent-id` LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-user`.`parent-uri-id` + LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` LEFT JOIN `verb` ON `verb`.`id` = `post-user`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` @@ -1919,6 +1928,8 @@ CREATE VIEW `post-thread-user-view` AS SELECT `post-user`.`parent-uri-id` AS `parent-uri-id`, `thr-parent-item-uri`.`uri` AS `thr-parent`, `post-user`.`thr-parent-id` AS `thr-parent-id`, + `conversation-item-uri`.`uri` AS `conversation`, + `post-thread-user`.`conversation-id` AS `conversation-id`, `item-uri`.`guid` AS `guid`, `post-thread-user`.`wall` AS `wall`, `post-user`.`gravity` AS `gravity`, @@ -2066,6 +2077,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT LEFT JOIN `item-uri` ON `item-uri`.`id` = `post-thread-user`.`uri-id` LEFT JOIN `item-uri` AS `thr-parent-item-uri` ON `thr-parent-item-uri`.`id` = `post-user`.`thr-parent-id` LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-user`.`parent-uri-id` + LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` LEFT JOIN `verb` ON `verb`.`id` = `post-user`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` @@ -2088,6 +2100,8 @@ CREATE VIEW `post-view` AS SELECT `post`.`parent-uri-id` AS `parent-uri-id`, `thr-parent-item-uri`.`uri` AS `thr-parent`, `post`.`thr-parent-id` AS `thr-parent-id`, + `conversation-item-uri`.`uri` AS `conversation`, + `post-thread`.`conversation-id` AS `conversation-id`, `item-uri`.`guid` AS `guid`, `post`.`gravity` AS `gravity`, `external-item-uri`.`uri` AS `extid`, @@ -2205,6 +2219,7 @@ CREATE VIEW `post-view` AS SELECT LEFT JOIN `item-uri` ON `item-uri`.`id` = `post`.`uri-id` LEFT JOIN `item-uri` AS `thr-parent-item-uri` ON `thr-parent-item-uri`.`id` = `post`.`thr-parent-id` LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post`.`parent-uri-id` + LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post`.`external-id` LEFT JOIN `verb` ON `verb`.`id` = `post`.`vid` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post`.`uri-id` @@ -2224,6 +2239,8 @@ CREATE VIEW `post-thread-view` AS SELECT `post`.`parent-uri-id` AS `parent-uri-id`, `thr-parent-item-uri`.`uri` AS `thr-parent`, `post`.`thr-parent-id` AS `thr-parent-id`, + `conversation-item-uri`.`uri` AS `conversation`, + `post-thread`.`conversation-id` AS `conversation-id`, `item-uri`.`guid` AS `guid`, `post`.`gravity` AS `gravity`, `external-item-uri`.`uri` AS `extid`, @@ -2341,6 +2358,7 @@ CREATE VIEW `post-thread-view` AS SELECT LEFT JOIN `item-uri` ON `item-uri`.`id` = `post-thread`.`uri-id` LEFT JOIN `item-uri` AS `thr-parent-item-uri` ON `thr-parent-item-uri`.`id` = `post`.`thr-parent-id` LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post`.`parent-uri-id` + LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post`.`external-id` LEFT JOIN `verb` ON `verb`.`id` = `post`.`vid` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-thread`.`uri-id` diff --git a/doc/database/db_post-thread-user.md b/doc/database/db_post-thread-user.md index 7c6eac868c..9026370491 100644 --- a/doc/database/db_post-thread-user.md +++ b/doc/database/db_post-thread-user.md @@ -6,31 +6,32 @@ Thread related data per user Fields ------ -| Field | Description | Type | Null | Key | Default | Extra | -| ------------ | ------------------------------------------------------------------------------------------------------- | ------------------ | ---- | --- | ------------------- | ----- | -| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | -| owner-id | Item owner | int unsigned | NO | | 0 | | -| author-id | Item author | int unsigned | NO | | 0 | | -| causer-id | Link to the contact table with uid=0 of the contact that caused the item creation | int unsigned | YES | | NULL | | -| network | | char(4) | NO | | | | -| created | | datetime | NO | | 0001-01-01 00:00:00 | | -| received | | datetime | NO | | 0001-01-01 00:00:00 | | -| changed | Date that something in the conversation changed, indicating clients should fetch the conversation again | datetime | NO | | 0001-01-01 00:00:00 | | -| commented | | datetime | NO | | 0001-01-01 00:00:00 | | -| uid | Owner id which owns this copy of the item | mediumint unsigned | NO | PRI | 0 | | -| pinned | deprecated | boolean | NO | | 0 | | -| starred | | boolean | NO | | 0 | | -| ignored | Ignore updates for this thread | boolean | NO | | 0 | | -| wall | This item was posted to the wall of uid | boolean | NO | | 0 | | -| mention | | boolean | NO | | 0 | | -| pubmail | | boolean | NO | | 0 | | -| forum_mode | Deprecated | tinyint unsigned | NO | | 0 | | -| contact-id | contact.id | int unsigned | NO | | 0 | | -| unseen | post has not been seen | boolean | NO | | 1 | | -| hidden | Marker to hide the post from the user | boolean | NO | | 0 | | -| origin | item originated at this site | boolean | NO | | 0 | | -| psid | ID of the permission set of this post | int unsigned | YES | | NULL | | -| post-user-id | Id of the post-user table | int unsigned | YES | | NULL | | +| Field | Description | Type | Null | Key | Default | Extra | +| --------------- | ------------------------------------------------------------------------------------------------------- | ------------------ | ---- | --- | ------------------- | ----- | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | +| conversation-id | Id of the item-uri table entry that contains the conversation uri | int unsigned | YES | | NULL | | +| owner-id | Item owner | int unsigned | NO | | 0 | | +| author-id | Item author | int unsigned | NO | | 0 | | +| causer-id | Link to the contact table with uid=0 of the contact that caused the item creation | int unsigned | YES | | NULL | | +| network | | char(4) | NO | | | | +| created | | datetime | NO | | 0001-01-01 00:00:00 | | +| received | | datetime | NO | | 0001-01-01 00:00:00 | | +| changed | Date that something in the conversation changed, indicating clients should fetch the conversation again | datetime | NO | | 0001-01-01 00:00:00 | | +| commented | | datetime | NO | | 0001-01-01 00:00:00 | | +| uid | Owner id which owns this copy of the item | mediumint unsigned | NO | PRI | 0 | | +| pinned | deprecated | boolean | NO | | 0 | | +| starred | | boolean | NO | | 0 | | +| ignored | Ignore updates for this thread | boolean | NO | | 0 | | +| wall | This item was posted to the wall of uid | boolean | NO | | 0 | | +| mention | | boolean | NO | | 0 | | +| pubmail | | boolean | NO | | 0 | | +| forum_mode | Deprecated | tinyint unsigned | NO | | 0 | | +| contact-id | contact.id | int unsigned | NO | | 0 | | +| unseen | post has not been seen | boolean | NO | | 1 | | +| hidden | Marker to hide the post from the user | boolean | NO | | 0 | | +| origin | item originated at this site | boolean | NO | | 0 | | +| psid | ID of the permission set of this post | int unsigned | YES | | NULL | | +| post-user-id | Id of the post-user table | int unsigned | YES | | NULL | | Indexes ------------ @@ -39,6 +40,7 @@ Indexes | ----------------- | ------------------- | | PRIMARY | uid, uri-id | | uri-id | uri-id | +| conversation-id | conversation-id | | owner-id | owner-id | | author-id | author-id | | causer-id | causer-id | @@ -59,6 +61,7 @@ Foreign Keys | Field | Target Table | Target Field | |-------|--------------|--------------| | uri-id | [item-uri](help/database/db_item-uri) | id | +| conversation-id | [item-uri](help/database/db_item-uri) | id | | owner-id | [contact](help/database/db_contact) | id | | author-id | [contact](help/database/db_contact) | id | | causer-id | [contact](help/database/db_contact) | id | diff --git a/doc/database/db_post-thread.md b/doc/database/db_post-thread.md index 96fd0d4275..b90fb2ab55 100644 --- a/doc/database/db_post-thread.md +++ b/doc/database/db_post-thread.md @@ -6,29 +6,31 @@ Thread related data Fields ------ -| Field | Description | Type | Null | Key | Default | Extra | -| --------- | ------------------------------------------------------------------------------------------------------- | ------------ | ---- | --- | ------------------- | ----- | -| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | -| owner-id | Item owner | int unsigned | NO | | 0 | | -| author-id | Item author | int unsigned | NO | | 0 | | -| causer-id | Link to the contact table with uid=0 of the contact that caused the item creation | int unsigned | YES | | NULL | | -| network | | char(4) | NO | | | | -| created | | datetime | NO | | 0001-01-01 00:00:00 | | -| received | | datetime | NO | | 0001-01-01 00:00:00 | | -| changed | Date that something in the conversation changed, indicating clients should fetch the conversation again | datetime | NO | | 0001-01-01 00:00:00 | | -| commented | | datetime | NO | | 0001-01-01 00:00:00 | | +| Field | Description | Type | Null | Key | Default | Extra | +| --------------- | ------------------------------------------------------------------------------------------------------- | ------------ | ---- | --- | ------------------- | ----- | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | +| conversation-id | Id of the item-uri table entry that contains the conversation uri | int unsigned | YES | | NULL | | +| owner-id | Item owner | int unsigned | NO | | 0 | | +| author-id | Item author | int unsigned | NO | | 0 | | +| causer-id | Link to the contact table with uid=0 of the contact that caused the item creation | int unsigned | YES | | NULL | | +| network | | char(4) | NO | | | | +| created | | datetime | NO | | 0001-01-01 00:00:00 | | +| received | | datetime | NO | | 0001-01-01 00:00:00 | | +| changed | Date that something in the conversation changed, indicating clients should fetch the conversation again | datetime | NO | | 0001-01-01 00:00:00 | | +| commented | | datetime | NO | | 0001-01-01 00:00:00 | | Indexes ------------ -| Name | Fields | -| --------- | --------- | -| PRIMARY | uri-id | -| owner-id | owner-id | -| author-id | author-id | -| causer-id | causer-id | -| received | received | -| commented | commented | +| Name | Fields | +| --------------- | --------------- | +| PRIMARY | uri-id | +| conversation-id | conversation-id | +| owner-id | owner-id | +| author-id | author-id | +| causer-id | causer-id | +| received | received | +| commented | commented | Foreign Keys ------------ @@ -36,6 +38,7 @@ Foreign Keys | Field | Target Table | Target Field | |-------|--------------|--------------| | uri-id | [item-uri](help/database/db_item-uri) | id | +| conversation-id | [item-uri](help/database/db_item-uri) | id | | owner-id | [contact](help/database/db_contact) | id | | author-id | [contact](help/database/db_contact) | id | | causer-id | [contact](help/database/db_contact) | id | diff --git a/src/Model/Item.php b/src/Model/Item.php index 7c3eea9eb1..4142953568 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -83,7 +83,7 @@ class Item // Field list that is used to display the items const DISPLAY_FIELDLIST = [ 'uid', 'id', 'parent', 'guid', 'network', 'gravity', - 'uri-id', 'uri', 'thr-parent-id', 'thr-parent', 'parent-uri-id', 'parent-uri', + 'uri-id', 'uri', 'thr-parent-id', 'thr-parent', 'parent-uri-id', 'parent-uri', 'conversation', 'commented', 'created', 'edited', 'received', 'verb', 'object-type', 'postopts', 'plink', 'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object', @@ -103,7 +103,7 @@ class Item // Field list that is used to deliver items via the protocols const DELIVER_FIELDLIST = ['uid', 'id', 'parent', 'uri-id', 'uri', 'thr-parent', 'parent-uri', 'guid', - 'parent-guid', 'received', 'created', 'edited', 'verb', 'object-type', 'object', 'target', + 'parent-guid', 'conversation', 'received', 'created', 'edited', 'verb', 'object-type', 'object', 'target', 'private', 'title', 'body', 'raw-body', 'location', 'coord', 'app', 'inform', 'deleted', 'extid', 'post-type', 'post-reason', 'gravity', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', @@ -116,7 +116,7 @@ class Item // All fields in the item table const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent', - 'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'vid', + 'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'conversation', 'vid', 'contact-id', 'wall', 'gravity', 'extid', 'psid', 'created', 'edited', 'commented', 'received', 'changed', 'verb', 'postopts', 'plink', 'resource-id', 'event-id', 'inform', @@ -687,6 +687,11 @@ class Item $params = ['order' => ['id' => false]]; $parent = Post::selectFirst($fields, $condition, $params); + if (!DBA::isResult($parent) && ($item['thr-parent-id'] != $item['parent-uri-id'])) { + $condition = ['uri-id' => $item['parent-uri-id'], 'uid' => $item['uid']]; + $parent = Post::selectFirst($fields, $condition, $params); + } + if (!DBA::isResult($parent) && $item['origin']) { $stored = Item::storeForUserByUriId($item['thr-parent-id'], $item['uid']); Logger::info('Stored thread parent item for user', ['uri-id' => $item['thr-parent-id'], 'uid' => $item['uid'], 'stored' => $stored]); @@ -787,10 +792,13 @@ class Item // Backward compatibility: parent-uri used to be the direct parent uri. // If it is provided without a thr-parent, it probably is the old behavior. - $item['thr-parent'] = trim($item['thr-parent'] ?? $item['parent-uri'] ?? $item['uri']); - $item['parent-uri'] = $item['thr-parent']; + if (empty($item['thr-parent']) || empty($item['parent-uri'])) { + $item['thr-parent'] = trim($item['thr-parent'] ?? $item['parent-uri'] ?? $item['uri']); + $item['parent-uri'] = $item['thr-parent']; + } - $item['thr-parent-id'] = $item['parent-uri-id'] = ItemURI::getIdByURI($item['thr-parent']); + $item['thr-parent-id'] = ItemURI::getIdByURI($item['thr-parent']); + $item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']); // Store conversation data $item = Conversation::insert($item); @@ -966,11 +974,19 @@ class Item } else { $parent_id = 0; $parent_origin = $item['origin']; + + if ($item['wall'] && empty($item['conversation'])) { + $item['conversation'] = $item['parent-uri'] . '#context'; + } } $item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']); $item['thr-parent-id'] = ItemURI::getIdByURI($item['thr-parent']); + if (!empty($item['conversation']) && empty($item['conversation-id'])) { + $item['conversation-id'] = ItemURI::getIdByURI($item['conversation']); + } + // Is this item available in the global items (with uid=0)? if ($item['uid'] == 0) { $item['global'] = true; diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index b896dfa147..28192d35ba 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -280,15 +280,26 @@ class Processor $item['object-type'] = Activity\ObjectType::COMMENT; } - if (empty($activity['directmessage']) && ($activity['id'] != $activity['reply-to-id']) && !Post::exists(['uri' => $activity['reply-to-id']])) { - if (Queue::hasWorker($activity)) { - Logger::notice('There is already a worker task to dfetch the post.', ['parent' => $activity['reply-to-id']]); - return []; - } + if (!empty($activity['context'])) { + $item['conversation'] = $activity['context']; + } elseif(!empty($activity['conversation'])) { + $item['conversation'] = $activity['conversation']; + } + if (!empty($item['conversation'])) { + $conversation = Post::selectFirstThread(['uri'], ['conversation' => $item['conversation']]); + if (!empty($conversation)) { + Logger::debug('Got conversation', ['conversation' => $item['conversation'], 'parent' => $conversation]); + $item['parent-uri'] = $conversation['uri']; + } + } else { + $conversation = []; + } + + if (empty($activity['directmessage']) && ($activity['id'] != $activity['reply-to-id']) && !Post::exists(['uri' => $activity['reply-to-id']])) { $recursion_depth = $activity['recursion-depth'] ?? 0; Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); - if ($recursion_depth < 10) { + if ($recursion_depth < 10000) { $result = self::fetchMissingActivity($activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); $fetch_by_worker = empty($result); } else { @@ -296,19 +307,32 @@ class Processor $fetch_by_worker = true; } + if ($fetch_by_worker && Queue::hasWorker($activity)) { + Logger::notice('There is already a worker task to fetch the post.', ['id' => $activity['id'], 'parent' => $activity['reply-to-id']]); + $fetch_by_worker = false; + if (!empty($conversation)) { + return []; + } + } + if ($fetch_by_worker) { Logger::notice('Fetching is done by worker.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); $activity['recursion-depth'] = 0; $wid = Worker::add(PRIORITY_HIGH, 'FetchMissingActivity', $activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); Queue::setWorkerId($activity, $wid); - return []; + if (!empty($conversation)) { + return []; + } + } elseif (!empty($result)) { + if (($item['thr-parent'] != $result) && Post::exists(['uri' => $result])) { + $item['thr-parent'] = $result; + } } } $item['diaspora_signed_text'] = $activity['diaspora:comment'] ?? ''; - /// @todo What to do with $activity['context']? - if (empty($activity['directmessage']) && ($item['gravity'] != GRAVITY_PARENT) && !Post::exists(['uri' => $item['thr-parent']])) { + if (empty($conversation) && empty($activity['directmessage']) && ($item['gravity'] != GRAVITY_PARENT) && !Post::exists(['uri' => $item['thr-parent']])) { Logger::info('Parent not found, message will be discarded.', ['thr-parent' => $item['thr-parent']]); return []; } @@ -484,6 +508,7 @@ class Processor */ public static function createActivity(array $activity, string $verb) { + $activity['reply-to-id'] = $activity['object_id']; $item = self::createItem($activity); if (empty($item)) { return; @@ -663,10 +688,11 @@ class Processor $item['raw-body'] = $content; $item['body'] = Item::improveSharedDataInBody($item); } else { - if (empty($activity['directmessage']) && ($item['thr-parent'] != $item['uri']) && ($item['gravity'] == GRAVITY_COMMENT)) { - $parent = Post::selectFirst(['id', 'uri-id', 'private', 'author-link', 'alias'], ['uri' => $item['thr-parent']]); + $parent_uri = $item['parent-uri'] ?? $item['thr-parent']; + if (empty($activity['directmessage']) && ($parent_uri != $item['uri']) && ($item['gravity'] == GRAVITY_COMMENT)) { + $parent = Post::selectFirst(['id', 'uri-id', 'private', 'author-link', 'alias'], ['uri' => $parent_uri]); if (!DBA::isResult($parent)) { - Logger::warning('Unknown parent item.', ['uri' => $item['thr-parent']]); + Logger::warning('Unknown parent item.', ['uri' => $parent_uri]); return false; } if (($parent['private'] == Item::PRIVATE) && ($parent['private'] != Item::PRIVATE)) { @@ -1327,8 +1353,8 @@ class Processor if (empty($contact)) { Contact::update(['hub-verify' => $activity['id'], 'protocol' => Protocol::ACTIVITYPUB], ['id' => $cid]); } - Logger::notice('Follow user ' . $uid . ' from contact ' . $cid . ' with id ' . $activity['id']); + Queue::remove($activity); } /** @@ -1426,6 +1452,7 @@ class Processor Contact\User::setIsBlocked($cid, $uid, true); Logger::info('Contact blocked user', ['contact' => $cid, 'user' => $uid]); + Queue::remove($activity); } /** @@ -1450,6 +1477,7 @@ class Processor Contact\User::setIsBlocked($cid, $uid, false); Logger::info('Contact unblocked user', ['contact' => $cid, 'user' => $uid]); + Queue::remove($activity); } /** diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index d5a308b109..055a728dfb 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -140,6 +140,14 @@ class Queue $activity['entry-id'] = $entry['id']; $activity['worker-id'] = $entry['wid']; + $receivers = DBA::select('inbox-entry-receiver', ['uid'], ['queue-id' => $entry['id']]); + while ($receiver = DBA::fetch($receivers)) { + if (!in_array($receiver['uid'], $activity['receiver'])) { + $activity['receiver'][] = $receiver['uid']; + } + } + DBA::close($receivers); + if (!Receiver::routeActivities($activity, $type, $push)) { self::remove($activity); } diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 045248e629..1cbd5c9e36 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -516,6 +516,10 @@ class Receiver } } + if (($type == 'as:Add') && is_array($activity['as:object']) && (count($activity['as:object']) == 1)) { + $trust_source = false; + } + // $trust_source is called by reference and is set to true if the content was retrieved successfully $object_data = self::prepareObjectData($activity, $uid, $push, $trust_source); if (empty($object_data)) { diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index a9c163964e..67c0c24801 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -1515,26 +1515,6 @@ class Transmitter return $body; } - /** - * Fetches the "context" value for a givem item array from the "conversation" table - * - * @param array $item Item array - * @return string with context url - * @throws \Exception - */ - private static function fetchContextURLForItem(array $item): string - { - $conversation = DBA::selectFirst('conversation', ['conversation-href', 'conversation-uri'], ['item-uri' => $item['parent-uri']]); - if (DBA::isResult($conversation) && !empty($conversation['conversation-href'])) { - $context_uri = $conversation['conversation-href']; - } elseif (DBA::isResult($conversation) && !empty($conversation['conversation-uri'])) { - $context_uri = $conversation['conversation-uri']; - } else { - $context_uri = $item['parent-uri'] . '#context'; - } - return $context_uri; - } - /** * Returns if the post contains sensitive content ("nsfw") * @@ -1646,7 +1626,7 @@ class Transmitter $data['url'] = $link ?? $item['plink']; $data['attributedTo'] = $item['author-link']; $data['sensitive'] = self::isSensitive($item['uri-id']); - $data['context'] = self::fetchContextURLForItem($item); + $data['conversation'] = $data['context'] = $item['conversation']; if (!empty($item['title'])) { $data['name'] = BBCode::toPlaintext($item['title'], false); diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php index d09376d60e..bbdfaa42c7 100644 --- a/src/Worker/ExpirePosts.php +++ b/src/Worker/ExpirePosts.php @@ -183,6 +183,7 @@ class ExpirePosts AND NOT EXISTS(SELECT `parent-uri-id` FROM `post-user` WHERE `parent-uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `thr-parent-id` FROM `post-user` WHERE `thr-parent-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `external-id` FROM `post-user` WHERE `external-id` = `item-uri`.`id`) + AND NOT EXISTS(SELECT `conversation-id` FROM `post-thread` WHERE `conversation-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `uri-id` FROM `mail` WHERE `uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `uri-id` FROM `event` WHERE `uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `uri-id` FROM `user-contact` WHERE `uri-id` = `item-uri`.`id`) diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index abaddc8d0a..74bcdc8161 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -1355,6 +1355,7 @@ return [ "comment" => "Thread related data", "fields" => [ "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "conversation-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the conversation uri"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item owner"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item author"], "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], @@ -1366,6 +1367,7 @@ return [ ], "indexes" => [ "PRIMARY" => ["uri-id"], + "conversation-id" => ["conversation-id"], "owner-id" => ["owner-id"], "author-id" => ["author-id"], "causer-id" => ["causer-id"], @@ -1434,6 +1436,7 @@ return [ "comment" => "Thread related data per user", "fields" => [ "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "conversation-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the conversation uri"], "owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item owner"], "author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Item author"], "causer-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Link to the contact table with uid=0 of the contact that caused the item creation"], @@ -1460,6 +1463,7 @@ return [ "indexes" => [ "PRIMARY" => ["uid", "uri-id"], "uri-id" => ["uri-id"], + "conversation-id" => ["conversation-id"], "owner-id" => ["owner-id"], "author-id" => ["author-id"], "causer-id" => ["causer-id"], diff --git a/static/dbview.config.php b/static/dbview.config.php index 040a9ebc30..0c91016d49 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -70,6 +70,8 @@ "parent-uri-id" => ["post-user", "parent-uri-id"], "thr-parent" => ["thr-parent-item-uri", "uri"], "thr-parent-id" => ["post-user", "thr-parent-id"], + "conversation" => ["conversation-item-uri", "uri"], + "conversation-id" => ["post-thread-user", "conversation-id"], "guid" => ["item-uri", "guid"], "wall" => ["post-user", "wall"], "gravity" => ["post-user", "gravity"], @@ -220,6 +222,7 @@ LEFT JOIN `item-uri` ON `item-uri`.`id` = `post-user`.`uri-id` LEFT JOIN `item-uri` AS `thr-parent-item-uri` ON `thr-parent-item-uri`.`id` = `post-user`.`thr-parent-id` LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-user`.`parent-uri-id` + LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` LEFT JOIN `verb` ON `verb`.`id` = `post-user`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` @@ -243,6 +246,8 @@ "parent-uri-id" => ["post-user", "parent-uri-id"], "thr-parent" => ["thr-parent-item-uri", "uri"], "thr-parent-id" => ["post-user", "thr-parent-id"], + "conversation" => ["conversation-item-uri", "uri"], + "conversation-id" => ["post-thread-user", "conversation-id"], "guid" => ["item-uri", "guid"], "wall" => ["post-thread-user", "wall"], "gravity" => ["post-user", "gravity"], @@ -392,6 +397,7 @@ LEFT JOIN `item-uri` ON `item-uri`.`id` = `post-thread-user`.`uri-id` LEFT JOIN `item-uri` AS `thr-parent-item-uri` ON `thr-parent-item-uri`.`id` = `post-user`.`thr-parent-id` LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post-user`.`parent-uri-id` + LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread-user`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post-user`.`external-id` LEFT JOIN `verb` ON `verb`.`id` = `post-user`.`vid` LEFT JOIN `event` ON `event`.`id` = `post-user`.`event-id` @@ -411,6 +417,8 @@ "parent-uri-id" => ["post", "parent-uri-id"], "thr-parent" => ["thr-parent-item-uri", "uri"], "thr-parent-id" => ["post", "thr-parent-id"], + "conversation" => ["conversation-item-uri", "uri"], + "conversation-id" => ["post-thread", "conversation-id"], "guid" => ["item-uri", "guid"], "gravity" => ["post", "gravity"], "extid" => ["external-item-uri", "uri"], @@ -530,6 +538,7 @@ LEFT JOIN `item-uri` ON `item-uri`.`id` = `post`.`uri-id` LEFT JOIN `item-uri` AS `thr-parent-item-uri` ON `thr-parent-item-uri`.`id` = `post`.`thr-parent-id` LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post`.`parent-uri-id` + LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post`.`external-id` LEFT JOIN `verb` ON `verb`.`id` = `post`.`vid` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post`.`uri-id` @@ -546,6 +555,8 @@ "parent-uri-id" => ["post", "parent-uri-id"], "thr-parent" => ["thr-parent-item-uri", "uri"], "thr-parent-id" => ["post", "thr-parent-id"], + "conversation" => ["conversation-item-uri", "uri"], + "conversation-id" => ["post-thread", "conversation-id"], "guid" => ["item-uri", "guid"], "gravity" => ["post", "gravity"], "extid" => ["external-item-uri", "uri"], @@ -665,6 +676,7 @@ LEFT JOIN `item-uri` ON `item-uri`.`id` = `post-thread`.`uri-id` LEFT JOIN `item-uri` AS `thr-parent-item-uri` ON `thr-parent-item-uri`.`id` = `post`.`thr-parent-id` LEFT JOIN `item-uri` AS `parent-item-uri` ON `parent-item-uri`.`id` = `post`.`parent-uri-id` + LEFT JOIN `item-uri` AS `conversation-item-uri` ON `conversation-item-uri`.`id` = `post-thread`.`conversation-id` LEFT JOIN `item-uri` AS `external-item-uri` ON `external-item-uri`.`id` = `post`.`external-id` LEFT JOIN `verb` ON `verb`.`id` = `post`.`vid` LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `post-thread`.`uri-id` From 8c602071c2070b78c4780c2d0d72b4599b6483cb Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 23 Jul 2022 12:50:15 +0000 Subject: [PATCH 16/26] Bookwyrm posts now work --- src/Model/APContact.php | 9 ++++++- src/Model/Item.php | 20 +++++++------- src/Protocol/ActivityPub/Processor.php | 2 +- src/Protocol/ActivityPub/Receiver.php | 36 ++++++++++++++++---------- src/Util/JsonLD.php | 7 +++++ 5 files changed, 49 insertions(+), 25 deletions(-) diff --git a/src/Model/APContact.php b/src/Model/APContact.php index ba2e3c9734..841c028909 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -539,6 +539,13 @@ class APContact HTTPSignature::setInboxStatus($url, true, $shared); } + /** + * Check if the apcontact is a relay account + * + * @param array $apcontact + * + * @return bool + */ public static function isRelay(array $apcontact): bool { if ($apcontact['nick'] != 'relay') { @@ -549,7 +556,7 @@ class APContact return true; } - if (in_array($apcontact['type'], ['Group', 'Service']) && ($apcontact['nick'] == 'relay') && is_null($apcontact['outbox'])) { + if (in_array($apcontact['type'], ['Group', 'Service']) && is_null($apcontact['outbox'])) { return true; } diff --git a/src/Model/Item.php b/src/Model/Item.php index 4142953568..3ae23412e6 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -683,19 +683,19 @@ class Item 'uri-id', 'parent-uri-id', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'wall', 'private', 'origin', 'author-id']; - $condition = ['uri-id' => $item['thr-parent-id'], 'uid' => $item['uid']]; + $condition = ['uri-id' => [$item['thr-parent-id'], $item['parent-uri-id']], 'uid' => $item['uid']]; $params = ['order' => ['id' => false]]; $parent = Post::selectFirst($fields, $condition, $params); - if (!DBA::isResult($parent) && ($item['thr-parent-id'] != $item['parent-uri-id'])) { - $condition = ['uri-id' => $item['parent-uri-id'], 'uid' => $item['uid']]; - $parent = Post::selectFirst($fields, $condition, $params); - } - - if (!DBA::isResult($parent) && $item['origin']) { + if (!DBA::isResult($parent) && Post::exists(['uri-id' => [$item['thr-parent-id'], $item['parent-uri-id']], 'uid' => 0])) { $stored = Item::storeForUserByUriId($item['thr-parent-id'], $item['uid']); - Logger::info('Stored thread parent item for user', ['uri-id' => $item['thr-parent-id'], 'uid' => $item['uid'], 'stored' => $stored]); - $parent = Post::selectFirst($fields, $condition, $params); + if (!$stored && ($item['thr-parent-id'] != $item['parent-uri-id'])) { + $stored = Item::storeForUserByUriId($item['parent-uri-id'], $item['uid']); + } + if ($stored) { + Logger::info('Stored thread parent item for user', ['uri-id' => $item['thr-parent-id'], 'uid' => $item['uid'], 'stored' => $stored]); + $parent = Post::selectFirst($fields, $condition, $params); + } } if (!DBA::isResult($parent)) { @@ -908,7 +908,7 @@ class Item $item['contact-id'] = self::contactId($item); if (!empty($item['direction']) && in_array($item['direction'], [Conversation::PUSH, Conversation::RELAY]) && - empty($item['origin']) &&self::isTooOld($item)) { + empty($item['origin']) && self::isTooOld($item)) { Logger::info('Item is too old', ['item' => $item]); return 0; } diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 28192d35ba..9c7cfef77d 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -299,7 +299,7 @@ class Processor if (empty($activity['directmessage']) && ($activity['id'] != $activity['reply-to-id']) && !Post::exists(['uri' => $activity['reply-to-id']])) { $recursion_depth = $activity['recursion-depth'] ?? 0; Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); - if ($recursion_depth < 10000) { + if ($recursion_depth < 10) { $result = self::fetchMissingActivity($activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); $fetch_by_worker = empty($result); } else { diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 1cbd5c9e36..92ed31922b 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -437,17 +437,18 @@ class Receiver $object_data['receiver'] = array_replace($object_data['receiver'] ?? [], $receivers); $object_data['reception_type'] = array_replace($object_data['reception_type'] ?? [], $reception_types); - $author = $object_data['author'] ?? $actor; - if (!empty($author) && !empty($object_data['id'])) { - $author_host = parse_url($author, PHP_URL_HOST); - $id_host = parse_url($object_data['id'], PHP_URL_HOST); - if ($author_host == $id_host) { - Logger::info('Valid hosts', ['type' => $type, 'host' => $id_host]); - } else { - Logger::notice('Differing hosts on author and id', ['type' => $type, 'author' => $author_host, 'id' => $id_host]); - $trust_source = false; - } - } +// This check here interferes with Hubzilla posts where the author host differs from the host the post was created +// $author = $object_data['author'] ?? $actor; +// if (!empty($author) && !empty($object_data['id'])) { +// $author_host = parse_url($author, PHP_URL_HOST); +// $id_host = parse_url($object_data['id'], PHP_URL_HOST); +// if ($author_host == $id_host) { +// Logger::info('Valid hosts', ['type' => $type, 'host' => $id_host]); +// } else { +// Logger::notice('Differing hosts on author and id', ['type' => $type, 'author' => $author_host, 'id' => $id_host]); +// $trust_source = false; +// } +// } Logger::info('Processing ' . $object_data['type'] . ' ' . $object_data['object_type'] . ' ' . $object_data['id']); @@ -580,7 +581,16 @@ class Receiver } } - public static function routeActivities($object_data, $type, $push) + /** + * Route activities + * + * @param array $object_data + * @param string $type + * @param boolean $push + * + * @return boolean Could the activity be routed? + */ + public static function routeActivities(array $object_data, string $type, bool $push): bool { $activity = $object_data['object_activity'] ?? []; @@ -627,7 +637,7 @@ class Receiver $item = ActivityPub\Processor::createItem($object_data); if (empty($item)) { - return; + return false; } $item['post-reason'] = Item::PR_ANNOUNCEMENT; diff --git a/src/Util/JsonLD.php b/src/Util/JsonLD.php index 1200a9cbed..947274134c 100644 --- a/src/Util/JsonLD.php +++ b/src/Util/JsonLD.php @@ -177,6 +177,13 @@ class JsonLD } } + // Bookwyrm transmits "id" fields with "null", which isn't allowed. + array_walk_recursive($json, function (&$value, $key) { + if ($key == 'id' && is_null($value)) { + $value = ''; + } + }); + $jsonobj = json_decode(json_encode($json, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); try { From 5d6eb4670aba01ea83458541b04bbc1a5d311181 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 23 Jul 2022 13:58:14 +0000 Subject: [PATCH 17/26] We now store the conversation as well --- database.sql | 1 + doc/database/db_inbox-entry.md | 1 + src/Protocol/ActivityPub/Processor.php | 2 +- src/Protocol/ActivityPub/Queue.php | 6 ++++++ static/dbstructure.config.php | 1 + 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/database.sql b/database.sql index eb97ec3074..42624d4d1d 100644 --- a/database.sql +++ b/database.sql @@ -732,6 +732,7 @@ CREATE TABLE IF NOT EXISTS `inbox-entry` ( `activity-id` varbinary(255) COMMENT 'id of the incoming activity', `object-id` varbinary(255) COMMENT '', `in-reply-to-id` varbinary(255) COMMENT '', + `conversation` varbinary(255) COMMENT '', `type` varchar(64) COMMENT 'Type of the activity', `object-type` varchar(64) COMMENT 'Type of the object activity', `object-object-type` varchar(64) COMMENT 'Type of the object\'s object activity', diff --git a/doc/database/db_inbox-entry.md b/doc/database/db_inbox-entry.md index 91126919e9..32dd1203dc 100644 --- a/doc/database/db_inbox-entry.md +++ b/doc/database/db_inbox-entry.md @@ -12,6 +12,7 @@ Fields | activity-id | id of the incoming activity | varbinary(255) | YES | | NULL | | | object-id | | varbinary(255) | YES | | NULL | | | in-reply-to-id | | varbinary(255) | YES | | NULL | | +| conversation | | varbinary(255) | YES | | NULL | | | type | Type of the activity | varchar(64) | YES | | NULL | | | object-type | Type of the object activity | varchar(64) | YES | | NULL | | | object-object-type | Type of the object's object activity | varchar(64) | YES | | NULL | | diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 9c7cfef77d..4b02ae8103 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -695,7 +695,7 @@ class Processor Logger::warning('Unknown parent item.', ['uri' => $parent_uri]); return false; } - if (($parent['private'] == Item::PRIVATE) && ($parent['private'] != Item::PRIVATE)) { + if (($item['private'] == Item::PRIVATE) && ($parent['private'] != Item::PRIVATE)) { Logger::warning('Item is private but the parent is not. Dropping.', ['item-uri' => $item['uri'], 'thr-parent' => $item['thr-parent']]); return false; } diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index 055a728dfb..279f2108b7 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -57,6 +57,12 @@ class Queue $fields['in-reply-to-id'] = $activity['reply-to-id']; } + if (!empty($activity['context'])) { + $fields['conversation'] = $activity['context']; + } elseif(!empty($activity['conversation'])) { + $fields['conversation'] = $activity['conversation']; + } + if (!empty($activity['object_object_type'])) { $fields['object-object-type'] = $activity['object_object_type']; } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 74bcdc8161..0b6af9b13a 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -791,6 +791,7 @@ return [ "activity-id" => ["type" => "varbinary(255)", "comment" => "id of the incoming activity"], "object-id" => ["type" => "varbinary(255)", "comment" => ""], "in-reply-to-id" => ["type" => "varbinary(255)", "comment" => ""], + "conversation" => ["type" => "varbinary(255)", "comment" => ""], "type" => ["type" => "varchar(64)", "comment" => "Type of the activity"], "object-type" => ["type" => "varchar(64)", "comment" => "Type of the object activity"], "object-object-type" => ["type" => "varchar(64)", "comment" => "Type of the object's object activity"], From 6a19639ac2ec09c432a1831b10f2cd82f8737393 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 23 Jul 2022 14:10:07 +0000 Subject: [PATCH 18/26] Standards --- src/Protocol/ActivityPub/Processor.php | 2 +- src/Protocol/ActivityPub/Queue.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 4b02ae8103..1a7458c0e4 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -282,7 +282,7 @@ class Processor if (!empty($activity['context'])) { $item['conversation'] = $activity['context']; - } elseif(!empty($activity['conversation'])) { + } elseif (!empty($activity['conversation'])) { $item['conversation'] = $activity['conversation']; } diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index 279f2108b7..6d0bed3e1c 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -59,7 +59,7 @@ class Queue if (!empty($activity['context'])) { $fields['conversation'] = $activity['context']; - } elseif(!empty($activity['conversation'])) { + } elseif (!empty($activity['conversation'])) { $fields['conversation'] = $activity['conversation']; } From 073695b33c5f9c5d89d91958b09259c59e12dd98 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 23 Jul 2022 23:10:47 +0000 Subject: [PATCH 19/26] Funkwhale context file moved --- src/Protocol/ActivityPub/Processor.php | 2 +- src/Protocol/ActivityPub/Queue.php | 6 ++++-- src/Worker/FetchMissingActivity.php | 1 + funkwhale.audio.jsonld => static/funkwhale.audio.jsonld | 0 4 files changed, 6 insertions(+), 3 deletions(-) rename funkwhale.audio.jsonld => static/funkwhale.audio.jsonld (100%) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 1a7458c0e4..aa169ab028 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -935,7 +935,7 @@ class Processor Logger::info('Item insertion successful', ['user' => $item['uid'], 'item_id' => $item_id]); $success = true; } else { - Logger::notice('Item insertion aborted', ['user' => $item['uid']]); + Logger::notice('Item insertion aborted', ['uri' => $item['uri'], 'uid' => $item['uid']]); } if ($item['uid'] == 0) { diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index 6d0bed3e1c..d150f9f9cc 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -179,7 +179,9 @@ class Queue */ public static function clear() { - DBA::delete('inbox-entry', ["`wid` IS NULL AND `received` < ?", DateTimeFormat::utc('now - 4 hours')]); + // We delete all entries that aren't associated with a worker entry after seven days. + // The other entries are deleted when the worker deferred for too long. + DBA::delete('inbox-entry', ["`wid` IS NULL AND `received` < ?", DateTimeFormat::utc('now - 7 days')]); } /** @@ -190,7 +192,7 @@ class Queue */ public static function processReplyByUri(string $uri) { - $entries = DBA::select('inbox-entry', ['id'], ['in-reply-to-id' => $uri], ['order' => ['id' => true]]); + $entries = DBA::select('inbox-entry', ['id'], ["`in-reply-to-id` = ? AND `object-id` != ?", $uri, $uri]); while ($entry = DBA::fetch($entries)) { self::process($entry['id']); } diff --git a/src/Worker/FetchMissingActivity.php b/src/Worker/FetchMissingActivity.php index 1a2818c3cc..6cb832c779 100644 --- a/src/Worker/FetchMissingActivity.php +++ b/src/Worker/FetchMissingActivity.php @@ -42,6 +42,7 @@ class FetchMissingActivity Queue::processReplyByUri($url); } elseif (!Worker::defer()) { Logger::info('Activity could not be fetched', ['url' => $url]); + // @todo performe recursive deletion of all entries } else { Logger::info('Fetching deferred', ['url' => $url]); } diff --git a/funkwhale.audio.jsonld b/static/funkwhale.audio.jsonld similarity index 100% rename from funkwhale.audio.jsonld rename to static/funkwhale.audio.jsonld From c2b2e8ae9f3a87261f048e83fc04078604be3840 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 23 Jul 2022 23:15:08 +0000 Subject: [PATCH 20/26] Standards --- src/Worker/FetchMissingActivity.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Worker/FetchMissingActivity.php b/src/Worker/FetchMissingActivity.php index 6cb832c779..ae0f8a7fbf 100644 --- a/src/Worker/FetchMissingActivity.php +++ b/src/Worker/FetchMissingActivity.php @@ -41,8 +41,8 @@ class FetchMissingActivity Logger::info('Successfully fetched missing activity', ['url' => $url]); Queue::processReplyByUri($url); } elseif (!Worker::defer()) { + // @todo perform recursive deletion of all entries Logger::info('Activity could not be fetched', ['url' => $url]); - // @todo performe recursive deletion of all entries } else { Logger::info('Fetching deferred', ['url' => $url]); } From 06280aa5a334d80f26d4d10c08df07bdabc61915 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 24 Jul 2022 09:26:52 +0000 Subject: [PATCH 21/26] Recursively delete failed worker tasks --- src/Protocol/ActivityPub/Processor.php | 34 ++++++++++-------- src/Protocol/ActivityPub/Queue.php | 48 ++++++++++++++++++++++++-- src/Protocol/ActivityPub/Receiver.php | 12 +++---- src/Worker/Cron.php | 7 ++-- src/Worker/FetchMissingActivity.php | 11 ++++-- 5 files changed, 82 insertions(+), 30 deletions(-) diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index aa169ab028..9d5f3f2f6e 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -551,16 +551,13 @@ class Processor } } - if ($activity['target_id'] != $actor['featured']) { - return null; + $parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]); + if (empty($parent['uri-id'])) { + if (self::fetchMissingActivity($activity['object_id'], $activity, '', Receiver::COMPLETION_AUTO)) { + $parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]); + } } - $id = Contact::getIdForURL($activity['actor']); - if (empty($id)) { - return null; - } - - $parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id'], 'author-id' => $id]); if (!empty($parent['uri-id'])) { return $parent['uri-id']; } @@ -1191,20 +1188,27 @@ class Processor return ''; } + $signer = []; + + if (!empty($object['attributedTo'])) { + $attributed_to = $object['attributedTo']; + if (is_array($attributed_to)) { + $compacted = JsonLD::compact($object); + $attributed_to = JsonLD::fetchElement($compacted, 'as:attributedTo', '@id'); + } + $signer[] = $attributed_to; + } + if (!empty($object['actor'])) { $object_actor = $object['actor']; - } elseif (!empty($object['attributedTo'])) { - $object_actor = $object['attributedTo']; - if (is_array($object_actor)) { - $compacted = JsonLD::compact($object); - $object_actor = JsonLD::fetchElement($compacted, 'as:attributedTo', '@id'); - } + } elseif (!empty($attributed_to)) { + $object_actor = $attributed_to; } else { // Shouldn't happen $object_actor = ''; } - $signer = [$object_actor]; + $signer[] = $object_actor; if (!empty($child['author'])) { $actor = $child['author']; diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index d150f9f9cc..3d40d71638 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -95,6 +95,42 @@ class Queue DBA::delete('inbox-entry', ['id' => $activity['entry-id']]); } + /** + * Delete all entries that depend on the given worker id + * + * @param integer $wid + * @return void + */ + public static function deleteByWorkerId(int $wid) + { + $entries = DBA::select('inbox-entry', ['id'], ['wid' => $wid]); + while ($entry = DBA::fetch($entries)) { + self::deleteById($entry['id']); + } + DBA::close($entries); + } + + /** + * Delete recursively an entry and all their children + * + * @param integer $id + * @return void + */ + private static function deleteById(int $id) + { + $entry = DBA::selectFirst('inbox-entry', ['id', 'object-id'], ['id' => $id]); + if (empty($entry)) { + return; + } + + $children = DBA::select('inbox-entry', ['id'], ['in-reply-to-id' => $entry['object-id']]); + while ($child = DBA::fetch($children)) { + self::deleteById($child['id']); + } + DBA::close($children); + DBA::delete('inbox-entry', ['id' => $entry['id']]); + } + /** * Set the worker id for the queue entry * @@ -143,8 +179,9 @@ class Queue $type = $entry['type']; $push = $entry['push']; - $activity['entry-id'] = $entry['id']; - $activity['worker-id'] = $entry['wid']; + $activity['entry-id'] = $entry['id']; + $activity['worker-id'] = $entry['wid']; + $activity['recursion-depth'] = 0; $receivers = DBA::select('inbox-entry-receiver', ['uid'], ['queue-id' => $entry['id']]); while ($receiver = DBA::fetch($receivers)) { @@ -166,8 +203,13 @@ class Queue */ public static function processAll() { - $entries = DBA::select('inbox-entry', ['id', 'type', 'object-type'], [], ['order' => ['id' => true]]); + $entries = DBA::select('inbox-entry', ['id', 'type', 'object-type', 'object-id', 'in-reply-to-id'], ["`wid` IS NULL"], ['order' => ['id' => true]]); while ($entry = DBA::fetch($entries)) { + // We don't need to process entries that depend on already existing entries. + if (!empty($entry['in-reply-to-id']) && DBA::exists('inbox-entry', ['object-id' => $entry['in-reply-to-id']])) { + continue; + } + Logger::debug('Process leftover entry', $entry); self::process($entry['id']); } } diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 92ed31922b..3e34510749 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -517,10 +517,6 @@ class Receiver } } - if (($type == 'as:Add') && is_array($activity['as:object']) && (count($activity['as:object']) == 1)) { - $trust_source = false; - } - // $trust_source is called by reference and is set to true if the content was retrieved successfully $object_data = self::prepareObjectData($activity, $uid, $push, $trust_source); if (empty($object_data)) { @@ -556,10 +552,6 @@ class Receiver $object_data['thread-children-type'] = $activity['thread-children-type']; } - if (!empty($activity['recursion-depth'])) { - $object_data['recursion-depth'] = $activity['recursion-depth']; - } - // Internal flag for posts that arrived via relay if (!empty($activity['from-relay'])) { $object_data['from-relay'] = $activity['from-relay']; @@ -571,6 +563,10 @@ class Receiver $object_data = Queue::add($object_data, $type, $uid, $http_signer, $push); + if (!empty($activity['recursion-depth'])) { + $object_data['recursion-depth'] = $activity['recursion-depth']; + } + if (in_array('as:Question', [$object_data['object_type'] ?? '', $object_data['object_object_type'] ?? ''])) { self::storeUnhandledActivity(false, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); } diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 68ad218014..c2109e66a3 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -128,11 +128,14 @@ class Cron if (DI::config()->get('system', 'optimize_tables')) { Worker::add(PRIORITY_LOW, 'OptimizeTables'); } - - DI::config()->set('system', 'last_cron_daily', time()); + + // Process all unprocessed entries + Queue::processAll(); // Resubscribe to relay servers Relay::reSubscribe(); + + DI::config()->set('system', 'last_cron_daily', time()); } Logger::notice('end'); diff --git a/src/Worker/FetchMissingActivity.php b/src/Worker/FetchMissingActivity.php index ae0f8a7fbf..91b473a6ef 100644 --- a/src/Worker/FetchMissingActivity.php +++ b/src/Worker/FetchMissingActivity.php @@ -23,6 +23,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Core\Worker; +use Friendica\DI; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\ActivityPub\Queue; use Friendica\Protocol\ActivityPub\Receiver; @@ -32,6 +33,8 @@ class FetchMissingActivity /** * Fetch missing activities * @param string $url Contact URL + * + * @return void */ public static function execute(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL) { @@ -39,10 +42,14 @@ class FetchMissingActivity $result = ActivityPub\Processor::fetchMissingActivity($url, $child, $relay_actor, $completion); if ($result) { Logger::info('Successfully fetched missing activity', ['url' => $url]); - Queue::processReplyByUri($url); } elseif (!Worker::defer()) { - // @todo perform recursive deletion of all entries Logger::info('Activity could not be fetched', ['url' => $url]); + + // recursively delete all entries that belong to this worker task + $queue = DI::app()->getQueue(); + if (!empty($queue['id'])) { + Queue::deleteByWorkerId($queue['id']); + } } else { Logger::info('Fetching deferred', ['url' => $url]); } From 6abb4d40d6456f58a748077f161f5101b4048659 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 24 Jul 2022 13:09:35 +0000 Subject: [PATCH 22/26] Handle changed parents --- src/Model/Post/User.php | 32 ++++++++++++++++++++++---- src/Protocol/ActivityPub/Processor.php | 28 ++++++++++++++++++++++ src/Protocol/ActivityPub/Queue.php | 4 ++-- src/Worker/Cron.php | 6 ++--- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/src/Model/Post/User.php b/src/Model/Post/User.php index 9da9718068..7bd5706997 100644 --- a/src/Model/Post/User.php +++ b/src/Model/Post/User.php @@ -24,7 +24,6 @@ namespace Friendica\Model\Post; use Friendica\Database\DBA; use \BadMethodCallException; use Friendica\Database\Database; -use Friendica\Database\DBStructure; use Friendica\DI; class User @@ -44,10 +43,6 @@ class User throw new BadMethodCallException('Empty URI_id'); } - if (DBA::exists('post-user', ['uri-id' => $uri_id, 'uid' => $uid])) { - return false; - } - $fields = DI::dbaDefinition()->truncateFieldsForTable('post-user', $data); // Additionally assign the key fields @@ -59,6 +54,33 @@ class User $fields['unseen'] = false; } + // Does the entry already exist? + if (DBA::exists('post-user', ['uri-id' => $uri_id, 'uid' => $uid])) { + $postuser = DBA::selectFirst('post-user', [], ['uri-id' => $uri_id, 'uid' => $uid]); + + // We quit here, when there are obvious differences + foreach (['created', 'owner-id', 'author-id', 'vid', 'network', 'private', 'wall', 'origin'] as $key) { + if ($fields[$key] != $postuser[$key]) { + return 0; + } + } + + $update = []; + foreach (['gravity', 'parent-uri-id', 'thr-parent-id'] as $key) { + if ($fields[$key] != $postuser[$key]) { + $update[$key] = $fields[$key]; + } + } + + // When the parents changed, we apply these changes to the existing entry + if (!empty($update)) { + DBA::update('post-user', $update, ['id' => $postuser['id']]); + return $postuser['id']; + } else { + return 0; + } + } + if (!DBA::insert('post-user', $fields, Database::INSERT_IGNORE)) { return 0; } diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 9d5f3f2f6e..d1c3994e2a 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -45,7 +45,9 @@ use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Relay; use Friendica\Util\DateTimeFormat; +use Friendica\Util\HTTPSignature; use Friendica\Util\JsonLD; +use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Worker\Delivery; @@ -301,6 +303,11 @@ class Processor Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); if ($recursion_depth < 10) { $result = self::fetchMissingActivity($activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); + if (empty($result) && self::ActivityIsGone($activity['reply-to-id'])) { + // Recursively delete this and all depending entries + Queue::deleteById($activity['entry-id']); + return []; + } $fetch_by_worker = empty($result); } else { Logger::notice('Recursion level is too high.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); @@ -452,6 +459,24 @@ class Processor return $item; } + /** + * Check if a given activity is no longer available + * + * @param string $url + * + * @return boolean + */ + private static function ActivityIsGone(string $url): bool + { + $curlResult = HTTPSignature::fetchRaw($url, 0); + + if (Network::isUrlBlocked($url)) { + return true; + } + + // @todo To ensure that the remote system is working correctly, we can check if the "Content-Type" contains JSON + return in_array($curlResult->getReturnCode(), [404]); + } /** * Delete items * @@ -933,6 +958,9 @@ class Processor $success = true; } else { Logger::notice('Item insertion aborted', ['uri' => $item['uri'], 'uid' => $item['uid']]); + if (Item::isTooOld($item) || !Item::isValid($item)) { + Queue::remove($activity); + } } if ($item['uid'] == 0) { diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index 3d40d71638..7b91609bdb 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -116,7 +116,7 @@ class Queue * @param integer $id * @return void */ - private static function deleteById(int $id) + public static function deleteById(int $id) { $entry = DBA::selectFirst('inbox-entry', ['id', 'object-id'], ['id' => $id]); if (empty($entry)) { @@ -206,7 +206,7 @@ class Queue $entries = DBA::select('inbox-entry', ['id', 'type', 'object-type', 'object-id', 'in-reply-to-id'], ["`wid` IS NULL"], ['order' => ['id' => true]]); while ($entry = DBA::fetch($entries)) { // We don't need to process entries that depend on already existing entries. - if (!empty($entry['in-reply-to-id']) && DBA::exists('inbox-entry', ['object-id' => $entry['in-reply-to-id']])) { + if (!empty($entry['in-reply-to-id']) && DBA::exists('inbox-entry', ["`id` != ? AND `object-id` = ?", $entry['id'], $entry['in-reply-to-id']])) { continue; } Logger::debug('Process leftover entry', $entry); diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index c2109e66a3..2ace97d830 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -92,6 +92,9 @@ class Cron // Remove old pending posts from the queue Queue::clear(); + // Process all unprocessed entries + Queue::processAll(); + // Search for new contacts in the directory if (DI::config()->get('system', 'synchronize_directory')) { Worker::add(PRIORITY_LOW, 'PullDirectory'); @@ -129,9 +132,6 @@ class Cron Worker::add(PRIORITY_LOW, 'OptimizeTables'); } - // Process all unprocessed entries - Queue::processAll(); - // Resubscribe to relay servers Relay::reSubscribe(); From 72bc92db623ac920b9ea05b57d7ff305be1cd8ee Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 24 Jul 2022 13:11:52 +0000 Subject: [PATCH 23/26] Standards --- src/Worker/FetchMissingActivity.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Worker/FetchMissingActivity.php b/src/Worker/FetchMissingActivity.php index 91b473a6ef..bc7d80f7c5 100644 --- a/src/Worker/FetchMissingActivity.php +++ b/src/Worker/FetchMissingActivity.php @@ -33,7 +33,7 @@ class FetchMissingActivity /** * Fetch missing activities * @param string $url Contact URL - * + * * @return void */ public static function execute(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL) From 55b6a89c7c42d2bde3da3c849c8ba0c124b40867 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 24 Jul 2022 13:49:57 +0000 Subject: [PATCH 24/26] Optimizing --- src/Protocol/ActivityPub/Queue.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index 7b91609bdb..011101eed7 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -24,6 +24,7 @@ namespace Friendica\Protocol\ActivityPub; use Friendica\Core\Logger; use Friendica\Database\Database; use Friendica\Database\DBA; +use Friendica\DI; use Friendica\Util\DateTimeFormat; /** @@ -224,6 +225,13 @@ class Queue // We delete all entries that aren't associated with a worker entry after seven days. // The other entries are deleted when the worker deferred for too long. DBA::delete('inbox-entry', ["`wid` IS NULL AND `received` < ?", DateTimeFormat::utc('now - 7 days')]); + + // Optimizing this table only last seconds + if (DI::config()->get('system', 'optimize_tables')) { + Logger::info('Optimize start'); + DBA::e("OPTIMIZE TABLE `inbox-entry`"); + Logger::info('Optimize end'); + } } /** From 51cc1f679f525c1ab3bea71c4c1026e1ad1386d4 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 24 Jul 2022 14:26:06 +0000 Subject: [PATCH 25/26] Added trust / isActivityGone --- database.sql | 3 ++- doc/database/db_inbox-entry.md | 31 +++++++++++++------------- src/Protocol/ActivityPub/Processor.php | 4 ++-- src/Protocol/ActivityPub/Queue.php | 5 +++-- src/Protocol/ActivityPub/Receiver.php | 12 +++++----- static/dbstructure.config.php | 3 ++- 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/database.sql b/database.sql index 42624d4d1d..6b8d7df2ff 100644 --- a/database.sql +++ b/database.sql @@ -739,7 +739,8 @@ CREATE TABLE IF NOT EXISTS `inbox-entry` ( `received` datetime COMMENT 'Receiving date', `activity` mediumtext COMMENT 'The JSON activity', `signer` varchar(255) COMMENT '', - `push` boolean NOT NULL DEFAULT '0' COMMENT '', + `push` boolean COMMENT 'Is the entry pushed or have pulled it?', + `trust` boolean COMMENT 'Do we trust this entry?', `wid` int unsigned COMMENT 'Workerqueue id', PRIMARY KEY(`id`), UNIQUE INDEX `activity-id` (`activity-id`), diff --git a/doc/database/db_inbox-entry.md b/doc/database/db_inbox-entry.md index 32dd1203dc..d10d3f242b 100644 --- a/doc/database/db_inbox-entry.md +++ b/doc/database/db_inbox-entry.md @@ -6,21 +6,22 @@ Incoming activity Fields ------ -| Field | Description | Type | Null | Key | Default | Extra | -| ------------------ | ------------------------------------ | -------------- | ---- | --- | ------- | -------------- | -| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment | -| activity-id | id of the incoming activity | varbinary(255) | YES | | NULL | | -| object-id | | varbinary(255) | YES | | NULL | | -| in-reply-to-id | | varbinary(255) | YES | | NULL | | -| conversation | | varbinary(255) | YES | | NULL | | -| type | Type of the activity | varchar(64) | YES | | NULL | | -| object-type | Type of the object activity | varchar(64) | YES | | NULL | | -| object-object-type | Type of the object's object activity | varchar(64) | YES | | NULL | | -| received | Receiving date | datetime | YES | | NULL | | -| activity | The JSON activity | mediumtext | YES | | NULL | | -| signer | | varchar(255) | YES | | NULL | | -| push | | boolean | NO | | 0 | | -| wid | Workerqueue id | int unsigned | YES | | NULL | | +| Field | Description | Type | Null | Key | Default | Extra | +| ------------------ | -------------------------------------- | -------------- | ---- | --- | ------- | -------------- | +| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment | +| activity-id | id of the incoming activity | varbinary(255) | YES | | NULL | | +| object-id | | varbinary(255) | YES | | NULL | | +| in-reply-to-id | | varbinary(255) | YES | | NULL | | +| conversation | | varbinary(255) | YES | | NULL | | +| type | Type of the activity | varchar(64) | YES | | NULL | | +| object-type | Type of the object activity | varchar(64) | YES | | NULL | | +| object-object-type | Type of the object's object activity | varchar(64) | YES | | NULL | | +| received | Receiving date | datetime | YES | | NULL | | +| activity | The JSON activity | mediumtext | YES | | NULL | | +| signer | | varchar(255) | YES | | NULL | | +| push | Is the entry pushed or have pulled it? | boolean | YES | | NULL | | +| trust | Do we trust this entry? | boolean | YES | | NULL | | +| wid | Workerqueue id | int unsigned | YES | | NULL | | Indexes ------------ diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index d1c3994e2a..1ca6fce902 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -303,7 +303,7 @@ class Processor Logger::notice('Parent not found. Try to refetch it.', ['parent' => $activity['reply-to-id'], 'recursion-depth' => $recursion_depth]); if ($recursion_depth < 10) { $result = self::fetchMissingActivity($activity['reply-to-id'], $activity, '', Receiver::COMPLETION_AUTO); - if (empty($result) && self::ActivityIsGone($activity['reply-to-id'])) { + if (empty($result) && self::isActivityGone($activity['reply-to-id'])) { // Recursively delete this and all depending entries Queue::deleteById($activity['entry-id']); return []; @@ -466,7 +466,7 @@ class Processor * * @return boolean */ - private static function ActivityIsGone(string $url): bool + private static function isActivityGone(string $url): bool { $curlResult = HTTPSignature::fetchRaw($url, 0); diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index 011101eed7..faaf0aa3bc 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -42,7 +42,7 @@ class Queue * @param boolean $push * @return array */ - public static function add(array $activity, string $type, int $uid, string $http_signer, bool $push): array + public static function add(array $activity, string $type, int $uid, string $http_signer, bool $push, bool $trust_source): array { $fields = [ 'activity-id' => $activity['id'], @@ -52,6 +52,7 @@ class Queue 'activity' => json_encode($activity, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT), 'received' => DateTimeFormat::utcNow(), 'push' => $push, + 'trust' => $trust_source, ]; if (!empty($activity['reply-to-id'])) { @@ -204,7 +205,7 @@ class Queue */ public static function processAll() { - $entries = DBA::select('inbox-entry', ['id', 'type', 'object-type', 'object-id', 'in-reply-to-id'], ["`wid` IS NULL"], ['order' => ['id' => true]]); + $entries = DBA::select('inbox-entry', ['id', 'type', 'object-type', 'object-id', 'in-reply-to-id'], ["`trust` AND `wid` IS NULL"], ['order' => ['id' => true]]); while ($entry = DBA::fetch($entries)) { // We don't need to process entries that depend on already existing entries. if (!empty($entry['in-reply-to-id']) && DBA::exists('inbox-entry', ["`id` != ? AND `object-id` = ?", $entry['id'], $entry['in-reply-to-id']])) { diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 3e34510749..84d1197514 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -530,11 +530,6 @@ class Receiver $type = $object_data['type']; } - if (!$trust_source) { - Logger::info('Activity trust could not be achieved.', ['id' => $object_data['object_id'], 'type' => $type, 'signer' => $signer, 'actor' => $actor, 'attributedTo' => $attributed_to]); - return; - } - if (!empty($body) && empty($object_data['raw'])) { $object_data['raw'] = $body; } @@ -561,7 +556,12 @@ class Receiver $object_data['object_activity'] = $activity; } - $object_data = Queue::add($object_data, $type, $uid, $http_signer, $push); + $object_data = Queue::add($object_data, $type, $uid, $http_signer, $push, $trust_source); + + if (!$trust_source) { + Logger::info('Activity trust could not be achieved.', ['id' => $object_data['object_id'], 'type' => $type, 'signer' => $signer, 'actor' => $actor, 'attributedTo' => $attributed_to]); + return; + } if (!empty($activity['recursion-depth'])) { $object_data['recursion-depth'] = $activity['recursion-depth']; diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 0b6af9b13a..53ceaa7ece 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -798,7 +798,8 @@ return [ "received" => ["type" => "datetime", "comment" => "Receiving date"], "activity" => ["type" => "mediumtext", "comment" => "The JSON activity"], "signer" => ["type" => "varchar(255)", "comment" => ""], - "push" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], + "push" => ["type" => "boolean", "comment" => "Is the entry pushed or have pulled it?"], + "trust" => ["type" => "boolean", "comment" => "Do we trust this entry?"], "wid" => ["type" => "int unsigned", "foreign" => ["workerqueue" => "id"], "comment" => "Workerqueue id"], ], "indexes" => [ "PRIMARY" => ["id"], From 5ab94ea921e430bd88d8bf263d1f1de90ac58b79 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 24 Jul 2022 19:31:31 +0000 Subject: [PATCH 26/26] Option to store untrusted entries --- src/Protocol/ActivityPub/Receiver.php | 37 +++++++++------------------ static/defaults.config.php | 4 +++ 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 84d1197514..2655da9f0e 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -273,6 +273,8 @@ class Receiver public static function prepareObjectData(array $activity, int $uid, bool $push, bool &$trust_source): array { $id = JsonLD::fetchElement($activity, '@id'); + $object_id = JsonLD::fetchElement($activity, 'as:object', '@id'); + if (!empty($id) && !$trust_source) { $fetch_uid = $uid ?: self::getBestUserForActivity($activity); @@ -283,7 +285,13 @@ class Receiver if ($fetched_id == $id) { Logger::info('Activity had been fetched successfully', ['id' => $id]); $trust_source = true; - $activity = $object; + if ($id != $object_id) { + $activity = $object; + } else { + Logger::info('Fetched data is the object instead of the activity', ['id' => $id]); + unset($object['@context']); + $activity['as:object'] = $object; + } } else { Logger::info('Activity id is not equal', ['id' => $id, 'fetched' => $fetched_id]); } @@ -556,7 +564,9 @@ class Receiver $object_data['object_activity'] = $activity; } - $object_data = Queue::add($object_data, $type, $uid, $http_signer, $push, $trust_source); + if ($trust_source || DI::config()->get('debug', 'ap_inbox_store_untrusted')) { + $object_data = Queue::add($object_data, $type, $uid, $http_signer, $push, $trust_source); + } if (!$trust_source) { Logger::info('Activity trust could not be achieved.', ['id' => $object_data['object_id'], 'type' => $type, 'signer' => $signer, 'actor' => $actor, 'attributedTo' => $attributed_to]); @@ -1830,29 +1840,6 @@ class Receiver unset($object_data['receiver'][-1]); unset($object_data['reception_type'][-1]); - // Common object data: - - // Unhandled - // @context, type, actor, signature, mediaType, duration, replies, icon - - // Also missing: (Defined in the standard, but currently unused) - // audience, preview, endTime, startTime, image - - // Data in Notes: - - // Unhandled - // contentMap, announcement_count, announcements, context_id, likes, like_count - // inReplyToStatusId, shares, quoteUrl, statusnetConversationId - - // Data in video: - - // To-Do? - // category, licence, language, commentsEnabled - - // Unhandled - // views, waitTranscoding, state, support, subtitleLanguage - // likes, dislikes, shares, comments - return $object_data; } } diff --git a/static/defaults.config.php b/static/defaults.config.php index c01172d55a..1876ea90d7 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -666,6 +666,10 @@ return [ // Logs every call to /inbox as a JSON file in Friendica's temporary directory 'ap_inbox_log' => false, + // ap_inbox_store_untrusted (Boolean) + // Store untrusted content in the inbox entries + 'ap_inbox_store_untrusted' => false, + // total_ap_delivery (Boolean) // Deliver via AP to every possible receiver and we suppress the delivery to these contacts with other protocols 'total_ap_delivery' => false,