From 13262395761fc781c04b443b824c0d8fc44f0c79 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 29 Apr 2022 05:32:12 +0000 Subject: [PATCH 01/35] Reduce the amount of queries --- src/Content/Conversation.php | 24 +++++++++++++++++------- src/Object/Post.php | 14 ++++---------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/Content/Conversation.php b/src/Content/Conversation.php index 7723ddd63f..7d17502428 100644 --- a/src/Content/Conversation.php +++ b/src/Content/Conversation.php @@ -108,6 +108,8 @@ class Conversation */ public function builtinActivityPuller(array $activity, array &$conv_responses) { + $thread_parent = $activity['thr-parent-row'] ?? []; + foreach ($conv_responses as $mode => $v) { $sparkle = ''; @@ -152,9 +154,8 @@ class Conversation $activity['thr-parent-id'] = $activity['parent-uri-id']; } - // Skip when the causer of the parent is the same than the author of the announce - if (($verb == Activity::ANNOUNCE) && Post::exists(['uri-id' => $activity['thr-parent-id'], - 'uid' => $activity['uid'], 'causer-id' => $activity['author-id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]])) { + // Skip when the causer of the parent is the same as the author of the announce + if (($verb == Activity::ANNOUNCE) && !empty($thread_parent['causer-id'] && ($thread_parent['causer-id'] == $activity['author-id']))) { continue; } @@ -809,12 +810,13 @@ class Conversation /** * Adds some information (Causer, post reason, direction) to the fetched post row. * - * @param array $row Post row - * @param array $activity Contact data of the resharer + * @param array $row Post row + * @param array $activity Contact data of the resharer + * @param array $thr_parent Thread parent row * * @return array items with parents and comments */ - private function addRowInformation(array $row, array $activity) + private function addRowInformation(array $row, array $activity, array $thr_parent) { $this->profiler->startRecording('rendering'); @@ -889,6 +891,8 @@ class Conversation break; } + $row['thr-parent-row'] = $thr_parent; + $this->profiler->stopRecording(); return $row; } @@ -954,6 +958,7 @@ class Conversation $thread_items = Post::selectForUser($uid, array_merge(ItemModel::DISPLAY_FIELDLIST, ['featured', 'contact-uid', 'gravity', 'post-type', 'post-reason']), $condition, $params); $items = []; + $thr_parent = []; while ($row = Post::fetch($thread_items)) { if (!empty($items[$row['uri-id']]) && ($row['uid'] == 0)) { @@ -968,7 +973,12 @@ class Conversation continue; } } - $items[$row['uri-id']] = $this->addRowInformation($row, $activities[$row['uri-id']] ?? []); + + if (empty($thr_parent[$row['thr-parent-id']])) { + $thr_parent[$row['thr-parent-id']] = Post::selectFirst(['causer-id'], ['uri-id' => $row['thr-parent-id'], 'uid' => $row['uid']]); + } + + $items[$row['uri-id']] = $this->addRowInformation($row, $activities[$row['uri-id']] ?? [], $thr_parent[$row['thr-parent-id']]); } DBA::close($thread_items); diff --git a/src/Object/Post.php b/src/Object/Post.php index 2cf02f8e2a..d84404634a 100644 --- a/src/Object/Post.php +++ b/src/Object/Post.php @@ -29,7 +29,6 @@ use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Core\Renderer; use Friendica\Core\Session; -use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Item; @@ -898,12 +897,7 @@ class Post } $owner = User::getOwnerDataById($a->getLoggedInUserId()); - - $item = PostModel::selectFirst(['author-addr', 'uri-id', 'network', 'gravity', 'content-warning'], ['id' => $this->getId()]); - if (!DBA::isResult($item) || empty($item['author-addr'])) { - // Should not happen - return ''; - } + $item = $this->getData(); if (!empty($item['content-warning']) && Feature::isEnabled(local_user(), 'add_abstract')) { $text = '[abstract=' . Protocol::ACTIVITYPUB . ']' . $item['content-warning'] . "[/abstract]\n"; @@ -967,7 +961,7 @@ class Post $uid = $conv->getProfileOwner(); $parent_uid = $this->getDataValue('uid'); - $contact = Contact::getById($a->getContactId()); + $owner = User::getOwnerDataById($a->getLoggedInUserId()); $default_text = $this->getDefaultText(); @@ -986,9 +980,9 @@ class Post '$qcomment' => $qcomment, '$default' => $default_text, '$profile_uid' => $uid, - '$mylink' => DI::baseUrl()->remove($contact['url'] ?? ''), + '$mylink' => DI::baseUrl()->remove($owner['url'] ?? ''), '$mytitle' => DI::l10n()->t('This is you'), - '$myphoto' => DI::baseUrl()->remove($contact['thumb'] ?? ''), + '$myphoto' => DI::baseUrl()->remove($owner['thumb'] ?? ''), '$comment' => DI::l10n()->t('Comment'), '$submit' => DI::l10n()->t('Submit'), '$loading' => DI::l10n()->t('Loading...'), From d44641e58c0cfa96c9c7b093c45854da87015b63 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 29 Apr 2022 07:30:13 +0000 Subject: [PATCH 02/35] Fetch avatar by id --- database.sql | 10 +++++++++- src/Content/Conversation.php | 14 ++++++++++++-- src/Model/Contact.php | 2 +- src/Model/Item.php | 4 ++-- src/Object/Post.php | 14 ++++++++++++-- static/dbstructure.config.php | 2 +- static/dbview.config.php | 8 ++++++++ 7 files changed, 45 insertions(+), 9 deletions(-) diff --git a/database.sql b/database.sql index 7511672f30..9952f8c3d2 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2022.05-dev (Siberian Iris) --- DB_UPDATE_VERSION 1460 +-- DB_UPDATE_VERSION 1461 -- ------------------------------------------ @@ -1719,6 +1719,7 @@ CREATE VIEW `post-user-view` AS SELECT `author`.`network` AS `author-network`, `author`.`blocked` AS `author-blocked`, `author`.`hidden` AS `author-hidden`, + `author`.`updated` AS `author-updated`, `post-user`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -1728,6 +1729,7 @@ CREATE VIEW `post-user-view` AS SELECT `owner`.`network` AS `owner-network`, `owner`.`blocked` AS `owner-blocked`, `owner`.`hidden` AS `owner-hidden`, + `owner`.`updated` AS `owner-updated`, `owner`.`contact-type` AS `owner-contact-type`, `post-user`.`causer-id` AS `causer-id`, `causer`.`url` AS `causer-link`, @@ -1884,6 +1886,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT `author`.`network` AS `author-network`, `author`.`blocked` AS `author-blocked`, `author`.`hidden` AS `author-hidden`, + `author`.`updated` AS `author-updated`, `post-thread-user`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -1893,6 +1896,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT `owner`.`network` AS `owner-network`, `owner`.`blocked` AS `owner-blocked`, `owner`.`hidden` AS `owner-hidden`, + `owner`.`updated` AS `owner-updated`, `owner`.`contact-type` AS `owner-contact-type`, `post-thread-user`.`causer-id` AS `causer-id`, `causer`.`url` AS `causer-link`, @@ -2035,6 +2039,7 @@ CREATE VIEW `post-view` AS SELECT `author`.`network` AS `author-network`, `author`.`blocked` AS `author-blocked`, `author`.`hidden` AS `author-hidden`, + `author`.`updated` AS `author-updated`, `post`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -2044,6 +2049,7 @@ CREATE VIEW `post-view` AS SELECT `owner`.`network` AS `owner-network`, `owner`.`blocked` AS `owner-blocked`, `owner`.`hidden` AS `owner-hidden`, + `owner`.`updated` AS `owner-updated`, `owner`.`contact-type` AS `owner-contact-type`, `post`.`causer-id` AS `causer-id`, `causer`.`url` AS `causer-link`, @@ -2162,6 +2168,7 @@ CREATE VIEW `post-thread-view` AS SELECT `author`.`network` AS `author-network`, `author`.`blocked` AS `author-blocked`, `author`.`hidden` AS `author-hidden`, + `author`.`updated` AS `author-updated`, `post-thread`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -2171,6 +2178,7 @@ CREATE VIEW `post-thread-view` AS SELECT `owner`.`network` AS `owner-network`, `owner`.`blocked` AS `owner-blocked`, `owner`.`hidden` AS `owner-hidden`, + `owner`.`updated` AS `owner-updated`, `owner`.`contact-type` AS `owner-contact-type`, `post-thread`.`causer-id` AS `causer-id`, `causer`.`url` AS `causer-link`, diff --git a/src/Content/Conversation.php b/src/Content/Conversation.php index 7d17502428..fd880a0452 100644 --- a/src/Content/Conversation.php +++ b/src/Content/Conversation.php @@ -659,6 +659,16 @@ class Conversation $pinned = ''; } + if (in_array($item['network'], [Protocol::FEED, Protocol::MAIL])) { + $owner_avatar = $author_avatar = $item['contact-id']; + $owner_updated = $author_updated = ''; + } else { + $owner_avatar = $item['owner-id']; + $owner_updated = $item['owner-updated']; + $author_avatar = $item['author-id']; + $author_updated = $item['author-updated']; + } + $tmp_item = [ 'template' => $tpl, 'id' => ($preview ? 'P0' : $item['id']), @@ -676,7 +686,7 @@ class Conversation 'name' => $profile_name, 'sparkle' => $sparkle, 'lock' => false, - 'thumb' => $this->baseURL->remove(Contact::getAvatarUrlForUrl($item['author-link'], $item['uid'], Proxy::SIZE_THUMB)), + 'thumb' => $this->baseURL->remove(Contact::getAvatarUrlForId($author_avatar, Proxy::SIZE_THUMB, $author_updated)), 'title' => $title, 'body_html' => $body_html, 'tags' => $tags['tags'], @@ -697,7 +707,7 @@ class Conversation 'indent' => '', 'owner_name' => '', 'owner_url' => '', - 'owner_photo' => $this->baseURL->remove(Contact::getAvatarUrlForUrl($item['owner-link'], $item['uid'], Proxy::SIZE_THUMB)), + 'owner_photo' => $this->baseURL->remove(Contact::getAvatarUrlForId($owner_avatar, Proxy::SIZE_THUMB, $owner_updated)), 'plink' => ItemModel::getPlink($item), 'edpost' => false, 'pinned' => $pinned, diff --git a/src/Model/Contact.php b/src/Model/Contact.php index 377d734a24..ab1737d42f 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -1799,7 +1799,7 @@ class Contact { // We have to fetch the "updated" variable when it wasn't provided // The parameter can be provided to improve performance - if (empty($updated) || empty($guid)) { + if (empty($updated)) { $account = DBA::selectFirst('account-user-view', ['updated', 'guid'], ['id' => $cid]); $updated = $account['updated'] ?? ''; $guid = $account['guid'] ?? ''; diff --git a/src/Model/Item.php b/src/Model/Item.php index 34f7e0d8b8..3900c557c1 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -87,8 +87,8 @@ class Item 'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'mention', 'global', - 'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network', - 'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'owner-network', 'owner-contact-type', + 'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network', 'author-updated', + 'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'owner-network', 'owner-contact-type', 'owner-updated', 'causer-id', 'causer-link', 'causer-name', 'causer-avatar', 'causer-contact-type', 'causer-network', 'contact-id', 'contact-uid', 'contact-link', 'contact-name', 'contact-avatar', 'writable', 'self', 'cid', 'alias', diff --git a/src/Object/Post.php b/src/Object/Post.php index d84404634a..7f5adde0ae 100644 --- a/src/Object/Post.php +++ b/src/Object/Post.php @@ -450,6 +450,16 @@ class Post $browsershare = null; } + if (in_array($item['network'], [Protocol::FEED, Protocol::MAIL])) { + $owner_avatar = $author_avatar = $item['contact-id']; + $owner_updated = $author_updated = ''; + } else { + $owner_avatar = $item['owner-id']; + $owner_updated = $item['owner-updated']; + $author_avatar = $item['author-id']; + $author_updated = $item['author-updated']; + } + $tmp_item = [ 'template' => $this->getTemplate(), 'type' => implode("", array_slice(explode("/", $item['verb']), -1)), @@ -481,7 +491,7 @@ class Post 'profile_url' => $profile_link, 'name' => $profile_name, 'item_photo_menu_html' => DI::contentItem()->photoMenu($item, $formSecurityToken), - 'thumb' => DI::baseUrl()->remove(Contact::getAvatarUrlForUrl($item['author-link'], $item['uid'], Proxy::SIZE_THUMB)), + 'thumb' => DI::baseUrl()->remove(Contact::getAvatarUrlForId($author_avatar, Proxy::SIZE_THUMB, $author_updated)), 'osparkle' => $osparkle, 'sparkle' => $sparkle, 'title' => $title, @@ -498,7 +508,7 @@ class Post 'shiny' => $shiny, 'owner_self' => $item['author-link'] == Session::get('my_url'), 'owner_url' => $this->getOwnerUrl(), - 'owner_photo' => DI::baseUrl()->remove(Contact::getAvatarUrlForUrl($item['owner-link'], $item['uid'], Proxy::SIZE_THUMB)), + 'owner_photo' => DI::baseUrl()->remove(Contact::getAvatarUrlForId($owner_avatar, Proxy::SIZE_THUMB, $owner_updated)), 'owner_name' => $this->getOwnerName(), 'plink' => Item::getPlink($item), 'browsershare' => $browsershare, diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 41758ed4b4..03748f36a5 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1460); + define('DB_UPDATE_VERSION', 1461); } return [ diff --git a/static/dbview.config.php b/static/dbview.config.php index 04fc406b43..3faec770f6 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -146,6 +146,7 @@ "author-network" => ["author", "network"], "author-blocked" => ["author", "blocked"], "author-hidden" => ["author", "hidden"], + "author-updated" => ["author", "updated"], "owner-id" => ["post-user", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -155,6 +156,7 @@ "owner-network" => ["owner", "network"], "owner-blocked" => ["owner", "blocked"], "owner-hidden" => ["owner", "hidden"], + "owner-updated" => ["owner", "updated"], "owner-contact-type" => ["owner", "contact-type"], "causer-id" => ["post-user", "causer-id"], "causer-link" => ["causer", "url"], @@ -309,6 +311,7 @@ "author-network" => ["author", "network"], "author-blocked" => ["author", "blocked"], "author-hidden" => ["author", "hidden"], + "author-updated" => ["author", "updated"], "owner-id" => ["post-thread-user", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -318,6 +321,7 @@ "owner-network" => ["owner", "network"], "owner-blocked" => ["owner", "blocked"], "owner-hidden" => ["owner", "hidden"], + "owner-updated" => ["owner", "updated"], "owner-contact-type" => ["owner", "contact-type"], "causer-id" => ["post-thread-user", "causer-id"], "causer-link" => ["causer", "url"], @@ -458,6 +462,7 @@ "author-network" => ["author", "network"], "author-blocked" => ["author", "blocked"], "author-hidden" => ["author", "hidden"], + "author-updated" => ["author", "updated"], "owner-id" => ["post", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -467,6 +472,7 @@ "owner-network" => ["owner", "network"], "owner-blocked" => ["owner", "blocked"], "owner-hidden" => ["owner", "hidden"], + "owner-updated" => ["owner", "updated"], "owner-contact-type" => ["owner", "contact-type"], "causer-id" => ["post", "causer-id"], "causer-link" => ["causer", "url"], @@ -583,6 +589,7 @@ "author-network" => ["author", "network"], "author-blocked" => ["author", "blocked"], "author-hidden" => ["author", "hidden"], + "author-updated" => ["author", "updated"], "owner-id" => ["post-thread", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -592,6 +599,7 @@ "owner-network" => ["owner", "network"], "owner-blocked" => ["owner", "blocked"], "owner-hidden" => ["owner", "hidden"], + "owner-updated" => ["owner", "updated"], "owner-contact-type" => ["owner", "contact-type"], "causer-id" => ["post-thread", "causer-id"], "causer-link" => ["causer", "url"], From d3de2497bcaf8f6547b9ec14fc7e66d2b04ff97d Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 29 Apr 2022 07:47:24 +0000 Subject: [PATCH 03/35] Use gsid for the network name --- database.sql | 4 ++++ src/Content/ContactSelector.php | 11 +++++++++-- src/Content/Conversation.php | 4 ++-- src/Model/Item.php | 2 +- src/Object/Post.php | 4 ++-- static/dbview.config.php | 4 ++++ 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/database.sql b/database.sql index 9952f8c3d2..785b0f7b54 100644 --- a/database.sql +++ b/database.sql @@ -1720,6 +1720,7 @@ CREATE VIEW `post-user-view` AS SELECT `author`.`blocked` AS `author-blocked`, `author`.`hidden` AS `author-hidden`, `author`.`updated` AS `author-updated`, + `author`.`gsid` AS `author-gsid`, `post-user`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -1887,6 +1888,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT `author`.`blocked` AS `author-blocked`, `author`.`hidden` AS `author-hidden`, `author`.`updated` AS `author-updated`, + `author`.`gsid` AS `author-gsid`, `post-thread-user`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -2040,6 +2042,7 @@ CREATE VIEW `post-view` AS SELECT `author`.`blocked` AS `author-blocked`, `author`.`hidden` AS `author-hidden`, `author`.`updated` AS `author-updated`, + `author`.`gsid` AS `author-gsid`, `post`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -2169,6 +2172,7 @@ CREATE VIEW `post-thread-view` AS SELECT `author`.`blocked` AS `author-blocked`, `author`.`hidden` AS `author-hidden`, `author`.`updated` AS `author-updated`, + `author`.`gsid` AS `author-gsid`, `post-thread`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, diff --git a/src/Content/ContactSelector.php b/src/Content/ContactSelector.php index 9035d37e73..6c7e09945f 100644 --- a/src/Content/ContactSelector.php +++ b/src/Content/ContactSelector.php @@ -184,7 +184,7 @@ class ContactSelector * @return string * @throws \Exception */ - public static function networkToIcon($network, $profile = "") + public static function networkToIcon($network, $profile = "", $gsid = 0) { $nets = [ Protocol::DFRN => 'friendica', @@ -218,7 +218,14 @@ class ContactSelector $network_icon = str_replace($search, $replace, $network); if ((in_array($network, Protocol::FEDERATED)) && ($profile != "")) { - $gserver = self::getServerForProfile($profile); + if (!empty($gsid) && !empty(self::$serverdata[$gsid])) { + $gserver = self::$serverdata[$gsid]; + } elseif (!empty($gsid)) { + $gserver = DBA::selectFirst('gserver', ['platform', 'network'], ['id' => $gsid]); + self::$serverdata[$gsid] = $gserver; + } else { + $gserver = self::getServerForProfile($profile); + } if (!empty($gserver['platform'])) { $network_icon = $platform_icons[strtolower($gserver['platform'])] ?? $network_icon; } diff --git a/src/Content/Conversation.php b/src/Content/Conversation.php index fd880a0452..48695b2193 100644 --- a/src/Content/Conversation.php +++ b/src/Content/Conversation.php @@ -678,8 +678,8 @@ class Conversation 'created_date' => $item['created'], 'uriid' => $item['uri-id'], 'network' => $item['network'], - 'network_name' => ContactSelector::networkToName($item['author-network'], $item['author-link'], $item['network']), - 'network_icon' => ContactSelector::networkToIcon($item['network'], $item['author-link']), + 'network_name' => ContactSelector::networkToName($item['author-network'], $item['author-link'], $item['network'], $item['author-gsid']), + 'network_icon' => ContactSelector::networkToIcon($item['network'], $item['author-link'], $item['author-gsid']), 'linktitle' => $this->l10n->t('View %s\'s profile @ %s', $profile_name, $item['author-link']), 'profile_url' => $profile_link, 'item_photo_menu_html' => $this->item->photoMenu($item, $formSecurityToken), diff --git a/src/Model/Item.php b/src/Model/Item.php index 3900c557c1..84fa0af11b 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -87,7 +87,7 @@ class Item 'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'mention', 'global', - 'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network', 'author-updated', + 'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network', 'author-updated', 'author-gsid', 'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'owner-network', 'owner-contact-type', 'owner-updated', 'causer-id', 'causer-link', 'causer-name', 'causer-avatar', 'causer-contact-type', 'causer-network', 'contact-id', 'contact-uid', 'contact-link', 'contact-name', 'contact-avatar', diff --git a/src/Object/Post.php b/src/Object/Post.php index 7f5adde0ae..a949c24aa7 100644 --- a/src/Object/Post.php +++ b/src/Object/Post.php @@ -538,8 +538,8 @@ class Post 'thread_level' => $thread_level, 'edited' => $edited, 'network' => $item["network"], - 'network_name' => ContactSelector::networkToName($item['author-network'], $item['author-link'], $item['network']), - 'network_icon' => ContactSelector::networkToIcon($item['network'], $item['author-link']), + 'network_name' => ContactSelector::networkToName($item['author-network'], $item['author-link'], $item['network'], $item['author-gsid']), + 'network_icon' => ContactSelector::networkToIcon($item['network'], $item['author-link'], $item['author-gsid']), 'received' => $item['received'], 'commented' => $item['commented'], 'created_date' => $item['created'], diff --git a/static/dbview.config.php b/static/dbview.config.php index 3faec770f6..a623e46365 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -147,6 +147,7 @@ "author-blocked" => ["author", "blocked"], "author-hidden" => ["author", "hidden"], "author-updated" => ["author", "updated"], + "author-gsid" => ["author", "gsid"], "owner-id" => ["post-user", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -312,6 +313,7 @@ "author-blocked" => ["author", "blocked"], "author-hidden" => ["author", "hidden"], "author-updated" => ["author", "updated"], + "author-gsid" => ["author", "gsid"], "owner-id" => ["post-thread-user", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -463,6 +465,7 @@ "author-blocked" => ["author", "blocked"], "author-hidden" => ["author", "hidden"], "author-updated" => ["author", "updated"], + "author-gsid" => ["author", "gsid"], "owner-id" => ["post", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -590,6 +593,7 @@ "author-blocked" => ["author", "blocked"], "author-hidden" => ["author", "hidden"], "author-updated" => ["author", "updated"], + "author-gsid" => ["author", "gsid"], "owner-id" => ["post-thread", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], From 8dc0ab9bb3379fec0be442160db1ae29ef8cea9d Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 29 Apr 2022 09:49:16 +0000 Subject: [PATCH 04/35] Test: Disable magiclink in posts --- src/Content/Item.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Content/Item.php b/src/Content/Item.php index f6906b7962..5ebd51c1e0 100644 --- a/src/Content/Item.php +++ b/src/Content/Item.php @@ -352,7 +352,7 @@ class Item $item['body'] = $this->l10n->t('%1$s tagged %2$s\'s %3$s with %4$s', $author, $objauthor, $plink, $tag); } } - +/* $matches = null; if (preg_match_all('/@\[url=(.*?)\]/is', $item['body'], $matches, PREG_SET_ORDER)) { foreach ($matches as $mtch) { @@ -361,7 +361,7 @@ class Item } } } - +*/ // add sparkle links to appropriate permalinks // Only create a redirection to a magic link when logged in if (!empty($item['plink']) && Session::isAuthenticated() && $item['private'] == ModelItem::PRIVATE) { From a2452b33eb121a083e22f0edabd45048a7e83e99 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 29 Apr 2022 10:13:23 +0000 Subject: [PATCH 05/35] Add missing fields to collection --- database.sql | 9 +++++++-- src/Model/Item.php | 2 +- static/dbview.config.php | 7 ++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/database.sql b/database.sql index 785b0f7b54..7a8ee0adb7 100644 --- a/database.sql +++ b/database.sql @@ -2246,9 +2246,14 @@ CREATE VIEW `collection-view` AS SELECT `post-collection`.`type` AS `type`, `post`.`author-id` AS `cid`, `post`.`received` AS `received`, - `post`.`created` AS `created` + `post`.`created` AS `created`, + `post-thread`.`commented` AS `commented`, + `post`.`thr-parent-id` AS `thr-parent-id`, + `post`.`author-id` AS `author-id`, + `post`.`gravity` AS `gravity` FROM `post-collection` - INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id`; + INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id` + INNER JOIN `post-thread` ON `post-thread`.`uri-id` = `post`.`parent-uri-id`; -- -- VIEW tag-view diff --git a/src/Model/Item.php b/src/Model/Item.php index 84fa0af11b..b63b98d773 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -87,7 +87,7 @@ class Item 'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'mention', 'global', - 'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network', 'author-updated', 'author-gsid', + 'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network', 'author-updated', 'author-gsid', 'author-addr', 'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'owner-network', 'owner-contact-type', 'owner-updated', 'causer-id', 'causer-link', 'causer-name', 'causer-avatar', 'causer-contact-type', 'causer-network', 'contact-id', 'contact-uid', 'contact-link', 'contact-name', 'contact-avatar', diff --git a/static/dbview.config.php b/static/dbview.config.php index a623e46365..ea991489d6 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -664,9 +664,14 @@ "cid" => ["post", "author-id"], "received" => ["post", "received"], "created" => ["post", "created"], + "commented" => ["post-thread", "commented"], + "thr-parent-id" => ["post", "thr-parent-id"], + "author-id" => ["post", "author-id"], + "gravity" => ["post", "gravity"], ], "query" => "FROM `post-collection` - INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id`" + INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id` + INNER JOIN `post-thread` ON `post-thread`.`uri-id` = `post`.`parent-uri-id`" ], "tag-view" => [ "fields" => [ From fdf70c10478d13ccc98489f7185ec493736e0af4 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 29 Apr 2022 12:34:10 +0000 Subject: [PATCH 06/35] Make post preview workable --- mod/item.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mod/item.php b/mod/item.php index 070934fb50..498cf27132 100644 --- a/mod/item.php +++ b/mod/item.php @@ -621,6 +621,9 @@ function item_post(App $a) { $datarray["id"] = -1; $datarray["uri-id"] = -1; $datarray["author-network"] = Protocol::DFRN; + $datarray["author-updated"] = ''; + $datarray["author-gsid"] = 0; + $datarray["owner-updated"] = ''; $o = DI::conversation()->create([array_merge($contact_record, $datarray)], 'search', false, true); From fb3353d4bd85fa1f3af8d4c3b6a2a63f41fc0cc3 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 30 Apr 2022 06:19:18 +0000 Subject: [PATCH 07/35] Separate loop to fetch thread parents --- src/Content/Conversation.php | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Content/Conversation.php b/src/Content/Conversation.php index 48695b2193..9f93d0efad 100644 --- a/src/Content/Conversation.php +++ b/src/Content/Conversation.php @@ -930,8 +930,6 @@ class Conversation $max_comments = $this->config->get('system', 'max_display_comments', 1000); } - $params = ['order' => ['uri-id' => true, 'uid' => true]]; - $activities = []; $uriids = []; $commentcounter = []; @@ -965,10 +963,21 @@ class Conversation $condition = DBA::mergeConditions($condition, ["`uid` IN (0, ?) AND (`vid` != ? OR `vid` IS NULL)", $uid, Verb::getID(Activity::FOLLOW)]); + $params = ['order' => ['uri-id' => false, 'uid' => true]]; + $thread_parents = Post::select(['uri-id', 'causer-id'], $condition, $params); + + $thr_parent = []; + + while ($row = Post::fetch($thread_parents)) { + $thr_parent[$row['uri-id']] = $row; + } + DBA::close($thread_parents); + + $params = ['order' => ['uri-id' => true, 'uid' => true]]; + $thread_items = Post::selectForUser($uid, array_merge(ItemModel::DISPLAY_FIELDLIST, ['featured', 'contact-uid', 'gravity', 'post-type', 'post-reason']), $condition, $params); $items = []; - $thr_parent = []; while ($row = Post::fetch($thread_items)) { if (!empty($items[$row['uri-id']]) && ($row['uid'] == 0)) { @@ -984,11 +993,7 @@ class Conversation } } - if (empty($thr_parent[$row['thr-parent-id']])) { - $thr_parent[$row['thr-parent-id']] = Post::selectFirst(['causer-id'], ['uri-id' => $row['thr-parent-id'], 'uid' => $row['uid']]); - } - - $items[$row['uri-id']] = $this->addRowInformation($row, $activities[$row['uri-id']] ?? [], $thr_parent[$row['thr-parent-id']]); + $items[$row['uri-id']] = $this->addRowInformation($row, $activities[$row['uri-id']] ?? [], $thr_parent[$row['thr-parent-id']] ?? []); } DBA::close($thread_items); From 1f43332a1ddf62850a91a19dfda202b76a2b964d Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 30 Apr 2022 06:57:22 +0000 Subject: [PATCH 08/35] Only fetch category when it exists --- src/Content/Item.php | 4 ++++ src/Model/Post/Category.php | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/Content/Item.php b/src/Content/Item.php index 5ebd51c1e0..0c60c4e26f 100644 --- a/src/Content/Item.php +++ b/src/Content/Item.php @@ -93,6 +93,10 @@ class Item $uid = $item['uid'] ?: $uid; + if (!Post\Category::existsForURIId($item['uri-id'], $uid)) { + return [$categories, $folders]; + } + foreach (Post\Category::getArrayByURIId($item['uri-id'], $uid, Post\Category::CATEGORY) as $savedFolderName) { if (!empty($item['author-link'])) { $url = $item['author-link'] . "?category=" . rawurlencode($savedFolderName); diff --git a/src/Model/Post/Category.php b/src/Model/Post/Category.php index 9d2a359ba3..60a33bd748 100644 --- a/src/Model/Post/Category.php +++ b/src/Model/Post/Category.php @@ -111,6 +111,11 @@ class Category return array_column($tags, 'name'); } + public static function existsForURIId(int $uri_id, int $uid) + { + return DBA::exists('post-category', ['uri-id' => $uri_id, 'uid' => $uid]); + } + /** * Generates an array of files or categories of a given uri-id * From e19681684b382721158a0cb4c20840b74ea25161 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 1 May 2022 06:57:29 +0000 Subject: [PATCH 09/35] Fix worker priorities --- src/Core/Worker.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 5c23d64456..8f79cc14df 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -661,7 +661,8 @@ class Worker } else { // Kill long running processes // Check if the priority is in a valid range - if (!in_array($entry["priority"], [PRIORITY_CRITICAL, PRIORITY_HIGH, PRIORITY_MEDIUM, PRIORITY_LOW, PRIORITY_NEGLIGIBLE])) { + if (!in_array($entry["priority"], PRIORITIES)) { + Logger::warning('Invalid priority', ['entry' => $entry, 'callstack' => System::callstack(20)]); $entry["priority"] = PRIORITY_MEDIUM; } @@ -1391,6 +1392,11 @@ class Worker $id = $queue['id']; $priority = $queue['priority']; + if (!in_array($priority, PRIORITIES)) { + Logger::warning('Invalid priority', ['queue' => $queue, 'callstack' => System::callstack(20)]); + $priority = PRIORITY_MEDIUM; + } + $max_level = DI::config()->get('system', 'worker_defer_limit'); $new_retrial = self::getNextRetrial($queue, $max_level); From 19c1b31ab2e81be97af3c776d390e512d17dd3f9 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 1 May 2022 07:03:10 +0000 Subject: [PATCH 10/35] Only add valid values to the worker --- src/Model/Contact.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Model/Contact.php b/src/Model/Contact.php index ab1737d42f..c6e8b608fc 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -828,7 +828,9 @@ class Contact if (in_array($contact['rel'], [self::SHARING, self::FRIEND])) { $cdata = Contact::getPublicAndUserContactID($contact['id'], $contact['uid']); - Worker::add(PRIORITY_HIGH, 'Contact\Unfollow', $cdata['public'], $contact['uid']); + if (!empty($cdata['public'])) { + Worker::add(PRIORITY_HIGH, 'Contact\Unfollow', $cdata['public'], $contact['uid']); + } } self::removeSharer($contact); @@ -855,7 +857,9 @@ class Contact if (in_array($contact['rel'], [self::FOLLOWER, self::FRIEND])) { $cdata = Contact::getPublicAndUserContactID($contact['id'], $contact['uid']); - Worker::add(PRIORITY_HIGH, 'Contact\RevokeFollow', $cdata['public'], $contact['uid']); + if (!empty($cdata['public'])) { + Worker::add(PRIORITY_HIGH, 'Contact\RevokeFollow', $cdata['public'], $contact['uid']); + } } self::removeFollower($contact); @@ -880,11 +884,11 @@ class Contact $cdata = Contact::getPublicAndUserContactID($contact['id'], $contact['uid']); - if (in_array($contact['rel'], [self::SHARING, self::FRIEND])) { + if (in_array($contact['rel'], [self::SHARING, self::FRIEND]) && !empty($cdata['public'])) { Worker::add(PRIORITY_HIGH, 'Contact\Unfollow', $cdata['public'], $contact['uid']); } - if (in_array($contact['rel'], [self::FOLLOWER, self::FRIEND])) { + if (in_array($contact['rel'], [self::FOLLOWER, self::FRIEND]) && !empty($cdata['public'])) { Worker::add(PRIORITY_HIGH, 'Contact\RevokeFollow', $cdata['public'], $contact['uid']); } From f7b6507438a551975016076784ef90079d1662fa Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 1 May 2022 08:58:48 +0000 Subject: [PATCH 11/35] More checks for strange priorities --- src/Core/Worker.php | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 8f79cc14df..3443e6608e 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -107,6 +107,10 @@ class Worker foreach ($r as $entry) { // Assure that the priority is an integer value $entry['priority'] = (int)$entry['priority']; + if (!in_array($entry['priority'], PRIORITIES)) { + Logger::warning('Invalid priority', ['entry' => $entry, 'callstack' => System::callstack(20)]); + $entry['priority'] = PRIORITY_MEDIUM; + } // The work will be done if (!self::execute($entry)) { @@ -261,7 +265,7 @@ class Worker $workerqueue = DBA::selectFirst('workerqueue', ['priority'], $condition, ['order' => ['priority']]); self::$db_duration += (microtime(true) - $stamp); if (DBA::isResult($workerqueue)) { - return $workerqueue["priority"]; + return $workerqueue['priority']; } else { return 0; } @@ -466,13 +470,13 @@ class Worker $cooldown = DI::config()->get("system", "worker_cooldown", 0); if ($cooldown > 0) { - Logger::info('Pre execution cooldown.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'cooldown' => $cooldown]); + Logger::info('Pre execution cooldown.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'cooldown' => $cooldown]); sleep($cooldown); } Logger::enableWorker($funcname); - Logger::info("Process start.", ['priority' => $queue["priority"], 'id' => $queue["id"]]); + Logger::info("Process start.", ['priority' => $queue['priority'], 'id' => $queue["id"]]); $stamp = (float)microtime(true); @@ -529,21 +533,21 @@ class Worker self::$lock_duration = 0; if ($duration > 3600) { - Logger::info('Longer than 1 hour.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]); + Logger::info('Longer than 1 hour.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]); } elseif ($duration > 600) { - Logger::info('Longer than 10 minutes.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]); + Logger::info('Longer than 10 minutes.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]); } elseif ($duration > 300) { - Logger::info('Longer than 5 minutes.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]); + Logger::info('Longer than 5 minutes.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]); } elseif ($duration > 120) { - Logger::info('Longer than 2 minutes.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]); + Logger::info('Longer than 2 minutes.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration/60, 3)]); } - Logger::info('Process done.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'duration' => round($duration, 3)]); + Logger::info('Process done.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'duration' => round($duration, 3)]); DI::profiler()->saveLog(DI::logger(), "ID " . $queue["id"] . ": " . $funcname); if ($cooldown > 0) { - Logger::info('Post execution cooldown.', ['priority' => $queue["priority"], 'id' => $queue["id"], 'cooldown' => $cooldown]); + Logger::info('Post execution cooldown.', ['priority' => $queue['priority'], 'id' => $queue["id"], 'cooldown' => $cooldown]); sleep($cooldown); } } @@ -661,14 +665,14 @@ class Worker } else { // Kill long running processes // Check if the priority is in a valid range - if (!in_array($entry["priority"], PRIORITIES)) { + if (!in_array($entry['priority'], PRIORITIES)) { Logger::warning('Invalid priority', ['entry' => $entry, 'callstack' => System::callstack(20)]); - $entry["priority"] = PRIORITY_MEDIUM; + $entry['priority'] = PRIORITY_MEDIUM; } // Define the maximum durations $max_duration_defaults = [PRIORITY_CRITICAL => 720, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 720]; - $max_duration = $max_duration_defaults[$entry["priority"]]; + $max_duration = $max_duration_defaults[$entry['priority']]; $argv = json_decode($entry['parameter'], true); if (!empty($entry['command'])) { @@ -690,12 +694,12 @@ class Worker // We killed the stale process. // To avoid a blocking situation we reschedule the process at the beginning of the queue. // Additionally we are lowering the priority. (But not PRIORITY_CRITICAL) - $new_priority = $entry["priority"]; - if ($entry["priority"] == PRIORITY_HIGH) { + $new_priority = $entry['priority']; + if ($entry['priority'] == PRIORITY_HIGH) { $new_priority = PRIORITY_MEDIUM; - } elseif ($entry["priority"] == PRIORITY_MEDIUM) { + } elseif ($entry['priority'] == PRIORITY_MEDIUM) { $new_priority = PRIORITY_LOW; - } elseif ($entry["priority"] != PRIORITY_CRITICAL) { + } elseif ($entry['priority'] != PRIORITY_CRITICAL) { $new_priority = PRIORITY_NEGLIGIBLE; } $stamp = (float)microtime(true); @@ -779,12 +783,12 @@ class Worker self::$db_duration_stat += (microtime(true) - $stamp); while ($entry = DBA::fetch($jobs)) { $stamp = (float)microtime(true); - $running = DBA::count('workerqueue-view', ['priority' => $entry["priority"]]); + $running = DBA::count('workerqueue-view', ['priority' => $entry['priority']]); self::$db_duration += (microtime(true) - $stamp); self::$db_duration_stat += (microtime(true) - $stamp); $idle_workers -= $running; $waiting_processes += $entry["entries"]; - $listitem[$entry["priority"]] = $entry["priority"] . ":" . $running . "/" . $entry["entries"]; + $listitem[$entry['priority']] = $entry['priority'] . ":" . $running . "/" . $entry["entries"]; } DBA::close($jobs); } else { @@ -796,7 +800,7 @@ class Worker while ($entry = DBA::fetch($jobs)) { $idle_workers -= $entry["running"]; - $listitem[$entry["priority"]] = $entry["priority"].":".$entry["running"]; + $listitem[$entry['priority']] = $entry['priority'].":".$entry["running"]; } DBA::close($jobs); } From 13e4144ba6b65c8aaeb8f8356b3e24e00c5c526b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 1 May 2022 09:29:31 +0000 Subject: [PATCH 12/35] Use a centralized function to check the priority --- src/Core/Worker.php | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 3443e6608e..5bd281638f 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -22,7 +22,6 @@ namespace Friendica\Core; use Friendica\App\Mode; -use Friendica\Core; use Friendica\Core\Worker\Entity\Process; use Friendica\Database\DBA; use Friendica\DI; @@ -105,12 +104,7 @@ class Worker // Don't refetch when a worker fetches tasks for multiple workers $refetched = DI::config()->get('system', 'worker_multiple_fetch'); foreach ($r as $entry) { - // Assure that the priority is an integer value - $entry['priority'] = (int)$entry['priority']; - if (!in_array($entry['priority'], PRIORITIES)) { - Logger::warning('Invalid priority', ['entry' => $entry, 'callstack' => System::callstack(20)]); - $entry['priority'] = PRIORITY_MEDIUM; - } + $entry = self::checkPriority($entry); // The work will be done if (!self::execute($entry)) { @@ -172,6 +166,24 @@ class Worker Logger::info("Couldn't select a workerqueue entry, quitting process", ['pid' => getmypid()]); } + /** + * Check and fix the priority of a worker task + * @param array $entry + * @return array + */ + private static function checkPriority(array $entry) + { + $entry['priority'] = (int)$entry['priority']; + + if (!in_array($entry['priority'], PRIORITIES)) { + Logger::warning('Invalid priority', ['entry' => $entry, 'callstack' => System::callstack(20)]); + DBA::update('workerqueue', ['priority' => PRIORITY_MEDIUM], ['id' => $entry['id']]); + $entry['priority'] = PRIORITY_MEDIUM; + } + + return $entry; + } + /** * Checks if the system is ready. * @@ -484,11 +496,6 @@ class Worker // For this reason the variables have to be initialized. DI::profiler()->reset(); - if (!in_array($queue['priority'], PRIORITIES)) { - Logger::warning('Invalid priority', ['queue' => $queue, 'callstack' => System::callstack(20)]); - $queue['priority'] = PRIORITY_MEDIUM; - } - $a->setQueue($queue); $up_duration = microtime(true) - self::$up_start; @@ -653,6 +660,8 @@ class Worker self::$db_duration += (microtime(true) - $stamp); while ($entry = DBA::fetch($entries)) { + $entry = self::checkPriority($entry); + if (!posix_kill($entry["pid"], 0)) { $stamp = (float)microtime(true); DBA::update( @@ -664,11 +673,6 @@ class Worker self::$db_duration_write += (microtime(true) - $stamp); } else { // Kill long running processes - // Check if the priority is in a valid range - if (!in_array($entry['priority'], PRIORITIES)) { - Logger::warning('Invalid priority', ['entry' => $entry, 'callstack' => System::callstack(20)]); - $entry['priority'] = PRIORITY_MEDIUM; - } // Define the maximum durations $max_duration_defaults = [PRIORITY_CRITICAL => 720, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 720]; @@ -1393,14 +1397,11 @@ class Worker return false; } + $queue = self::checkPriority($queue); + $id = $queue['id']; $priority = $queue['priority']; - if (!in_array($priority, PRIORITIES)) { - Logger::warning('Invalid priority', ['queue' => $queue, 'callstack' => System::callstack(20)]); - $priority = PRIORITY_MEDIUM; - } - $max_level = DI::config()->get('system', 'worker_defer_limit'); $new_retrial = self::getNextRetrial($queue, $max_level); From 40aa67f8beca2735f2f1d70a503a1b58500b3e3f Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 2 May 2022 05:15:27 +0000 Subject: [PATCH 13/35] Bulk transmission for AP posts --- src/Model/Post/Delivery.php | 64 ++++++++++++++++++++++++++++ src/Util/HTTPSignature.php | 7 ++- src/Worker/APDelivery.php | 80 ++++++++++++++++++++++++++++++++--- src/Worker/ExpirePosts.php | 3 ++ src/Worker/Notifier.php | 6 ++- static/dbstructure.config.php | 19 ++++++++- 6 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 src/Model/Post/Delivery.php diff --git a/src/Model/Post/Delivery.php b/src/Model/Post/Delivery.php new file mode 100644 index 0000000000..8cd01c3a85 --- /dev/null +++ b/src/Model/Post/Delivery.php @@ -0,0 +1,64 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use Friendica\Database\DBA; +use BadMethodCallException; +use Friendica\Database\Database; +use Friendica\Model\ItemURI; + +class Delivery +{ + /** + * Add a post to an inbox + * + * @param integer $uri_id + * @param string $inbox + * @param string $created + */ + public static function add(int $uri_id, int $uid, string $inbox, string $created, string $command) + { + if (empty($uri_id)) { + throw new BadMethodCallException('Empty URI_id'); + } + + $fields = ['uri-id' => $uri_id, 'uid' => $uid, 'inbox-id' => ItemURI::getIdByURI($inbox), 'created' => $created, 'command' => $command]; + + DBA::insert('post-delivery', $fields, Database::INSERT_IGNORE); + } + + /** + * Remove post from an inbox after delivery + * + * @param integer $uri_id + * @param string $inbox + */ + public static function remove(int $uri_id, string $inbox) + { + DBA::delete('post-delivery', ['uri-id' => $uri_id, 'inbox-id' => ItemURI::getIdByURI($inbox)]); + } + + public static function selectForInbox(string $inbox) + { + return DBA::selectToArray('post-delivery', [], ['inbox-id' => ItemURI::getIdByURI($inbox)], ['order' => ['created']]); + } +} diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index cc25f14918..db1d7a2799 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -27,6 +27,7 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\APContact; use Friendica\Model\Contact; +use Friendica\Model\ItemURI; use Friendica\Model\User; use Friendica\Network\HTTPClient\Client\HttpClientAccept; use Friendica\Network\HTTPClient\Client\HttpClientOptions; @@ -329,7 +330,7 @@ class HTTPSignature $status = DBA::selectFirst('inbox-status', [], ['url' => $url]); if (!DBA::isResult($status)) { - DBA::insert('inbox-status', ['url' => $url, 'created' => $now, 'shared' => $shared], Database::INSERT_IGNORE); + DBA::insert('inbox-status', ['url' => $url, 'uri-id' => ItemURI::getIdByURI($url), 'created' => $now, 'shared' => $shared], Database::INSERT_IGNORE); $status = DBA::selectFirst('inbox-status', [], ['url' => $url]); } @@ -369,6 +370,10 @@ class HTTPSignature $fields['archive'] = false; } + if (empty($status['uri-id'])) { + $fields['uri-id'] = ItemURI::getIdByURI($url); + } + DBA::update('inbox-status', $fields, ['url' => $url]); } diff --git a/src/Worker/APDelivery.php b/src/Worker/APDelivery.php index dc826e3315..299eeba359 100644 --- a/src/Worker/APDelivery.php +++ b/src/Worker/APDelivery.php @@ -25,6 +25,7 @@ use Friendica\Core\Logger; use Friendica\Core\Worker; use Friendica\Model\Contact; use Friendica\Model\GServer; +use Friendica\Model\Item; use Friendica\Model\Post; use Friendica\Protocol\ActivityPub; use Friendica\Util\HTTPSignature; @@ -56,6 +57,67 @@ class APDelivery Logger::info('Invoked', ['cmd' => $cmd, 'inbox' => $inbox, 'id' => $item_id, 'uri-id' => $uri_id, 'uid' => $uid]); + if (empty($uri_id)) { + $result = self::deliver($inbox); + $success = $result['success']; + $uri_ids = $result['uri_ids']; + } + + if (empty($uri_ids)) { + $success = self::deliverToInbox($cmd, $item_id, $inbox, $uid, $receivers, $uri_id); + } + + if (!$success && !Worker::defer() && in_array($cmd, [Delivery::POST])) { + if ($uri_id) { + Post\Delivery::remove($uri_id, $inbox); + Post\DeliveryData::incrementQueueFailed($uri_id); + } elseif (!empty($uri_ids)) { + foreach ($uri_ids as $uri_id) { + Post\Delivery::remove($uri_id, $inbox); + Post\DeliveryData::incrementQueueFailed($uri_id); + } + } + } elseif ($success && in_array($cmd, [Delivery::POST])) { + if ($uri_id) { + Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB); + } elseif (!empty($uri_ids)) { + foreach ($uri_ids as $uri_id) { + Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB); + } + } + } + } + + private static function deliver(string $inbox) + { + $uri_ids = []; + $success = true; + + $posts = Post\Delivery::selectForInbox($inbox); + foreach ($posts as $post) { + $uri_ids[] = $post['uri-id']; + if ($success) { + $success = self::deliverToInbox($post['command'], 0, $inbox, $post['uid'], [], $post['uri-id']); + } + } + + return ['success' => $success, 'uri_ids' => $uri_ids]; + } + + private static function deliverToInbox(string $cmd, int $item_id, string $inbox, int $uid, array $receivers, int $uri_id) + { + if (empty($item_id) && !empty($uri_id) && !empty($uid)) { + $item = Post::selectFirst(['id', 'parent'], ['uri-id' => $uri_id, 'uid' => $uid]); + $item_id = $item['id'] ?? 0; + if (empty($receivers) && !empty($item)) { + $parent = Post::selectFirst(Item::DELIVER_FIELDLIST, ['id' => $item['parent']]); + + $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true, $item['id']); + + $receivers = $inboxes[$inbox] ?? []; + } + } + $success = true; if ($cmd == Delivery::MAIL) { @@ -77,9 +139,21 @@ class APDelivery $data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id); if (!empty($data)) { $success = HTTPSignature::transmit($data, $inbox, $uid); + if ($success && $uri_id) { + Post\Delivery::remove($uri_id, $inbox); + } } } + self::setSuccess($receivers, $success); + + Logger::info('Delivered', ['cmd' => $cmd, 'inbox' => $inbox, 'id' => $item_id, 'uri-id' => $uri_id, 'uid' => $uid, 'success' => $success]); + + return $success; + } + + private static function setSuccess(array $receivers, bool $success) + { $gsid = null; foreach ($receivers as $receiver) { @@ -100,11 +174,5 @@ class APDelivery if (!empty($gsid)) { GServer::setProtocol($gsid, Post\DeliveryData::ACTIVITYPUB); } - - if (!$success && !Worker::defer() && in_array($cmd, [Delivery::POST])) { - Post\DeliveryData::incrementQueueFailed($uri_id); - } elseif ($success && in_array($cmd, [Delivery::POST])) { - Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB); - } } } diff --git a/src/Worker/ExpirePosts.php b/src/Worker/ExpirePosts.php index b566976e9e..52d792ea62 100644 --- a/src/Worker/ExpirePosts.php +++ b/src/Worker/ExpirePosts.php @@ -189,6 +189,9 @@ class ExpirePosts AND NOT EXISTS(SELECT `uri-id` FROM `contact` WHERE `uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `uri-id` FROM `apcontact` WHERE `uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `uri-id` FROM `fcontact` WHERE `uri-id` = `item-uri`.`id`) + AND NOT EXISTS(SELECT `uri-id` FROM `inbox-status` WHERE `uri-id` = `item-uri`.`id`) + AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `uri-id` = `item-uri`.`id`) + AND NOT EXISTS(SELECT `uri-id` FROM `post-delivery` WHERE `inbox-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `parent-uri-id` FROM `mail` WHERE `parent-uri-id` = `item-uri`.`id`) AND NOT EXISTS(SELECT `thr-parent-id` FROM `mail` WHERE `thr-parent-id` = `item-uri`.`id`)", $item['uri-id']]); diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php index 80628d7dba..fdec949a71 100644 --- a/src/Worker/Notifier.php +++ b/src/Worker/Notifier.php @@ -789,8 +789,9 @@ class Notifier Logger::info('Delivery via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]); if (Worker::add(['priority' => $priority, 'created' => $created, 'dont_fork' => true], - 'APDelivery', $cmd, $target_item['id'], $inbox, $uid, $receivers, $target_item['uri-id'])) { + 'APDelivery', $cmd, 0, $inbox, $uid)) { $delivery_queue_count++; + Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); } } @@ -798,8 +799,9 @@ class Notifier foreach ($relay_inboxes as $inbox) { Logger::info('Delivery to relay servers via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]); - if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, $target_item['id'], $inbox, $uid, [], $target_item['uri-id'])) { + if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, 0, $inbox, $uid)) { $delivery_queue_count++; + Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); } } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 03748f36a5..e3435989cf 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -772,6 +772,7 @@ return [ "comment" => "Status of ActivityPub inboxes", "fields" => [ "url" => ["type" => "varbinary(255)", "not null" => "1", "primary" => "1", "comment" => "URL of the inbox"], + "uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of inbox url"], "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation date of this entry"], "success" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last successful delivery"], "failure" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last failed delivery"], @@ -780,7 +781,8 @@ return [ "shared" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Is it a shared inbox?"] ], "indexes" => [ - "PRIMARY" => ["url"] + "PRIMARY" => ["url"], + "uri-id" => ["uri-id"], ] ], "intro" => [ @@ -1155,6 +1157,21 @@ return [ "title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"], ] ], + "post-delivery" => [ + "comment" => "Status of ActivityPub inboxes", + "fields" => [ + "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "inbox-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of inbox url"], + "uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Delivering user"], + "created" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], + "command" => ["type" => "varbinary(32)", "comment" => ""], + ], + "indexes" => [ + "PRIMARY" => ["uri-id", "inbox-id"], + "inbox-id_created" => ["inbox-id", "created"], + "uid" => ["uid"], + ] + ], "post-delivery-data" => [ "comment" => "Delivery data for items", "fields" => [ From a9990db98ce7f84e48df6a4d78177275f8d973c5 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 2 May 2022 05:16:02 +0000 Subject: [PATCH 14/35] Clean the worker queue directly from cron --- src/Core/Worker.php | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 5bd281638f..a548bbc8c7 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -1149,6 +1149,32 @@ class Worker // Cleaning dead processes self::killStaleWorkers(); + + // Remove old entries from the workerqueue + self::cleanWorkerQueue(); + } + + /** + * Remove old entries from the workerqueue + * + * @return void + */ + private static function cleanWorkerQueue() + { + DBA::delete('workerqueue', ["`done` AND `executed` < ?", DateTimeFormat::utc('now - 1 hour')]); + + // Optimizing this table only last seconds + if (DI::config()->get('system', 'optimize_tables')) { + // We are acquiring the two locks from the worker to avoid locking problems + if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) { + if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) { + DBA::e("OPTIMIZE TABLE `workerqueue`"); + DBA::e("OPTIMIZE TABLE `process`"); + DI::lock()->release(Worker::LOCK_WORKER); + } + DI::lock()->release(Worker::LOCK_PROCESS); + } + } } /** From c608d857077cc3c830637dbcbd40a92f842f456d Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 2 May 2022 05:17:42 +0000 Subject: [PATCH 15/35] Updated database description --- database.sql | 22 +++++++++++++++++++- doc/database.md | 1 + doc/database/db_inbox-status.md | 8 ++++++++ doc/database/db_post-delivery.md | 35 ++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 doc/database/db_post-delivery.md diff --git a/database.sql b/database.sql index 7a8ee0adb7..58d79fb4bb 100644 --- a/database.sql +++ b/database.sql @@ -712,13 +712,16 @@ CREATE TABLE IF NOT EXISTS `hook` ( -- CREATE TABLE IF NOT EXISTS `inbox-status` ( `url` varbinary(255) NOT NULL COMMENT 'URL of the inbox', + `uri-id` int unsigned COMMENT 'Item-uri id of inbox url', `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation date of this entry', `success` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last successful delivery', `failure` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last failed delivery', `previous` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Previous delivery date', `archive` boolean NOT NULL DEFAULT '0' COMMENT 'Is the inbox archived?', `shared` boolean NOT NULL DEFAULT '0' COMMENT 'Is it a shared inbox?', - PRIMARY KEY(`url`) + PRIMARY KEY(`url`), + INDEX `uri-id` (`uri-id`), + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Status of ActivityPub inboxes'; -- @@ -1114,6 +1117,23 @@ CREATE TABLE IF NOT EXISTS `post-content` ( FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts'; +-- +-- TABLE post-delivery +-- +CREATE TABLE IF NOT EXISTS `post-delivery` ( + `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `inbox-id` int unsigned NOT NULL COMMENT 'Item-uri id of inbox url', + `uid` mediumint unsigned COMMENT 'Delivering user', + `created` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', + `command` varbinary(32) COMMENT '', + PRIMARY KEY(`uri-id`,`inbox-id`), + INDEX `inbox-id_created` (`inbox-id`,`created`), + INDEX `uid` (`uid`), + FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`inbox-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Status of ActivityPub inboxes'; + -- -- TABLE post-delivery-data -- diff --git a/doc/database.md b/doc/database.md index dff31657fd..961e8183ab 100644 --- a/doc/database.md +++ b/doc/database.md @@ -50,6 +50,7 @@ Database Tables | [post-category](help/database/db_post-category) | post relation to categories | | [post-collection](help/database/db_post-collection) | Collection of posts | | [post-content](help/database/db_post-content) | Content for all posts | +| [post-delivery](help/database/db_post-delivery) | Status of ActivityPub inboxes | | [post-delivery-data](help/database/db_post-delivery-data) | Delivery data for items | | [post-link](help/database/db_post-link) | Post related external links | | [post-media](help/database/db_post-media) | Attached media | diff --git a/doc/database/db_inbox-status.md b/doc/database/db_inbox-status.md index 3b82cf46a0..79df149a59 100644 --- a/doc/database/db_inbox-status.md +++ b/doc/database/db_inbox-status.md @@ -9,6 +9,7 @@ Fields | Field | Description | Type | Null | Key | Default | Extra | | -------- | ------------------------------------ | -------------- | ---- | --- | ------------------- | ----- | | url | URL of the inbox | varbinary(255) | NO | PRI | NULL | | +| uri-id | Item-uri id of inbox url | int unsigned | YES | | NULL | | | created | Creation date of this entry | datetime | NO | | 0001-01-01 00:00:00 | | | success | Date of the last successful delivery | datetime | NO | | 0001-01-01 00:00:00 | | | failure | Date of the last failed delivery | datetime | NO | | 0001-01-01 00:00:00 | | @@ -22,6 +23,13 @@ Indexes | Name | Fields | | ------- | ------ | | PRIMARY | url | +| uri-id | uri-id | +Foreign Keys +------------ + +| Field | Target Table | Target Field | +|-------|--------------|--------------| +| uri-id | [item-uri](help/database/db_item-uri) | id | Return to [database documentation](help/database) diff --git a/doc/database/db_post-delivery.md b/doc/database/db_post-delivery.md new file mode 100644 index 0000000000..1937e059a5 --- /dev/null +++ b/doc/database/db_post-delivery.md @@ -0,0 +1,35 @@ +Table post-delivery +=========== + +Status of ActivityPub inboxes + +Fields +------ + +| Field | Description | Type | Null | Key | Default | Extra | +| -------- | --------------------------------------------------------- | ------------------ | ---- | --- | ------------------- | ----- | +| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | | +| inbox-id | Item-uri id of inbox url | int unsigned | NO | PRI | NULL | | +| uid | Delivering user | mediumint unsigned | YES | | NULL | | +| created | | datetime | YES | | 0001-01-01 00:00:00 | | +| command | | varbinary(32) | YES | | NULL | | + +Indexes +------------ + +| Name | Fields | +| ---------------- | ----------------- | +| PRIMARY | uri-id, inbox-id | +| inbox-id_created | inbox-id, created | +| uid | uid | + +Foreign Keys +------------ + +| Field | Target Table | Target Field | +|-------|--------------|--------------| +| uri-id | [item-uri](help/database/db_item-uri) | id | +| inbox-id | [item-uri](help/database/db_item-uri) | id | +| uid | [user](help/database/db_user) | uid | + +Return to [database documentation](help/database) From 9c2fe81ac6ddd24de1768a56dca7529d8d0d8286 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 2 May 2022 05:53:11 +0000 Subject: [PATCH 16/35] Transmit via the sharedInbox --- src/Worker/APDelivery.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Worker/APDelivery.php b/src/Worker/APDelivery.php index 299eeba359..6a1241293e 100644 --- a/src/Worker/APDelivery.php +++ b/src/Worker/APDelivery.php @@ -107,12 +107,18 @@ class APDelivery private static function deliverToInbox(string $cmd, int $item_id, string $inbox, int $uid, array $receivers, int $uri_id) { if (empty($item_id) && !empty($uri_id) && !empty($uid)) { - $item = Post::selectFirst(['id', 'parent'], ['uri-id' => $uri_id, 'uid' => $uid]); + $item = Post::selectFirst(['id', 'parent', 'origin'], ['uri-id' => $uri_id, 'uid' => $uid]); $item_id = $item['id'] ?? 0; if (empty($receivers) && !empty($item)) { $parent = Post::selectFirst(Item::DELIVER_FIELDLIST, ['id' => $item['parent']]); - $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true, $item['id']); + if ($item['origin']) { + $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid); + } else { + // Remote items are transmitted via the personal inboxes. + // Doing so ensures that the dedicated receiver will get the message. + $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true, $item_id); + } $receivers = $inboxes[$inbox] ?? []; } From fcb245947e68adb3a035de0419b8ecee805c78cc Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 2 May 2022 06:10:36 +0000 Subject: [PATCH 17/35] Fetch the receiver list from sharedinbox or personal inbox --- src/Worker/APDelivery.php | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Worker/APDelivery.php b/src/Worker/APDelivery.php index 6a1241293e..370e3713ca 100644 --- a/src/Worker/APDelivery.php +++ b/src/Worker/APDelivery.php @@ -112,15 +112,14 @@ class APDelivery if (empty($receivers) && !empty($item)) { $parent = Post::selectFirst(Item::DELIVER_FIELDLIST, ['id' => $item['parent']]); - if ($item['origin']) { - $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid); - } else { - // Remote items are transmitted via the personal inboxes. - // Doing so ensures that the dedicated receiver will get the message. - $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true, $item_id); - } - + $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid); $receivers = $inboxes[$inbox] ?? []; + + // When we haven't fetched the receiver list, it can be a personal inbox + if (empty($receivers)) { + $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true); + $receivers = $inboxes[$inbox] ?? []; + } } } From 76789acacec5f3c6d75206d5ef437c9a6e61706e Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 2 May 2022 14:35:57 +0000 Subject: [PATCH 18/35] Reduce network request by transmitting to shared inboxes --- src/Worker/Notifier.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php index fdec949a71..5ec35a656b 100644 --- a/src/Worker/Notifier.php +++ b/src/Worker/Notifier.php @@ -677,7 +677,7 @@ class Notifier } DBA::close($contacts_stmt); - $inboxes = ActivityPub\Transmitter::fetchTargetInboxesforUser(0); + $inboxes = ActivityPub\Transmitter::fetchTargetInboxesforUser($self_user_id); foreach ($inboxes as $inbox => $receivers) { Logger::info('Account removal via ActivityPub', ['uid' => $self_user_id, 'inbox' => $inbox]); Worker::add(['priority' => PRIORITY_NEGLIGIBLE, 'created' => $created, 'dont_fork' => true], @@ -750,9 +750,7 @@ class Notifier Logger::info('Remote item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' is no AP post. It will not be distributed.'); return ['count' => 0, 'contacts' => []]; } elseif ($parent['origin']) { - // Remote items are transmitted via the personal inboxes. - // Doing so ensures that the dedicated receiver will get the message. - $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true, $target_item['id']); + $inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, false, $target_item['id']); if (in_array($target_item['private'], [Item::PUBLIC])) { $inboxes = ActivityPub\Transmitter::addRelayServerInboxesForItem($parent['id'], $inboxes); From 6e394ac6ff127efab5015a7c53f72b07737ab512 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 2 May 2022 14:36:21 +0000 Subject: [PATCH 19/35] Whitespaces removed --- src/Core/Worker.php | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index a548bbc8c7..45811d40a7 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -168,8 +168,8 @@ class Worker /** * Check and fix the priority of a worker task - * @param array $entry - * @return array + * @param array $entry + * @return array */ private static function checkPriority(array $entry) { @@ -177,7 +177,7 @@ class Worker if (!in_array($entry['priority'], PRIORITIES)) { Logger::warning('Invalid priority', ['entry' => $entry, 'callstack' => System::callstack(20)]); - DBA::update('workerqueue', ['priority' => PRIORITY_MEDIUM], ['id' => $entry['id']]); + DBA::update('workerqueue', ['priority' => PRIORITY_MEDIUM], ['id' => $entry['id']]); $entry['priority'] = PRIORITY_MEDIUM; } @@ -300,41 +300,41 @@ class Worker /** * Checks if the given file is valid to be included * - * @param mixed $file - * @return bool + * @param mixed $file + * @return bool */ private static function validateInclude(&$file) { $orig_file = $file; - + $file = realpath($file); - + if (strpos($file, getcwd()) !== 0) { return false; } - + $file = str_replace(getcwd() . "/", "", $file, $count); if ($count != 1) { return false; } - + if ($orig_file !== $file) { return false; } - + $valid = false; if (strpos($file, "include/") === 0) { $valid = true; } - + if (strpos($file, "addon/") === 0) { $valid = true; } - + // Simply return flag return $valid; } - + /** * Execute a worker entry * @@ -1169,7 +1169,7 @@ class Worker if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) { if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) { DBA::e("OPTIMIZE TABLE `workerqueue`"); - DBA::e("OPTIMIZE TABLE `process`"); + DBA::e("OPTIMIZE TABLE `process`"); DI::lock()->release(Worker::LOCK_WORKER); } DI::lock()->release(Worker::LOCK_PROCESS); From 31637603435e3eb299a830c041fb4bfcd67f07b1 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 2 May 2022 17:34:40 +0000 Subject: [PATCH 20/35] Added configuration for the bulk delivery --- src/Worker/Notifier.php | 27 ++++++++++++++++++++------- static/defaults.config.php | 4 ++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php index 5ec35a656b..0ce5ed92a7 100644 --- a/src/Worker/Notifier.php +++ b/src/Worker/Notifier.php @@ -786,10 +786,17 @@ class Notifier Logger::info('Delivery via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]); - if (Worker::add(['priority' => $priority, 'created' => $created, 'dont_fork' => true], - 'APDelivery', $cmd, 0, $inbox, $uid)) { - $delivery_queue_count++; - Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); + if (DI::config()->get('system', 'bulk_delivery')) { + if (Worker::add(['priority' => $priority, 'created' => $created, 'dont_fork' => true], + 'APDelivery', $cmd, 0, $inbox, $uid)) { + $delivery_queue_count++; + Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); + } + } else { + if (Worker::add(['priority' => $priority, 'created' => $created, 'dont_fork' => true], + 'APDelivery', $cmd, $target_item['id'], $inbox, $uid, $receivers, $target_item['uri-id'])) { + $delivery_queue_count++; + } } } @@ -797,9 +804,15 @@ class Notifier foreach ($relay_inboxes as $inbox) { Logger::info('Delivery to relay servers via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]); - if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, 0, $inbox, $uid)) { - $delivery_queue_count++; - Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); + if (DI::config()->get('system', 'bulk_delivery')) { + if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, 0, $inbox, $uid)) { + $delivery_queue_count++; + Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); + } + } else { + if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, $target_item['id'], $inbox, $uid, [], $target_item['uri-id'])) { + $delivery_queue_count++; + } } } diff --git a/static/defaults.config.php b/static/defaults.config.php index 006c3d4f57..336916ceff 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -122,6 +122,10 @@ return [ // Display "Emoji Only" posts in big. 'big_emojis' => false, + // bulk_delivery (Boolean) + // Delivers AP messages in a bulk (experimental) + 'bulk_delivery' => false, + // block_local_dir (Boolean) // Deny public access to the local user directory. 'block_local_dir' => false, From 632a98965c4671389622fbca369553c97875d012 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 3 May 2022 08:19:35 +0000 Subject: [PATCH 21/35] Fetch contact via uri-id to improve performance --- src/Protocol/ActivityPub/Receiver.php | 45 +++++++++++++++++---------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index c8d131b725..4a13872983 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -938,7 +938,7 @@ class Receiver // Fetch the receivers for the public and the followers collection if ((($receiver == $followers) || (($receiver == self::PUBLIC_COLLECTION) && !$is_forum)) && !empty($actor)) { - $receivers = self::getReceiverForActor($actor, $tags, $receivers, $follower_target); + $receivers = self::getReceiverForActor($actor, $tags, $receivers, $follower_target, $profile); continue; } @@ -1000,33 +1000,46 @@ class Receiver * @param array $tags * @param array $receivers * @param integer $target_type + * @param array $profile * * @return array with receivers (user id) * @throws \Exception */ - private static function getReceiverForActor($actor, $tags, $receivers, $target_type) + private static function getReceiverForActor($actor, $tags, $receivers, $target_type, $profile) { $basecondition = ['rel' => [Contact::SHARING, Contact::FRIEND, Contact::FOLLOWER], 'network' => Protocol::FEDERATED, 'archive' => false, 'pending' => false]; - $condition = DBA::mergeConditions($basecondition, ["`nurl` = ? AND `uid` != ?", Strings::normaliseLink($actor), 0]); - $contacts = DBA::select('contact', ['uid', 'rel'], $condition); - while ($contact = DBA::fetch($contacts)) { - if (empty($receivers[$contact['uid']]) && self::isValidReceiverForActor($contact, $tags)) { - $receivers[$contact['uid']] = ['uid' => $contact['uid'], 'type' => $target_type]; + if (!empty($profile['uri-id'])) { + $condition = DBA::mergeConditions($basecondition, ["`uri-id` = ? AND `uid` != ?", $profile['uri-id'], 0]); + $contacts = DBA::select('contact', ['uid', 'rel'], $condition); + while ($contact = DBA::fetch($contacts)) { + if (empty($receivers[$contact['uid']]) && self::isValidReceiverForActor($contact, $tags)) { + $receivers[$contact['uid']] = ['uid' => $contact['uid'], 'type' => $target_type]; + } } - } - DBA::close($contacts); + DBA::close($contacts); + } else { + // This part will only be called while post update 1426 wasn't finished + $condition = DBA::mergeConditions($basecondition, ["`nurl` = ? AND `uid` != ?", Strings::normaliseLink($actor), 0]); + $contacts = DBA::select('contact', ['uid', 'rel'], $condition); + while ($contact = DBA::fetch($contacts)) { + if (empty($receivers[$contact['uid']]) && self::isValidReceiverForActor($contact, $tags)) { + $receivers[$contact['uid']] = ['uid' => $contact['uid'], 'type' => $target_type]; + } + } + DBA::close($contacts); - // The queries are split because of performance issues - $condition = DBA::mergeConditions($basecondition, ["`alias` IN (?, ?) AND `uid` != ?", Strings::normaliseLink($actor), $actor, 0]); - $contacts = DBA::select('contact', ['uid', 'rel'], $condition); - while ($contact = DBA::fetch($contacts)) { - if (empty($receivers[$contact['uid']]) && self::isValidReceiverForActor($contact, $tags)) { - $receivers[$contact['uid']] = ['uid' => $contact['uid'], 'type' => $target_type]; + // The queries are split because of performance issues + $condition = DBA::mergeConditions($basecondition, ["`alias` IN (?, ?) AND `uid` != ?", Strings::normaliseLink($actor), $actor, 0]); + $contacts = DBA::select('contact', ['uid', 'rel'], $condition); + while ($contact = DBA::fetch($contacts)) { + if (empty($receivers[$contact['uid']]) && self::isValidReceiverForActor($contact, $tags)) { + $receivers[$contact['uid']] = ['uid' => $contact['uid'], 'type' => $target_type]; + } } + DBA::close($contacts); } - DBA::close($contacts); return $receivers; } From 690682a37e25e7fa4dcc89f17b2348d8cfdc6c4b Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 3 May 2022 08:20:05 +0000 Subject: [PATCH 22/35] Only send "accept" headers on some HTTP methods --- src/Network/HTTPClient/Client/HttpClient.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Network/HTTPClient/Client/HttpClient.php b/src/Network/HTTPClient/Client/HttpClient.php index 28b0f0369e..98d58e03c1 100644 --- a/src/Network/HTTPClient/Client/HttpClient.php +++ b/src/Network/HTTPClient/Client/HttpClient.php @@ -140,7 +140,7 @@ class HttpClient implements ICanSendHttpRequests } }; - if (empty($conf[HttpClientOptions::HEADERS]['Accept'])) { + if (empty($conf[HttpClientOptions::HEADERS]['Accept']) && in_array($method, ['get', 'head'])) { $this->logger->info('Accept header was missing, using default.', ['url' => $url, 'callstack' => System::callstack()]); $conf[HttpClientOptions::HEADERS]['Accept'] = HttpClientAccept::DEFAULT; } From e6440471ae1a3bfce1b284daeea26ece9fb0af80 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 3 May 2022 08:20:26 +0000 Subject: [PATCH 23/35] Throw an error when the feed is invalid --- src/Module/Feed.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Module/Feed.php b/src/Module/Feed.php index 8db92c275a..9acde7bb30 100644 --- a/src/Module/Feed.php +++ b/src/Module/Feed.php @@ -25,6 +25,7 @@ use Friendica\BaseModule; use Friendica\Core\System; use Friendica\DI; use Friendica\Protocol\Feed as ProtocolFeed; +use Friendica\Network\HTTPException; /** * Provides public Atom feeds @@ -42,7 +43,7 @@ use Friendica\Protocol\Feed as ProtocolFeed; */ class Feed extends BaseModule { - protected function content(array $request = []): string + protected function rawContent(array $request = []) { $last_update = $request['last_update'] ?? ''; $nocache = !empty($request['nocache']) && local_user(); @@ -66,6 +67,11 @@ class Feed extends BaseModule $type = 'posts'; } - System::httpExit(ProtocolFeed::atom($this->parameters['nickname'], $last_update, 10, $type, $nocache, true), Response::TYPE_ATOM); + $feed = ProtocolFeed::atom($this->parameters['nickname'], $last_update, 10, $type, $nocache, true); + if (empty($feed)) { + throw new HTTPException\NotFoundException(DI::l10n()->t('User not found.')); + } + + System::httpExit($feed, Response::TYPE_ATOM); } } From 49ec5e5e6e94a9bf7569ea76887ed79cd186b21c Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 3 May 2022 08:20:48 +0000 Subject: [PATCH 24/35] Check for empty value --- src/Worker/APDelivery.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Worker/APDelivery.php b/src/Worker/APDelivery.php index 370e3713ca..37b47dd46d 100644 --- a/src/Worker/APDelivery.php +++ b/src/Worker/APDelivery.php @@ -68,7 +68,7 @@ class APDelivery } if (!$success && !Worker::defer() && in_array($cmd, [Delivery::POST])) { - if ($uri_id) { + if (!empty($uri_id)) { Post\Delivery::remove($uri_id, $inbox); Post\DeliveryData::incrementQueueFailed($uri_id); } elseif (!empty($uri_ids)) { @@ -78,7 +78,7 @@ class APDelivery } } } elseif ($success && in_array($cmd, [Delivery::POST])) { - if ($uri_id) { + if (!empty($uri_id)) { Post\DeliveryData::incrementQueueDone($uri_id, Post\DeliveryData::ACTIVITYPUB); } elseif (!empty($uri_ids)) { foreach ($uri_ids as $uri_id) { From fdf8002df1cd0235700ef4d80938b821665ee145 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 3 May 2022 08:26:38 +0000 Subject: [PATCH 25/35] Update database.sql --- database.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.sql b/database.sql index 58d79fb4bb..bb53181e88 100644 --- a/database.sql +++ b/database.sql @@ -1,5 +1,5 @@ -- ------------------------------------------ --- Friendica 2022.05-dev (Siberian Iris) +-- Friendica 2022.05-rc (Siberian Iris) -- DB_UPDATE_VERSION 1461 -- ------------------------------------------ From f6218427a4a22088991c0975f9b3f8a7ad65c551 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 3 May 2022 21:51:56 +0000 Subject: [PATCH 26/35] Use the uri-id to fetch the contact --- database.sql | 4 ++++ src/Content/Item.php | 3 +-- src/Model/Item.php | 2 +- static/dbview.config.php | 4 ++++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/database.sql b/database.sql index bb53181e88..5aa9a5b706 100644 --- a/database.sql +++ b/database.sql @@ -1741,6 +1741,7 @@ CREATE VIEW `post-user-view` AS SELECT `author`.`hidden` AS `author-hidden`, `author`.`updated` AS `author-updated`, `author`.`gsid` AS `author-gsid`, + `author`.`uri-id` AS `author-uri-id`, `post-user`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -1909,6 +1910,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT `author`.`hidden` AS `author-hidden`, `author`.`updated` AS `author-updated`, `author`.`gsid` AS `author-gsid`, + `author`.`uri-id` AS `author-uri-id`, `post-thread-user`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -2063,6 +2065,7 @@ CREATE VIEW `post-view` AS SELECT `author`.`hidden` AS `author-hidden`, `author`.`updated` AS `author-updated`, `author`.`gsid` AS `author-gsid`, + `author`.`uri-id` AS `author-uri-id`, `post`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, @@ -2193,6 +2196,7 @@ CREATE VIEW `post-thread-view` AS SELECT `author`.`hidden` AS `author-hidden`, `author`.`updated` AS `author-updated`, `author`.`gsid` AS `author-gsid`, + `author`.`uri-id` AS `author-uri-id`, `post-thread`.`owner-id` AS `owner-id`, `owner`.`url` AS `owner-link`, `owner`.`addr` AS `owner-addr`, diff --git a/src/Content/Item.php b/src/Content/Item.php index 0c60c4e26f..9136396ed1 100644 --- a/src/Content/Item.php +++ b/src/Content/Item.php @@ -35,7 +35,6 @@ use Friendica\Model\Tag; use Friendica\Model\Post; use Friendica\Protocol\Activity; use Friendica\Util\Profiler; -use Friendica\Util\Strings; use Friendica\Util\XML; /** @@ -402,7 +401,7 @@ class Item $pcid = $item['author-id']; $network = ''; $rel = 0; - $condition = ['uid' => local_user(), 'nurl' => Strings::normaliseLink($item['author-link'])]; + $condition = ['uid' => local_user(), 'uri-id' => $item['author-uri-id']]; $contact = DBA::selectFirst('contact', ['id', 'network', 'rel'], $condition); if (DBA::isResult($contact)) { $cid = $contact['id']; diff --git a/src/Model/Item.php b/src/Model/Item.php index b63b98d773..96a12a1fd3 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -87,7 +87,7 @@ class Item 'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'mention', 'global', - 'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network', 'author-updated', 'author-gsid', 'author-addr', + 'author-id', 'author-link', 'author-name', 'author-avatar', 'author-network', 'author-updated', 'author-gsid', 'author-addr', 'author-uri-id', 'owner-id', 'owner-link', 'owner-name', 'owner-avatar', 'owner-network', 'owner-contact-type', 'owner-updated', 'causer-id', 'causer-link', 'causer-name', 'causer-avatar', 'causer-contact-type', 'causer-network', 'contact-id', 'contact-uid', 'contact-link', 'contact-name', 'contact-avatar', diff --git a/static/dbview.config.php b/static/dbview.config.php index ea991489d6..e2e29d8f4f 100644 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -148,6 +148,7 @@ "author-hidden" => ["author", "hidden"], "author-updated" => ["author", "updated"], "author-gsid" => ["author", "gsid"], + "author-uri-id" => ["author", "uri-id"], "owner-id" => ["post-user", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -314,6 +315,7 @@ "author-hidden" => ["author", "hidden"], "author-updated" => ["author", "updated"], "author-gsid" => ["author", "gsid"], + "author-uri-id" => ["author", "uri-id"], "owner-id" => ["post-thread-user", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -466,6 +468,7 @@ "author-hidden" => ["author", "hidden"], "author-updated" => ["author", "updated"], "author-gsid" => ["author", "gsid"], + "author-uri-id" => ["author", "uri-id"], "owner-id" => ["post", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], @@ -594,6 +597,7 @@ "author-hidden" => ["author", "hidden"], "author-updated" => ["author", "updated"], "author-gsid" => ["author", "gsid"], + "author-uri-id" => ["author", "uri-id"], "owner-id" => ["post-thread", "owner-id"], "owner-link" => ["owner", "url"], "owner-addr" => ["owner", "addr"], From 60532ee7e4152af9997e0d7b69c7b35262d5528d Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 4 May 2022 06:17:34 +0000 Subject: [PATCH 27/35] Fill the item cache when storing item --- src/Model/Item.php | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index 96a12a1fd3..3ff7909aa1 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1215,9 +1215,26 @@ class Item Worker::add(['priority' => $priority, 'dont_fork' => true], 'Notifier', $notify_type, (int)$posted_item['uri-id'], (int)$posted_item['uid']); } + // Fill the cache with the rendered content. + if (in_array($posted_item['gravity'], [GRAVITY_PARENT, GRAVITY_COMMENT]) && ($posted_item['uid'] == 0)) { + self::updateDisplayCache($posted_item['uri-id']); + } + return $post_user_id; } + /** + * Update the display cache + * + * @param integer $uri_id + * @return void + */ + public static function updateDisplayCache(int $uri_id) + { + $item = Post::selectFirst(self::DISPLAY_FIELDLIST, ['uri-id' => $uri_id]); + self::prepareBody($item, false, false, true); + } + /** * Change the owner of a parent item if it had been shared by a forum * @@ -2763,6 +2780,7 @@ class Item * @param array $item * @param boolean $attach * @param boolean $is_preview + * @param boolean $only_cache * @return string item body html * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException @@ -2771,7 +2789,7 @@ class Item * @hook prepare_body ('item'=>item array, 'html'=>body string, 'is_preview'=>boolean, 'filter_reasons'=>string array) after first bbcode to html * @hook prepare_body_final ('item'=>item array, 'html'=>body string) after attach icons and blockquote special case handling (spoiler, author) */ - public static function prepareBody(array &$item, $attach = false, $is_preview = false) + public static function prepareBody(array &$item, $attach = false, $is_preview = false, $only_cache = false) { $a = DI::app(); Hook::callAll('prepare_body_init', $item); @@ -2812,6 +2830,10 @@ class Item $item['body'] = $body; $s = $item["rendered-html"]; + if ($only_cache) { + return; + } + // Compile eventual content filter reasons $filter_reasons = []; if (!$is_preview && public_contact() != $item['author-id']) { From 10b453bfc28b8fec5f411072fdfe524aadf7fc29 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 4 May 2022 07:49:37 +0000 Subject: [PATCH 28/35] Deprecated function removed --- src/Model/Item.php | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index 3ff7909aa1..d57700c33d 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -2709,7 +2709,7 @@ class Item * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @todo Remove reference, simply return "rendered-html" and "rendered-hash" */ - public static function putInCache(&$item) + private static function putInCache(&$item) { // Save original body to prevent addons to modify it $body = $item['body']; @@ -2722,8 +2722,6 @@ class Item || $rendered_hash != hash('md5', BBCode::VERSION . '::' . $body) || DI::config()->get('system', 'ignore_cache') ) { - self::addRedirToImageTags($item); - $item['rendered-html'] = BBCode::convertForUriId($item['uri-id'], $item['body']); $item['rendered-hash'] = hash('md5', BBCode::VERSION . '::' . $body); @@ -2748,31 +2746,6 @@ class Item $item['body'] = $body; } - /** - * Find any non-embedded images in private items and add redir links to them - * - * @param array &$item The field array of an item row - */ - private static function addRedirToImageTags(array &$item) - { - $app = DI::app(); - - $matches = []; - $cnt = preg_match_all('|\[img\](http[^\[]*?/photo/[a-fA-F0-9]+?(-[0-9]\.[\w]+?)?)\[\/img\]|', $item['body'], $matches, PREG_SET_ORDER); - if ($cnt) { - foreach ($matches as $mtch) { - if (strpos($mtch[1], '/redir') !== false) { - continue; - } - - if ((local_user() == $item['uid']) && ($item['private'] == self::PRIVATE) && ($item['contact-id'] != $app->getContactId()) && ($item['network'] == Protocol::DFRN)) { - $img_url = 'redir/' . $item['contact-id'] . '?url=' . urlencode($mtch[1]); - $item['body'] = str_replace($mtch[0], '[img]' . $img_url . '[/img]', $item['body']); - } - } - } - } - /** * Given an item array, convert the body element from bbcode to html and add smilie icons. * If attach is true, also add icons for item attachments. From 2f74a7bca447211f2eb51ca1af447829f576b382 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 4 May 2022 07:50:14 +0000 Subject: [PATCH 29/35] Update display cache when the post had been edited --- mod/item.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mod/item.php b/mod/item.php index 498cf27132..34475188ea 100644 --- a/mod/item.php +++ b/mod/item.php @@ -666,21 +666,16 @@ function item_post(App $a) { $datarray['uri-id'] = ItemURI::getIdByURI($datarray['uri']); if ($orig_post) { - // Fill the cache field - // This could be done in Item::update as well - but we have to check for the existance of some fields. - Item::putInCache($datarray); - $fields = [ 'title' => $datarray['title'], 'body' => $datarray['body'], 'attach' => $datarray['attach'], 'file' => $datarray['file'], - 'rendered-html' => $datarray['rendered-html'], - 'rendered-hash' => $datarray['rendered-hash'], 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()]; Item::update($fields, ['id' => $post_id]); + Item::updateDisplayCache($datarray['uri-id']); if ($return_path) { DI::baseUrl()->redirect($return_path); From d34432c517a97a5d1b61fd90e117c1a9cb3e172c Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 May 2022 06:12:16 +0000 Subject: [PATCH 30/35] Ignore if the worker was added --- src/Worker/Notifier.php | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php index 0ce5ed92a7..48b81b9a99 100644 --- a/src/Worker/Notifier.php +++ b/src/Worker/Notifier.php @@ -787,11 +787,9 @@ class Notifier Logger::info('Delivery via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]); if (DI::config()->get('system', 'bulk_delivery')) { - if (Worker::add(['priority' => $priority, 'created' => $created, 'dont_fork' => true], - 'APDelivery', $cmd, 0, $inbox, $uid)) { - $delivery_queue_count++; - Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); - } + $delivery_queue_count++; + Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); + Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, 0, $inbox, $uid); } else { if (Worker::add(['priority' => $priority, 'created' => $created, 'dont_fork' => true], 'APDelivery', $cmd, $target_item['id'], $inbox, $uid, $receivers, $target_item['uri-id'])) { @@ -805,10 +803,9 @@ class Notifier Logger::info('Delivery to relay servers via ActivityPub', ['cmd' => $cmd, 'id' => $target_item['id'], 'inbox' => $inbox]); if (DI::config()->get('system', 'bulk_delivery')) { - if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, 0, $inbox, $uid)) { - $delivery_queue_count++; - Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); - } + $delivery_queue_count++; + Post\Delivery::add($target_item['uri-id'], $uid, $inbox, $target_item['created'], $cmd); + Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, 0, $inbox, $uid); } else { if (Worker::add(['priority' => $priority, 'dont_fork' => true], 'APDelivery', $cmd, $target_item['id'], $inbox, $uid, [], $target_item['uri-id'])) { $delivery_queue_count++; From 4d359b7de1f5ee3522ed0e5cff037b88f0ed6a4f Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 May 2022 07:57:52 +0000 Subject: [PATCH 31/35] Move the plink functionality to the right function --- src/Content/Item.php | 18 +----------------- src/Model/Item.php | 6 ++++++ 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/src/Content/Item.php b/src/Content/Item.php index 9136396ed1..d361128651 100644 --- a/src/Content/Item.php +++ b/src/Content/Item.php @@ -355,23 +355,7 @@ class Item $item['body'] = $this->l10n->t('%1$s tagged %2$s\'s %3$s with %4$s', $author, $objauthor, $plink, $tag); } } -/* - $matches = null; - if (preg_match_all('/@\[url=(.*?)\]/is', $item['body'], $matches, PREG_SET_ORDER)) { - foreach ($matches as $mtch) { - if (!strpos($mtch[1], 'zrl=')) { - $item['body'] = str_replace($mtch[0], '@[url=' . Contact::magicLink($mtch[1]) . ']', $item['body']); - } - } - } -*/ - // add sparkle links to appropriate permalinks - // Only create a redirection to a magic link when logged in - if (!empty($item['plink']) && Session::isAuthenticated() && $item['private'] == ModelItem::PRIVATE) { - $author = ['uid' => 0, 'id' => $item['author-id'], - 'network' => $item['author-network'], 'url' => $item['author-link']]; - $item['plink'] = Contact::magicLinkByContact($author, $item['plink']); - } + $this->profiler->stopRecording(); } diff --git a/src/Model/Item.php b/src/Model/Item.php index d57700c33d..496757910a 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -3200,6 +3200,12 @@ class Item 'orig_title' => DI::l10n()->t('View on separate page'), ]; + if (!empty($plink) && ($item['private'] == self::PRIVATE)) { + $author = ['uid' => 0, 'id' => $item['author-id'], + 'network' => $item['author-network'], 'url' => $item['author-link']]; + $plink = Contact::magicLinkByContact($author, $plink); + } + if (!empty($plink)) { $ret['href'] = DI::baseUrl()->remove($plink); $ret['title'] = DI::l10n()->t('Link to source'); From 2a4b5b4cb46b6e08047210f60aa1bacd6446d2e8 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 May 2022 08:40:50 +0000 Subject: [PATCH 32/35] New index --- database.sql | 1 + doc/database/db_post-media.md | 1 + static/dbstructure.config.php | 1 + 3 files changed, 3 insertions(+) diff --git a/database.sql b/database.sql index 5aa9a5b706..93f9bbef7e 100644 --- a/database.sql +++ b/database.sql @@ -1191,6 +1191,7 @@ CREATE TABLE IF NOT EXISTS `post-media` ( `publisher-image` varbinary(255) COMMENT 'Image of the publisher of the media', PRIMARY KEY(`id`), UNIQUE INDEX `uri-id-url` (`uri-id`,`url`), + INDEX `uri-id-id` (`uri-id`,`id`), FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Attached media'; diff --git a/doc/database/db_post-media.md b/doc/database/db_post-media.md index e17a04e5fb..5c2a1bfd97 100644 --- a/doc/database/db_post-media.md +++ b/doc/database/db_post-media.md @@ -35,6 +35,7 @@ Indexes | ---------- | ------------------- | | PRIMARY | id | | uri-id-url | UNIQUE, uri-id, url | +| uri-id-id | uri-id, id | Foreign Keys ------------ diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index e3435989cf..6d28b29141 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -1230,6 +1230,7 @@ return [ "indexes" => [ "PRIMARY" => ["id"], "uri-id-url" => ["UNIQUE", "uri-id", "url"], + "uri-id-id" => ["uri-id", "id"], ] ], "post-question" => [ From ab42fd88a4c05f6e76bc836c42b751d9c6d22dbd Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 May 2022 08:45:32 +0000 Subject: [PATCH 33/35] Changed index for the categories --- database.sql | 4 ++-- doc/database/db_post-category.md | 10 +++++----- static/dbstructure.config.php | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/database.sql b/database.sql index 93f9bbef7e..f0d5854623 100644 --- a/database.sql +++ b/database.sql @@ -1071,8 +1071,8 @@ CREATE TABLE IF NOT EXISTS `post-category` ( `type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '', `tid` int unsigned NOT NULL DEFAULT 0 COMMENT '', PRIMARY KEY(`uri-id`,`uid`,`type`,`tid`), - INDEX `uri-id` (`tid`), - INDEX `uid` (`uid`), + INDEX `tid` (`tid`), + INDEX `uid_uri-id` (`uid`,`uri-id`), FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT diff --git a/doc/database/db_post-category.md b/doc/database/db_post-category.md index b825a8f4b1..f28e306f17 100644 --- a/doc/database/db_post-category.md +++ b/doc/database/db_post-category.md @@ -16,11 +16,11 @@ Fields Indexes ------------ -| Name | Fields | -| ------- | ---------------------- | -| PRIMARY | uri-id, uid, type, tid | -| uri-id | tid | -| uid | uid | +| Name | Fields | +| ---------- | ---------------------- | +| PRIMARY | uri-id, uid, type, tid | +| tid | tid | +| uid_uri-id | uid, uri-id | Foreign Keys ------------ diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 6d28b29141..df0d469fce 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -1114,8 +1114,8 @@ return [ ], "indexes" => [ "PRIMARY" => ["uri-id", "uid", "type", "tid"], - "uri-id" => ["tid"], - "uid" => ["uid"], + "tid" => ["tid"], + "uid_uri-id" => ["uid", "uri-id"], ] ], "post-collection" => [ From 2433fee461a8c3e284681f402442388350a3734d Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 May 2022 12:23:44 +0000 Subject: [PATCH 34/35] Changes after code review --- database.sql | 2 +- doc/database.md | 2 +- doc/database/db_post-delivery.md | 2 +- mod/item.php | 3 +- src/Worker/CleanWorkerQueue.php | 51 -------------------------------- src/Worker/Cron.php | 3 -- static/dbstructure.config.php | 2 +- 7 files changed, 6 insertions(+), 59 deletions(-) delete mode 100644 src/Worker/CleanWorkerQueue.php diff --git a/database.sql b/database.sql index f0d5854623..1d632eb3fb 100644 --- a/database.sql +++ b/database.sql @@ -1132,7 +1132,7 @@ CREATE TABLE IF NOT EXISTS `post-delivery` ( FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`inbox-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE -) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Status of ActivityPub inboxes'; +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for posts for the batch processing'; -- -- TABLE post-delivery-data diff --git a/doc/database.md b/doc/database.md index 961e8183ab..bd5bef445b 100644 --- a/doc/database.md +++ b/doc/database.md @@ -50,7 +50,7 @@ Database Tables | [post-category](help/database/db_post-category) | post relation to categories | | [post-collection](help/database/db_post-collection) | Collection of posts | | [post-content](help/database/db_post-content) | Content for all posts | -| [post-delivery](help/database/db_post-delivery) | Status of ActivityPub inboxes | +| [post-delivery](help/database/db_post-delivery) | Delivery data for posts for the batch processing | | [post-delivery-data](help/database/db_post-delivery-data) | Delivery data for items | | [post-link](help/database/db_post-link) | Post related external links | | [post-media](help/database/db_post-media) | Attached media | diff --git a/doc/database/db_post-delivery.md b/doc/database/db_post-delivery.md index 1937e059a5..79e858193a 100644 --- a/doc/database/db_post-delivery.md +++ b/doc/database/db_post-delivery.md @@ -1,7 +1,7 @@ Table post-delivery =========== -Status of ActivityPub inboxes +Delivery data for posts for the batch processing Fields ------ diff --git a/mod/item.php b/mod/item.php index 34475188ea..71c1568f6d 100644 --- a/mod/item.php +++ b/mod/item.php @@ -672,7 +672,8 @@ function item_post(App $a) { 'attach' => $datarray['attach'], 'file' => $datarray['file'], 'edited' => DateTimeFormat::utcNow(), - 'changed' => DateTimeFormat::utcNow()]; + 'changed' => DateTimeFormat::utcNow() + ]; Item::update($fields, ['id' => $post_id]); Item::updateDisplayCache($datarray['uri-id']); diff --git a/src/Worker/CleanWorkerQueue.php b/src/Worker/CleanWorkerQueue.php deleted file mode 100644 index eb9b329a55..0000000000 --- a/src/Worker/CleanWorkerQueue.php +++ /dev/null @@ -1,51 +0,0 @@ -. - * - */ - -namespace Friendica\Worker; - -use Friendica\Core\Worker; -use Friendica\Database\DBA; -use Friendica\DI; -use Friendica\Util\DateTimeFormat; - -/** - * Delete all done workerqueue entries - */ -class CleanWorkerQueue -{ - public static function execute() - { - DBA::delete('workerqueue', ["`done` AND `executed` < ?", DateTimeFormat::utc('now - 1 hour')]); - - // Optimizing this table only last seconds - if (DI::config()->get('system', 'optimize_tables')) { - // We are acquiring the two locks from the worker to avoid locking problems - if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) { - if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) { - DBA::e("OPTIMIZE TABLE `workerqueue`"); - DBA::e("OPTIMIZE TABLE `process`"); - DI::lock()->release(Worker::LOCK_WORKER); - } - DI::lock()->release(Worker::LOCK_PROCESS); - } - } - } -} diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 0cf5e26323..01c54fed18 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -92,9 +92,6 @@ class Cron Worker::add(PRIORITY_LOW, 'PullDirectory'); } - // Delete all done workerqueue entries - Worker::add(PRIORITY_LOW, 'CleanWorkerQueue'); - // Clear cache entries Worker::add(PRIORITY_LOW, 'ClearCache'); diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index df0d469fce..08634290b7 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -1158,7 +1158,7 @@ return [ ] ], "post-delivery" => [ - "comment" => "Status of ActivityPub inboxes", + "comment" => "Delivery data for posts for the batch processing", "fields" => [ "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], "inbox-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Item-uri id of inbox url"], From 1e7a55180a3740f4b6e5ed8dd40ecb6b9d84cbc4 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 5 May 2022 22:40:20 +0200 Subject: [PATCH 35/35] Update src/Content/Conversation.php Co-authored-by: Hypolite Petovan --- src/Content/Conversation.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Content/Conversation.php b/src/Content/Conversation.php index 9f93d0efad..4ad1890a26 100644 --- a/src/Content/Conversation.php +++ b/src/Content/Conversation.php @@ -963,8 +963,7 @@ class Conversation $condition = DBA::mergeConditions($condition, ["`uid` IN (0, ?) AND (`vid` != ? OR `vid` IS NULL)", $uid, Verb::getID(Activity::FOLLOW)]); - $params = ['order' => ['uri-id' => false, 'uid' => true]]; - $thread_parents = Post::select(['uri-id', 'causer-id'], $condition, $params); + $thread_parents = Post::select(['uri-id', 'causer-id'], $condition, ['order' => ['uri-id' => false, 'uid' => true]]); $thr_parent = [];