From 610f8a086fd912b27c711a0d11c7bcdf5a27739f Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 May 2022 08:17:41 +0000 Subject: [PATCH 1/6] Use cache key constants --- src/Protocol/ActivityPub/Transmitter.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index 6254bdf41b..4651d6778f 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -44,7 +44,6 @@ use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Relay; use Friendica\Util\DateTimeFormat; use Friendica\Util\HTTPSignature; -use Friendica\Util\JsonLD; use Friendica\Util\LDSignature; use Friendica\Util\Map; use Friendica\Util\Network; @@ -59,6 +58,10 @@ use Friendica\Util\XML; */ class Transmitter { + const CACHEKEY_FEATURED = 'transmitter:getFeatured:'; + const CACHEKEY_CONTACTS = 'transmitter:getContacts:'; + const CACHEKEY_OUTBOX = 'transmitter:getOutbox:'; + /** * Add relay servers to the list of inboxes * @@ -159,7 +162,7 @@ class Transmitter public static function getContacts(array $owner, array $rel, string $module, int $page = null, string $requester = null, $nocache = false) { if (empty($page)) { - $cachekey = 'transmitter:getContacts:' . $module . ':'. $owner['uid']; + $cachekey = self::CACHEKEY_CONTACTS . $module . ':'. $owner['uid']; $result = DI::cache()->get($cachekey); if (!$nocache && !is_null($result)) { return $result; @@ -251,7 +254,7 @@ class Transmitter public static function getOutbox(array $owner, int $page = null, string $requester = '', $nocache = false) { if (empty($page)) { - $cachekey = 'transmitter:getOutbox:' . $owner['uid']; + $cachekey = self::CACHEKEY_OUTBOX . $owner['uid']; $result = DI::cache()->get($cachekey); if (!$nocache && !is_null($result)) { return $result; @@ -339,15 +342,16 @@ class Transmitter */ public static function getFeatured(array $owner, int $page = null, $nocache = false) { - $owner_cid = Contact::getIdForURL($owner['url'], 0, false); if (empty($page)) { - $cachekey = 'transmitter:getFeatured:' . $owner_cid; + $cachekey = self::CACHEKEY_FEATURED . $owner['uid']; $result = DI::cache()->get($cachekey); if (!$nocache && !is_null($result)) { return $result; } } + $owner_cid = Contact::getIdForURL($owner['url'], 0, false); + $condition = ["`uri-id` IN (SELECT `uri-id` FROM `collection-view` WHERE `cid` = ? AND `type` = ?)", $owner_cid, Post\Collection::FEATURED]; From 73019284ce0b57e0cd8177f2a4739beddeac9753 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 May 2022 12:32:25 +0000 Subject: [PATCH 2/6] Increase cache lifespan / clear cache upon changes --- src/Model/Contact.php | 23 +++++++++++++++++++--- src/Model/Item.php | 4 ++++ src/Model/Post/Collection.php | 16 +++++++++++++-- src/Module/Api/Mastodon/Statuses/Pin.php | 2 +- src/Module/Api/Mastodon/Statuses/Unpin.php | 2 +- src/Module/Item/Pin.php | 4 ++-- src/Protocol/ActivityPub/Transmitter.php | 8 ++++---- 7 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/Model/Contact.php b/src/Model/Contact.php index d0febc8bba..eeaf5f81df 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -797,11 +797,13 @@ class Contact public static function remove($id) { // We want just to make sure that we don't delete our "self" contact - $contact = DBA::selectFirst('contact', ['uri-id', 'photo', 'thumb', 'micro'], ['id' => $id, 'self' => false]); + $contact = DBA::selectFirst('contact', ['uri-id', 'photo', 'thumb', 'micro', 'uid'], ['id' => $id, 'self' => false]); if (!DBA::isResult($contact)) { return; } + self::clearFollowerFollowingEndpointCache($contact['uid']); + // Archive the contact self::update(['archive' => true, 'network' => Protocol::PHANTOM, 'deleted' => true], ['id' => $id]); @@ -899,6 +901,15 @@ class Contact self::remove($contact['id']); } + private static function clearFollowerFollowingEndpointCache(int $uid) + { + if (empty($uid)) { + return; + } + + DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'followers:' . $uid); + DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'following:' . $uid); + } /** * Marks a contact for archival after a communication issue delay @@ -1764,7 +1775,7 @@ class Contact break; default: /** - * Use a random picture. + * Use a random picture. * The service provides random pictures from Unsplash. * @license https://unsplash.com/license */ @@ -2321,7 +2332,7 @@ class Contact Worker::add(PRIORITY_LOW, 'FetchFeaturedPosts', $ret['url']); } } - + $ret['last-item'] = Probe::getLastUpdate($ret); Logger::info('Fetched last item', ['id' => $id, 'probed_url' => $ret['url'], 'last-item' => $ret['last-item'], 'callstack' => System::callstack(20)]); } @@ -2707,6 +2718,8 @@ class Contact $contact = DBA::selectFirst('contact', [], ['id' => $cid]); } + self::clearFollowerFollowingEndpointCache($importer['uid']); + if (!empty($contact)) { if (!empty($contact['pending'])) { Logger::info('Pending contact request already exists.', ['url' => $url, 'uid' => $importer['uid']]); @@ -2830,6 +2843,8 @@ class Contact return; } + self::clearFollowerFollowingEndpointCache($contact['uid']); + $cdata = self::getPublicAndUserContactID($contact['id'], $contact['uid']); DI::notification()->deleteForUserByVerb($contact['uid'], Activity::FOLLOW, ['actor-id' => $cdata['public']]); @@ -2844,6 +2859,8 @@ class Contact */ public static function removeSharer(array $contact) { + self::clearFollowerFollowingEndpointCache($contact['uid']); + if ($contact['rel'] == self::SHARING || in_array($contact['network'], [Protocol::FEED, Protocol::MAIL])) { self::remove($contact['id']); } else { diff --git a/src/Model/Item.php b/src/Model/Item.php index 8b2a4651dc..822580aa20 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1223,6 +1223,10 @@ class Item self::updateDisplayCache($posted_item['uri-id']); } + if ($posted_item['origin'] && ($posted_item['uid'] != 0) && in_array($posted_item['gravity'], [GRAVITY_PARENT, GRAVITY_COMMENT])) { + DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_OUTBOX . $posted_item['uid']); + } + return $post_user_id; } diff --git a/src/Model/Post/Collection.php b/src/Model/Post/Collection.php index 476e9d3b33..34b5dc6dee 100644 --- a/src/Model/Post/Collection.php +++ b/src/Model/Post/Collection.php @@ -24,6 +24,8 @@ namespace Friendica\Model\Post; use Friendica\Database\DBA; use BadMethodCallException; use Friendica\Database\Database; +use Friendica\DI; +use Friendica\Protocol\ActivityPub; class Collection { @@ -34,14 +36,19 @@ class Collection * * @param integer $uri_id * @param integer $type + * @param integer $cache_uid If set to a non zero value, the featured cache is cleared */ - public static function add(int $uri_id, int $type) + public static function add(int $uri_id, int $type, int $cache_uid = 0) { if (empty($uri_id)) { throw new BadMethodCallException('Empty URI_id'); } DBA::insert('post-collection', ['uri-id' => $uri_id, 'type' => $type], Database::INSERT_IGNORE); + + if (!empty($cache_uid) && ($type == self::FEATURED)) { + DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_FEATURED . $cache_uid); + } } /** @@ -49,14 +56,19 @@ class Collection * * @param integer $uri_id * @param integer $type + * @param integer $cache_uid If set to a non zero value, the featured cache is cleared */ - public static function remove(int $uri_id, int $type) + public static function remove(int $uri_id, int $type, int $cache_uid = 0) { if (empty($uri_id)) { throw new BadMethodCallException('Empty URI_id'); } DBA::delete('post-collection', ['uri-id' => $uri_id, 'type' => $type]); + + if (!empty($cache_uid) && ($type == self::FEATURED)) { + DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_FEATURED . $cache_uid); + } } /** diff --git a/src/Module/Api/Mastodon/Statuses/Pin.php b/src/Module/Api/Mastodon/Statuses/Pin.php index 61a675b612..6bf8a67396 100644 --- a/src/Module/Api/Mastodon/Statuses/Pin.php +++ b/src/Module/Api/Mastodon/Statuses/Pin.php @@ -46,7 +46,7 @@ class Pin extends BaseApi DI::mstdnError()->RecordNotFound(); } - Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED); + Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED, $uid); System::jsonExit(DI::mstdnStatus()->createFromUriId($this->parameters['id'], $uid)->toArray()); } diff --git a/src/Module/Api/Mastodon/Statuses/Unpin.php b/src/Module/Api/Mastodon/Statuses/Unpin.php index e54df31ebd..ee6763ff28 100644 --- a/src/Module/Api/Mastodon/Statuses/Unpin.php +++ b/src/Module/Api/Mastodon/Statuses/Unpin.php @@ -46,7 +46,7 @@ class Unpin extends BaseApi DI::mstdnError()->RecordNotFound(); } - Post\Collection::remove($this->parameters['id'], Post\Collection::FEATURED); + Post\Collection::remove($this->parameters['id'], Post\Collection::FEATURED, $uid); System::jsonExit(DI::mstdnStatus()->createFromUriId($this->parameters['id'], $uid)->toArray()); } diff --git a/src/Module/Item/Pin.php b/src/Module/Item/Pin.php index 3fdd783c6d..9735e495c3 100644 --- a/src/Module/Item/Pin.php +++ b/src/Module/Item/Pin.php @@ -60,9 +60,9 @@ class Pin extends BaseModule $pinned = !$item['featured']; if ($pinned) { - Post\Collection::add($item['uri-id'], Post\Collection::FEATURED); + Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, local_user()); } else { - Post\Collection::remove($item['uri-id'], Post\Collection::FEATURED); + Post\Collection::remove($item['uri-id'], Post\Collection::FEATURED, local_user()); } // See if we've been passed a return path to redirect to diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index 4651d6778f..023bd5c83c 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -205,7 +205,7 @@ class Transmitter if (!$show_contacts) { if (!empty($cachekey)) { - DI::cache()->set($cachekey, $data, Duration::QUARTER_HOUR); + DI::cache()->set($cachekey, $data, Duration::DAY); } return $data; @@ -233,7 +233,7 @@ class Transmitter } if (!empty($cachekey)) { - DI::cache()->set($cachekey, $data, Duration::QUARTER_HOUR); + DI::cache()->set($cachekey, $data, Duration::DAY); } return $data; @@ -323,7 +323,7 @@ class Transmitter } if (!empty($cachekey)) { - DI::cache()->set($cachekey, $data, Duration::QUARTER_HOUR); + DI::cache()->set($cachekey, $data, Duration::DAY); } return $data; @@ -407,7 +407,7 @@ class Transmitter $data['orderedItems'] = $list; if (!empty($cachekey)) { - DI::cache()->set($cachekey, $data, Duration::QUARTER_HOUR); + DI::cache()->set($cachekey, $data, Duration::DAY); } return $data; From 75534fa3f36690638fa636200892c24055616180 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 May 2022 12:38:39 +0000 Subject: [PATCH 3/6] Fix warning " Trying to access array offset on value of type bool" --- src/Model/Item.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index 822580aa20..50ef7c8c06 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -2793,7 +2793,7 @@ class Item $shared_item = Post::selectFirst(['uri-id', 'plink', 'has-media'], ['guid' => $shared['guid']]); $shared_uri_id = $shared_item['uri-id'] ?? 0; $shared_links = [strtolower($shared_item['plink'] ?? '')]; - $shared_attachments = Post\Media::splitAttachments($shared_uri_id, $shared['guid'], [], $shared_item['has-media']); + $shared_attachments = Post\Media::splitAttachments($shared_uri_id, $shared['guid'], [], $shared_item['has-media'] ?? false); $shared_links = array_merge($shared_links, array_column($shared_attachments['visual'], 'url')); $shared_links = array_merge($shared_links, array_column($shared_attachments['link'], 'url')); $shared_links = array_merge($shared_links, array_column($shared_attachments['additional'], 'url')); From 4e3e9dc7638d30fc481be7ebdcfe2139ad12c75e Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 May 2022 12:46:55 +0000 Subject: [PATCH 4/6] Cache the profile as well --- src/Model/Item.php | 1 + src/Protocol/ActivityPub/Transmitter.php | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index 50ef7c8c06..8bf133f5d8 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1225,6 +1225,7 @@ class Item if ($posted_item['origin'] && ($posted_item['uid'] != 0) && in_array($posted_item['gravity'], [GRAVITY_PARENT, GRAVITY_COMMENT])) { DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_OUTBOX . $posted_item['uid']); + DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_PROFILE . $posted_item['uid']); } return $post_user_id; diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index 023bd5c83c..6701f4254c 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -61,6 +61,7 @@ class Transmitter const CACHEKEY_FEATURED = 'transmitter:getFeatured:'; const CACHEKEY_CONTACTS = 'transmitter:getContacts:'; const CACHEKEY_OUTBOX = 'transmitter:getOutbox:'; + const CACHEKEY_PROFILE = 'transmitter:getProfile:'; /** * Add relay servers to the list of inboxes @@ -429,12 +430,19 @@ class Transmitter * Return the ActivityPub profile of the given user * * @param int $uid User ID + * @param boolean $nocache Wether to bypass caching * @return array with profile data * @throws HTTPException\NotFoundException * @throws HTTPException\InternalServerErrorException */ - public static function getProfile(int $uid): array + public static function getProfile(int $uid, $nocache = false): array { + $cachekey = self::CACHEKEY_PROFILE . $uid; + $result = DI::cache()->get($cachekey); + if (!$nocache && !is_null($result)) { + return $result; + } + $owner = User::getOwnerDataById($uid); if (!isset($owner['id'])) { DI::logger()->error('Unable to find owner data for uid', ['uid' => $uid, 'callstack' => System::callstack(20)]); @@ -528,6 +536,8 @@ class Transmitter $data['generator'] = self::getService(); + DI::cache()->set($cachekey, $data, Duration::DAY); + // tags: https://kitty.town/@inmysocks/100656097926961126.json return $data; } From c3fd8b39aa3e03cf95979e04d371e8192dc908dc Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 May 2022 12:53:31 +0000 Subject: [PATCH 5/6] Caching for profiles removed --- src/Model/Item.php | 1 - src/Protocol/ActivityPub/Transmitter.php | 12 +----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index 8bf133f5d8..50ef7c8c06 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1225,7 +1225,6 @@ class Item if ($posted_item['origin'] && ($posted_item['uid'] != 0) && in_array($posted_item['gravity'], [GRAVITY_PARENT, GRAVITY_COMMENT])) { DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_OUTBOX . $posted_item['uid']); - DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_PROFILE . $posted_item['uid']); } return $post_user_id; diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index 6701f4254c..023bd5c83c 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -61,7 +61,6 @@ class Transmitter const CACHEKEY_FEATURED = 'transmitter:getFeatured:'; const CACHEKEY_CONTACTS = 'transmitter:getContacts:'; const CACHEKEY_OUTBOX = 'transmitter:getOutbox:'; - const CACHEKEY_PROFILE = 'transmitter:getProfile:'; /** * Add relay servers to the list of inboxes @@ -430,19 +429,12 @@ class Transmitter * Return the ActivityPub profile of the given user * * @param int $uid User ID - * @param boolean $nocache Wether to bypass caching * @return array with profile data * @throws HTTPException\NotFoundException * @throws HTTPException\InternalServerErrorException */ - public static function getProfile(int $uid, $nocache = false): array + public static function getProfile(int $uid): array { - $cachekey = self::CACHEKEY_PROFILE . $uid; - $result = DI::cache()->get($cachekey); - if (!$nocache && !is_null($result)) { - return $result; - } - $owner = User::getOwnerDataById($uid); if (!isset($owner['id'])) { DI::logger()->error('Unable to find owner data for uid', ['uid' => $uid, 'callstack' => System::callstack(20)]); @@ -536,8 +528,6 @@ class Transmitter $data['generator'] = self::getService(); - DI::cache()->set($cachekey, $data, Duration::DAY); - // tags: https://kitty.town/@inmysocks/100656097926961126.json return $data; } From 0b50dc363bc5db93f96c22fc6a5546b17d0bdb79 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 May 2022 15:39:04 +0000 Subject: [PATCH 6/6] Caching of "noscrape" --- src/Model/Contact.php | 2 ++ src/Module/NoScrape.php | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/Model/Contact.php b/src/Model/Contact.php index eeaf5f81df..bf1a50cbc4 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -35,6 +35,7 @@ use Friendica\Core\Worker; use Friendica\Database\Database; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Module\NoScrape; use Friendica\Network\HTTPException; use Friendica\Network\Probe; use Friendica\Protocol\Activity; @@ -909,6 +910,7 @@ class Contact DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'followers:' . $uid); DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'following:' . $uid); + DI::cache()->delete(NoScrape::CACHEKEY . $uid); } /** diff --git a/src/Module/NoScrape.php b/src/Module/NoScrape.php index c315ff0454..fbc01c2d3c 100644 --- a/src/Module/NoScrape.php +++ b/src/Module/NoScrape.php @@ -22,6 +22,7 @@ namespace Friendica\Module; use Friendica\BaseModule; +use Friendica\Core\Cache\Enum\Duration; use Friendica\Core\Protocol; use Friendica\Core\System; use Friendica\Database\DBA; @@ -35,6 +36,8 @@ use Friendica\Model\User; */ class NoScrape extends BaseModule { + const CACHEKEY = 'noscrape:'; + protected function rawContent(array $request = []) { $a = DI::app(); @@ -55,6 +58,12 @@ class NoScrape extends BaseModule System::jsonError(404, 'Profile not found'); } + $cachekey = self::CACHEKEY . $owner['uid']; + $result = DI::cache()->get($cachekey); + if (!is_null($result)) { + System::jsonExit($result); + } + $json_info = [ 'addr' => $owner['addr'], 'nick' => $which, @@ -126,6 +135,8 @@ class NoScrape extends BaseModule } } + DI::cache()->set($cachekey, $json_info, Duration::DAY); + System::jsonExit($json_info); } }