From b6ab8c9e9455728a79fc1e4995651c845ded5dbf Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 Mar 2020 08:12:29 +0000 Subject: [PATCH 01/11] New function to fetch endpoint items --- src/Protocol/ActivityPub.php | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index f1cd652f47..c22bbb333d 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -191,7 +191,7 @@ class ActivityPub */ public static function fetchOutbox($url, $uid) { - $data = self::fetchContent($url); + $data = self::fetchContent($url, $uid); if (empty($data)) { return; } @@ -213,6 +213,37 @@ class ActivityPub } } + /** + * Fetch items from AP endpoints + * + * @param string $url Address of the endpoint + * @param integer $uid Optional user id + * @return array Endpoint items + */ + public static function fetchItems(string $url, int $uid = 0) + { + $data = self::fetchContent($url, $uid); + if (empty($data)) { + return []; + } + + if (!empty($data['orderedItems'])) { + $items = $data['orderedItems']; + } elseif (!empty($data['first']['orderedItems'])) { + $items = $data['first']['orderedItems']; + } elseif (!empty($data['first'])) { + return self::fetchItems($data['first'], $uid); + } else { + $items = []; + } + + if (!empty($data['next'])) { + $items = array_merge($items, self::fetchItems($data['next'], $uid)); + } + + return $items; + } + /** * Checks if the given contact url does support ActivityPub * From 4d6953bf795d90d7c636047fc51c6b8c03cd7595 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 Mar 2020 08:13:18 +0000 Subject: [PATCH 02/11] Don't show feeds / avoiding errors with empty data --- src/Protocol/ActivityPub/Transmitter.php | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index 0b80e97863..ead0c84c0d 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -71,7 +71,7 @@ class Transmitter */ public static function getFollowers($owner, $page = null) { - $condition = ['rel' => [Contact::FOLLOWER, Contact::FRIEND], 'network' => Protocol::NATIVE_SUPPORT, 'uid' => $owner['uid'], + $condition = ['rel' => [Contact::FOLLOWER, Contact::FRIEND], 'network' => Protocol::FEDERATED, 'uid' => $owner['uid'], 'self' => false, 'deleted' => false, 'hidden' => false, 'archive' => false, 'pending' => false]; $count = DBA::count('contact', $condition); @@ -120,7 +120,7 @@ class Transmitter */ public static function getFollowing($owner, $page = null) { - $condition = ['rel' => [Contact::SHARING, Contact::FRIEND], 'network' => Protocol::NATIVE_SUPPORT, 'uid' => $owner['uid'], + $condition = ['rel' => [Contact::SHARING, Contact::FRIEND], 'network' => Protocol::FEDERATED, 'uid' => $owner['uid'], 'self' => false, 'deleted' => false, 'hidden' => false, 'archive' => false, 'pending' => false]; $count = DBA::count('contact', $condition); @@ -341,6 +341,10 @@ class Transmitter } foreach ($activity[$element] as $receiver) { + if (empty($receiver)) { + continue; + } + if ($receiver == $profile['followers'] && !empty($item_profile['followers'])) { $permissions[$element][] = $item_profile['followers']; } elseif (!in_array($receiver, $exclude)) { @@ -649,7 +653,7 @@ class Transmitter $blindcopy = in_array($element, ['bto', 'bcc']); foreach ($permissions[$element] as $receiver) { - if (Network::isUrlBlocked($receiver)) { + if (empty($receiver) || Network::isUrlBlocked($receiver)) { continue; } From a6e7b5f295175ad112460ba41ff6a5c2904b499d Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 Mar 2020 22:03:24 +0000 Subject: [PATCH 03/11] GContact discovery added --- src/Model/GContact.php | 70 ++++++++++++++++++++++++ src/Module/Admin/Site.php | 3 + src/Protocol/ActivityPub.php | 4 +- src/Protocol/ActivityPub/Transmitter.php | 2 +- src/Worker/UpdateGContact.php | 6 +- view/templates/admin/site.tpl | 1 + view/theme/frio/templates/admin/site.tpl | 1 + 7 files changed, 83 insertions(+), 4 deletions(-) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 95afc1be6e..bd2d14064e 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -28,6 +28,7 @@ use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Core\System; use Friendica\Core\Search; +use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Network\Probe; @@ -1275,6 +1276,75 @@ class GContact } } + /** + * Fetches the followers of a given profile and adds them + * + * @param string $url URL of a profile + * @return void + */ + public static function discoverFollowers(string $url) + { + $apcontact = APContact::getByURL($url); + + if (!empty($apcontact['followers']) && is_string($apcontact['followers'])) { + $followers = ActivityPub::fetchItems($apcontact['followers']); + Logger::info('Discover AP followers', ['url' => $url, 'contacts' => count($followers)]); + foreach ($followers as $follower) { + if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($follower))])) { + continue; + } + Logger::info('Discover new AP contact', ['url' => $follower]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $follower); + } + Logger::info('AP followers discovery finished', ['url' => $url]); + } + + if (!empty($apcontact['following']) && is_string($apcontact['following'])) { + $followings = ActivityPub::fetchItems($apcontact['following']); + Logger::info('Discover AP followings', ['url' => $url, 'contacts' => count($followings)]); + foreach ($followings as $following) { + if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($following))])) { + continue; + } + Logger::info('Discover new AP contact', ['url' => $following]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $following); + } + Logger::info('AP followings discovery finished', ['url' => $url]); + } + + + $data = Probe::uri($url); + if (empty($data['poco'])) { + return; + } + + $curlResult = Network::curl($data['poco']); + if (!$curlResult->isSuccess()) { + return; + } + $poco = json_decode($curlResult->getBody(), true); + if (empty($poco['entry'])) { + return; + } + + Logger::info('PoCo Discovery started', ['url' => $url, 'contacts' => count($poco['entry'])]); + + foreach ($poco['entry'] as $entries) { + if (!empty($entries['urls'])) { + foreach ($entries['urls'] as $url) { + if ($url['type'] == 'profile') { + if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($url['value']))])) { + continue; + } + Logger::info('Discover new PoCo contact', ['url' => $$url['value']]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $$url['value']); + } + } + } + } + Logger::info('PoCo Discovery finished', ['url' => $url]); + } + /** * Returns a random, global contact of the current node * diff --git a/src/Module/Admin/Site.php b/src/Module/Admin/Site.php index ec6f01afa0..078f169299 100644 --- a/src/Module/Admin/Site.php +++ b/src/Module/Admin/Site.php @@ -178,6 +178,7 @@ class Site extends BaseAdmin $optimize_max_tablesize = (!empty($_POST['optimize_max_tablesize']) ? intval(trim($_POST['optimize_max_tablesize'])) : 100); $optimize_fragmentation = (!empty($_POST['optimize_fragmentation']) ? intval(trim($_POST['optimize_fragmentation'])) : 30); $poco_completion = (!empty($_POST['poco_completion']) ? intval(trim($_POST['poco_completion'])) : false); + $gcontact_discovery = (!empty($_POST['gcontact_discovery']) ? intval(trim($_POST['gcontact_discovery'])) : false); $poco_requery_days = (!empty($_POST['poco_requery_days']) ? intval(trim($_POST['poco_requery_days'])) : 7); $poco_discovery = (!empty($_POST['poco_discovery']) ? intval(trim($_POST['poco_discovery'])) : PortableContact::DISABLED); $poco_discovery_since = (!empty($_POST['poco_discovery_since']) ? intval(trim($_POST['poco_discovery_since'])) : 30); @@ -305,6 +306,7 @@ class Site extends BaseAdmin DI::config()->set('system', 'optimize_max_tablesize', $optimize_max_tablesize); DI::config()->set('system', 'optimize_fragmentation', $optimize_fragmentation); DI::config()->set('system', 'poco_completion' , $poco_completion); + DI::config()->set('system', 'gcontact_discovery' , $gcontact_discovery); DI::config()->set('system', 'poco_requery_days' , $poco_requery_days); DI::config()->set('system', 'poco_discovery' , $poco_discovery); DI::config()->set('system', 'poco_discovery_since' , $poco_discovery_since); @@ -669,6 +671,7 @@ class Site extends BaseAdmin '$optimize_fragmentation' => ['optimize_fragmentation', DI::l10n()->t('Minimum level of fragmentation'), DI::config()->get('system', 'optimize_fragmentation', 30), DI::l10n()->t('Minimum fragmenation level to start the automatic optimization - default value is 30%.')], '$poco_completion' => ['poco_completion', DI::l10n()->t('Periodical check of global contacts'), DI::config()->get('system', 'poco_completion'), DI::l10n()->t('If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.')], + '$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), DI::config()->get('system', 'gcontact_discovery'), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts.')], '$poco_requery_days' => ['poco_requery_days', DI::l10n()->t('Days between requery'), DI::config()->get('system', 'poco_requery_days'), DI::l10n()->t('Number of days after which a server is requeried for his contacts.')], '$poco_discovery' => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), (string)intval(DI::config()->get('system', 'poco_discovery')), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices], '$poco_discovery_since' => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), (string)intval(DI::config()->get('system', 'poco_discovery_since')), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices], diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index c22bbb333d..894c7f6d36 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -231,13 +231,13 @@ class ActivityPub $items = $data['orderedItems']; } elseif (!empty($data['first']['orderedItems'])) { $items = $data['first']['orderedItems']; - } elseif (!empty($data['first'])) { + } elseif (!empty($data['first']) && is_string($data['first'])) { return self::fetchItems($data['first'], $uid); } else { $items = []; } - if (!empty($data['next'])) { + if (!empty($data['next']) && is_string($data['next'])) { $items = array_merge($items, self::fetchItems($data['next'], $uid)); } diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index ead0c84c0d..f6c9311fcc 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -345,7 +345,7 @@ class Transmitter continue; } - if ($receiver == $profile['followers'] && !empty($item_profile['followers'])) { + if (!empty($profile['followers']) && $receiver == $profile['followers'] && !empty($item_profile['followers'])) { $permissions[$element][] = $item_profile['followers']; } elseif (!in_array($receiver, $exclude)) { $permissions[$element][] = $receiver; diff --git a/src/Worker/UpdateGContact.php b/src/Worker/UpdateGContact.php index 7bdaec464c..0d0d85fc01 100644 --- a/src/Worker/UpdateGContact.php +++ b/src/Worker/UpdateGContact.php @@ -22,8 +22,8 @@ namespace Friendica\Worker; use Friendica\Core\Logger; +use Friendica\DI; use Friendica\Model\GContact; -use Friendica\Database\DBA; class UpdateGContact { @@ -39,5 +39,9 @@ class UpdateGContact $success = GContact::updateFromProbe($url, $force); Logger::info('Updated from probe', ['url' => $url, 'force' => $force, 'success' => $success]); + + if ($success && DI::config()->get('system', 'gcontact_discovery')) { + GContact::discoverFollowers($url); + } } } diff --git a/view/templates/admin/site.tpl b/view/templates/admin/site.tpl index 9ccda33eeb..9e601b078c 100644 --- a/view/templates/admin/site.tpl +++ b/view/templates/admin/site.tpl @@ -98,6 +98,7 @@

{{$portable_contacts}}

{{include file="field_checkbox.tpl" field=$poco_completion}} + {{include file="field_checkbox.tpl" field=$gcontact_discovery}} {{include file="field_input.tpl" field=$poco_requery_days}} {{include file="field_select.tpl" field=$poco_discovery}} {{include file="field_select.tpl" field=$poco_discovery_since}} diff --git a/view/theme/frio/templates/admin/site.tpl b/view/theme/frio/templates/admin/site.tpl index e13b2960e0..e4b05d453d 100644 --- a/view/theme/frio/templates/admin/site.tpl +++ b/view/theme/frio/templates/admin/site.tpl @@ -213,6 +213,7 @@
{{include file="field_checkbox.tpl" field=$poco_completion}} + {{include file="field_checkbox.tpl" field=$gcontact_discovery}} {{include file="field_input.tpl" field=$poco_requery_days}} {{include file="field_select.tpl" field=$poco_discovery}} {{include file="field_select.tpl" field=$poco_discovery_since}} From 652a4ec9c7d4fa5aa191d7b1f1bbf29d503c7abf Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 Mar 2020 22:17:17 +0000 Subject: [PATCH 04/11] Bugfixing --- src/Model/GContact.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index bd2d14064e..dc62ec460f 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -1331,13 +1331,13 @@ class GContact foreach ($poco['entry'] as $entries) { if (!empty($entries['urls'])) { - foreach ($entries['urls'] as $url) { - if ($url['type'] == 'profile') { - if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($url['value']))])) { + foreach ($entries['urls'] as $entry) { + if ($entry['type'] == 'profile') { + if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($entry['value']))])) { continue; } - Logger::info('Discover new PoCo contact', ['url' => $$url['value']]); - Worker::add(PRIORITY_LOW, 'UpdateGContact', $$url['value']); + Logger::info('Discover new PoCo contact', ['url' => $entry['value']]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $entry['value']); } } } From 2f63249f3be7689766d80e7dc87f12eca23e871c Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 5 Mar 2020 22:24:31 +0000 Subject: [PATCH 05/11] Only log when there is data --- src/Model/GContact.php | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index dc62ec460f..a9233e8cf2 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -1288,28 +1288,32 @@ class GContact if (!empty($apcontact['followers']) && is_string($apcontact['followers'])) { $followers = ActivityPub::fetchItems($apcontact['followers']); - Logger::info('Discover AP followers', ['url' => $url, 'contacts' => count($followers)]); - foreach ($followers as $follower) { - if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($follower))])) { - continue; + if (!empty($followers)) { + Logger::info('Discover AP followers', ['url' => $url, 'contacts' => count($followers)]); + foreach ($followers as $follower) { + if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($follower))])) { + continue; + } + Logger::info('Discover new AP contact', ['url' => $follower]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $follower); } - Logger::info('Discover new AP contact', ['url' => $follower]); - Worker::add(PRIORITY_LOW, 'UpdateGContact', $follower); + Logger::info('AP followers discovery finished', ['url' => $url]); } - Logger::info('AP followers discovery finished', ['url' => $url]); } if (!empty($apcontact['following']) && is_string($apcontact['following'])) { $followings = ActivityPub::fetchItems($apcontact['following']); - Logger::info('Discover AP followings', ['url' => $url, 'contacts' => count($followings)]); - foreach ($followings as $following) { - if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($following))])) { - continue; + if (!empty($followings)) { + Logger::info('Discover AP followings', ['url' => $url, 'contacts' => count($followings)]); + foreach ($followings as $following) { + if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($following))])) { + continue; + } + Logger::info('Discover new AP contact', ['url' => $following]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $following); } - Logger::info('Discover new AP contact', ['url' => $following]); - Worker::add(PRIORITY_LOW, 'UpdateGContact', $following); + Logger::info('AP followings discovery finished', ['url' => $url]); } - Logger::info('AP followings discovery finished', ['url' => $url]); } From 1ce9a31ca443fb8249074acae0e139e418535d91 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 6 Mar 2020 06:44:17 +0000 Subject: [PATCH 06/11] Simplified code --- src/Model/GContact.php | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index a9233e8cf2..e61b27d84e 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -1288,34 +1288,28 @@ class GContact if (!empty($apcontact['followers']) && is_string($apcontact['followers'])) { $followers = ActivityPub::fetchItems($apcontact['followers']); - if (!empty($followers)) { - Logger::info('Discover AP followers', ['url' => $url, 'contacts' => count($followers)]); - foreach ($followers as $follower) { - if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($follower))])) { - continue; - } - Logger::info('Discover new AP contact', ['url' => $follower]); - Worker::add(PRIORITY_LOW, 'UpdateGContact', $follower); - } - Logger::info('AP followers discovery finished', ['url' => $url]); - } + } else { + $followers = []; } if (!empty($apcontact['following']) && is_string($apcontact['following'])) { $followings = ActivityPub::fetchItems($apcontact['following']); - if (!empty($followings)) { - Logger::info('Discover AP followings', ['url' => $url, 'contacts' => count($followings)]); - foreach ($followings as $following) { - if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($following))])) { - continue; - } - Logger::info('Discover new AP contact', ['url' => $following]); - Worker::add(PRIORITY_LOW, 'UpdateGContact', $following); - } - Logger::info('AP followings discovery finished', ['url' => $url]); - } + } else { + $followings = []; } + if (!empty($followers) || !empty($followings)) { + $contacts = array_unique(array_merge($followers, $followings)); + Logger::info('Discover AP contacts', ['url' => $url, 'contacts' => count($contacts)]); + foreach ($contacts as $contact) { + if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($contact))])) { + continue; + } + Logger::info('Discover new AP contact', ['url' => $contact]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact); + } + Logger::info('AP contacts discovery finished', ['url' => $url]); + } $data = Probe::uri($url); if (empty($data['poco'])) { From 10d866bad9867a3cdc27273aba4bf49484135795 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 6 Mar 2020 07:06:02 +0000 Subject: [PATCH 07/11] Only perform a PoCo discovery when hadn't been one via AP --- src/Model/GContact.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index e61b27d84e..6f41fd7325 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -1309,6 +1309,7 @@ class GContact Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact); } Logger::info('AP contacts discovery finished', ['url' => $url]); + return; } $data = Probe::uri($url); From bd77556b494f167adb675630f44b6e74aa451927 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 6 Mar 2020 08:08:49 +0000 Subject: [PATCH 08/11] New table for relations between global contacts - will replace glink in the future --- database.sql | 12 +++++++++++- src/Model/GContact.php | 17 ++++++++++++++++- static/dbstructure.config.php | 13 ++++++++++++- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/database.sql b/database.sql index 3cb87fc90a..a5e94f3f54 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2020.03-dev (Dalmatian Bellflower) --- DB_UPDATE_VERSION 1335 +-- DB_UPDATE_VERSION 1336 -- ------------------------------------------ @@ -419,6 +419,16 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( INDEX `updated` (`updated`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='global contacts'; +-- +-- TABLE gfollower +-- +CREATE TABLE IF NOT EXISTS `gfollower` ( + `gcid` int unsigned NOT NULL DEFAULT 0 COMMENT 'global contact', + `follower-gcid` int unsigned NOT NULL DEFAULT 0 COMMENT 'global contact of the follower', + PRIMARY KEY(`gcid`,`follower-gcid`), + INDEX `follower-gcid` (`follower-gcid`) +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Followers of global contacts'; + -- -- TABLE glink -- diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 6f41fd7325..86b6bb94cd 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -1299,10 +1299,25 @@ class GContact } if (!empty($followers) || !empty($followings)) { + $gcontact = DBA::selectFirst('gcontact', ['id'], ['nurl' => Strings::normaliseLink(($url))]); + $gcid = $gcontact['id']; + if (!empty($followers)) { + // Clear the follower list, since it will be recreated in the next step + DBA::delete('gfollower', ['gcid' => $gcid]); + } + $contacts = array_unique(array_merge($followers, $followings)); Logger::info('Discover AP contacts', ['url' => $url, 'contacts' => count($contacts)]); foreach ($contacts as $contact) { - if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($contact))])) { + $gcontact = DBA::selectFirst('gcontact', ['id'], ['nurl' => Strings::normaliseLink(($contact))]); + if (DBA::isResult($gcontact)) { + if (in_array($contact, $followers)) { + $fields = ['gcid' => $gcid, 'follower-gcid' => $gcontact['id']]; + } elseif (in_array($contact, $followings)) { + $fields = ['gcid' => $gcontact['id'], 'follower-gcid' => $gcid]; + } + Logger::info('Set relation between contacts', $fields); + DBA::update('gfollower', $fields, $fields, true); continue; } Logger::info('Discover new AP contact', ['url' => $contact]); diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 0c0b289414..11695b5924 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -51,7 +51,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1335); + define('DB_UPDATE_VERSION', 1336); } return [ @@ -490,6 +490,17 @@ return [ "updated" => ["updated"], ] ], + "gfollower" => [ + "comment" => "Followers of global contacts", + "fields" => [ + "gcid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["gcontact" => "id"], "comment" => "global contact"], + "follower-gcid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["gcontact" => "id"], "comment" => "global contact of the follower"], + ], + "indexes" => [ + "PRIMARY" => ["gcid", "follower-gcid"], + "follower-gcid" => ["follower-gcid"], + ] + ], "glink" => [ "comment" => "'friends of friends' linkages derived from poco", "fields" => [ From 9adf09be51fa35920aaed3d2463089caa17be75f Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 6 Mar 2020 13:51:36 +0000 Subject: [PATCH 09/11] Don't delete and recreate the rows over and over again --- src/Model/GContact.php | 8 ++++++-- static/dbstructure.config.php | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 86b6bb94cd..ae1bd665e3 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -1303,7 +1303,7 @@ class GContact $gcid = $gcontact['id']; if (!empty($followers)) { // Clear the follower list, since it will be recreated in the next step - DBA::delete('gfollower', ['gcid' => $gcid]); + DBA::update('gfollower', ['deleted' => true], ['gcid' => $gcid]); } $contacts = array_unique(array_merge($followers, $followings)); @@ -1317,12 +1317,16 @@ class GContact $fields = ['gcid' => $gcontact['id'], 'follower-gcid' => $gcid]; } Logger::info('Set relation between contacts', $fields); - DBA::update('gfollower', $fields, $fields, true); + DBA::update('gfollower', ['deleted' => false], $fields, true); continue; } Logger::info('Discover new AP contact', ['url' => $contact]); Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact); } + if (!empty($followers)) { + // Delete all followers that aren't undeleted + DBA::delete('gfollower', ['gcid' => $gcid, 'deleted' => true]); + } Logger::info('AP contacts discovery finished', ['url' => $url]); return; } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 11695b5924..3c9bc50fb9 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -495,6 +495,7 @@ return [ "fields" => [ "gcid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["gcontact" => "id"], "comment" => "global contact"], "follower-gcid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["gcontact" => "id"], "comment" => "global contact of the follower"], + "deleted" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "1 indicates that the connection has been deleted"], ], "indexes" => [ "PRIMARY" => ["gcid", "follower-gcid"], From d6905e29cff4bf8968fb88d8dae4c673f3095517 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 7 Mar 2020 05:31:03 +0000 Subject: [PATCH 10/11] Query the same contact only once a month --- database.sql | 2 ++ src/Model/GContact.php | 44 +++++++++++++++++++++++++++++++---- src/Module/Admin/Site.php | 2 +- src/Worker/UpdateGContact.php | 10 ++++---- static/dbstructure.config.php | 1 + 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/database.sql b/database.sql index a5e94f3f54..b4c65fb89a 100644 --- a/database.sql +++ b/database.sql @@ -393,6 +393,7 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( `updated` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', `last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', `last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', + `last_discovery` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last contact discovery', `archive_date` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', `archived` boolean NOT NULL DEFAULT '0' COMMENT '', `location` varchar(255) NOT NULL DEFAULT '' COMMENT '', @@ -425,6 +426,7 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( CREATE TABLE IF NOT EXISTS `gfollower` ( `gcid` int unsigned NOT NULL DEFAULT 0 COMMENT 'global contact', `follower-gcid` int unsigned NOT NULL DEFAULT 0 COMMENT 'global contact of the follower', + `deleted` boolean NOT NULL DEFAULT '0' COMMENT '1 indicates that the connection has been deleted', PRIMARY KEY(`gcid`,`follower-gcid`), INDEX `follower-gcid` (`follower-gcid`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Followers of global contacts'; diff --git a/src/Model/GContact.php b/src/Model/GContact.php index ae1bd665e3..ee30cf3c59 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -1282,8 +1282,30 @@ class GContact * @param string $url URL of a profile * @return void */ - public static function discoverFollowers(string $url) + public static function discoverFollowers(string $url, int $following_gcid = 0, int $follower_gcid = 0) { + $gcontact = DBA::selectFirst('gcontact', ['id', 'last_discovery'], ['nurl' => Strings::normaliseLink(($url))]); + if (!DBA::isResult($gcontact)) { + return; + } + + if ($gcontact['last_discovery'] > DateTimeFormat::utc('now - 1 month')) { + Logger::info('Last discovery was less then a month before.', ['url' => $url, 'discovery' => $gcontact['last_discovery']]); + return; + } + + $gcid = $gcontact['id']; + + if (!empty($following_gcid)) { + $fields = ['gcid' => $following_gcid, 'follower-gcid' => $gcid]; + Logger::info('Set relation for followed gcontact', $fields); + DBA::update('gfollower', ['deleted' => false], $fields, true); + } elseif (!empty($follower_gcid)) { + $fields = ['gcid' => $gcid, 'follower-gcid' => $follower_gcid]; + Logger::info('Set relation for following gcontact', $fields); + DBA::update('gfollower', ['deleted' => false], $fields, true); + } + $apcontact = APContact::getByURL($url); if (!empty($apcontact['followers']) && is_string($apcontact['followers'])) { @@ -1299,8 +1321,6 @@ class GContact } if (!empty($followers) || !empty($followings)) { - $gcontact = DBA::selectFirst('gcontact', ['id'], ['nurl' => Strings::normaliseLink(($url))]); - $gcid = $gcontact['id']; if (!empty($followers)) { // Clear the follower list, since it will be recreated in the next step DBA::update('gfollower', ['deleted' => true], ['gcid' => $gcid]); @@ -1320,14 +1340,26 @@ class GContact DBA::update('gfollower', ['deleted' => false], $fields, true); continue; } + + $follower_gcid = 0; + $following_gcid = 0; + + if (in_array($contact, $followers)) { + $following_gcid = $gcid; + } elseif (in_array($contact, $followings)) { + $follower_gcid = $gcid; + } + Logger::info('Discover new AP contact', ['url' => $contact]); - Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact, '', $following_gcid, $follower_gcid); } if (!empty($followers)) { // Delete all followers that aren't undeleted DBA::delete('gfollower', ['gcid' => $gcid, 'deleted' => true]); } - Logger::info('AP contacts discovery finished', ['url' => $url]); + + DBA::update('gcontact', ['last_discovery' => DateTimeFormat::utcNow()], ['id' => $gcid]); + Logger::info('AP contacts discovery finished, last discovery set', ['url' => $url]); return; } @@ -1360,6 +1392,8 @@ class GContact } } } + + DBA::update('gcontact', ['last_discovery' => DateTimeFormat::utcNow()], ['id' => $gcid]); Logger::info('PoCo Discovery finished', ['url' => $url]); } diff --git a/src/Module/Admin/Site.php b/src/Module/Admin/Site.php index 078f169299..846280c4f1 100644 --- a/src/Module/Admin/Site.php +++ b/src/Module/Admin/Site.php @@ -671,7 +671,7 @@ class Site extends BaseAdmin '$optimize_fragmentation' => ['optimize_fragmentation', DI::l10n()->t('Minimum level of fragmentation'), DI::config()->get('system', 'optimize_fragmentation', 30), DI::l10n()->t('Minimum fragmenation level to start the automatic optimization - default value is 30%.')], '$poco_completion' => ['poco_completion', DI::l10n()->t('Periodical check of global contacts'), DI::config()->get('system', 'poco_completion'), DI::l10n()->t('If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.')], - '$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), DI::config()->get('system', 'gcontact_discovery'), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts.')], + '$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), DI::config()->get('system', 'gcontact_discovery'), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.')], '$poco_requery_days' => ['poco_requery_days', DI::l10n()->t('Days between requery'), DI::config()->get('system', 'poco_requery_days'), DI::l10n()->t('Number of days after which a server is requeried for his contacts.')], '$poco_discovery' => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), (string)intval(DI::config()->get('system', 'poco_discovery')), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices], '$poco_discovery_since' => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), (string)intval(DI::config()->get('system', 'poco_discovery_since')), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices], diff --git a/src/Worker/UpdateGContact.php b/src/Worker/UpdateGContact.php index 0d0d85fc01..fda1a650b4 100644 --- a/src/Worker/UpdateGContact.php +++ b/src/Worker/UpdateGContact.php @@ -29,10 +29,12 @@ class UpdateGContact { /** * Update global contact via probe - * @param string $url Global contact url - * @param string $command + * @param string $url Global contact url + * @param string $command + * @param integer $following_gcid gcontact ID of the contact that is followed by this one + * @param integer $follower_gcid gcontact ID of the contact that is following this one */ - public static function execute($url, $command = '') + public static function execute(string $url, string $command = '', int $following_gcid = 0, int $follower_gcid = 0) { $force = ($command == "force"); @@ -41,7 +43,7 @@ class UpdateGContact Logger::info('Updated from probe', ['url' => $url, 'force' => $force, 'success' => $success]); if ($success && DI::config()->get('system', 'gcontact_discovery')) { - GContact::discoverFollowers($url); + GContact::discoverFollowers($url, $following_gcid, $follower_gcid); } } } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 3c9bc50fb9..8cd01b4aeb 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -462,6 +462,7 @@ return [ "updated" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], "last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], "last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], + "last_discovery" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last contact discovery"], "archive_date" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], "archived" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], "location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], From 2722c8e595e61341e6a475972b09fff75aaf1ac0 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 7 Mar 2020 11:16:10 +0000 Subject: [PATCH 11/11] Some systems return an array instead of a string for the followers/following --- src/Model/GContact.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index ee30cf3c59..a13b719a9c 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -1326,7 +1326,16 @@ class GContact DBA::update('gfollower', ['deleted' => true], ['gcid' => $gcid]); } - $contacts = array_unique(array_merge($followers, $followings)); + $contacts = []; + foreach (array_merge($followers, $followings) as $contact) { + if (is_string($contact)) { + $contacts[] = $contact; + } elseif (!empty($contact['url']) && is_string($contact['url'])) { + $contacts[] = $contact['url']; + } + } + $contacts = array_unique($contacts); + Logger::info('Discover AP contacts', ['url' => $url, 'contacts' => count($contacts)]); foreach ($contacts as $contact) { $gcontact = DBA::selectFirst('gcontact', ['id'], ['nurl' => Strings::normaliseLink(($contact))]);