Merge pull request #7996 from annando/poco-cleanup

The PoCo-Workers had been split

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2019.12-rc (Dalmatian Bellflower)
--- DB_UPDATE_VERSION 1326
+-- DB_UPDATE_VERSION 1328
 -- ------------------------------------------
 
 
@@ -470,6 +470,7 @@ CREATE TABLE IF NOT EXISTS `gserver` (
 	`info` text COMMENT '',
 	`register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '',
 	`registered-users` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of registered users',
+	`directory-type` tinyint DEFAULT 0 COMMENT 'Type of directory service (Poco, Mastodon)',
 	`poco` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`noscrape` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`network` char(4) NOT NULL DEFAULT '' COMMENT '',

src/Core/Search.php

@@ -242,7 +242,7 @@ class Search extends BaseObject
 		DBA::close($data);
 
 		// Add found profiles from the global directory to the local directory
-		Worker::add(PRIORITY_LOW, 'DiscoverPoCo', "dirsearch", urlencode($search));
+		Worker::add(PRIORITY_LOW, 'SearchDirectory', $search);
 
 		return $resultList;
 	}

src/Model/GContact.php

@@ -580,9 +580,6 @@ class GContact
 	public static function getId($contact)
 	{
 		$gcontact_id = 0;
-		$doprobing = false;
-		$last_failure_str = '';
-		$last_contact_str = '';
 
 		if (empty($contact['network'])) {
 			Logger::notice('Empty network', ['url' => $contact['url'], 'callstack' => System::callstack()]);
@@ -613,15 +610,6 @@ class GContact
 		$gcnt = DBA::selectFirst('gcontact', $fields, ['nurl' => Strings::normaliseLink($contact['url'])]);
 		if (DBA::isResult($gcnt)) {
 			$gcontact_id = $gcnt['id'];
-
-			// Update every 90 days
-			if (empty($gcnt['network']) || in_array($gcnt['network'], Protocol::FEDERATED)) {
-				$last_failure_str = $gcnt['last_failure'];
-				$last_failure = strtotime($gcnt['last_failure']);
-				$last_contact_str = $gcnt['last_contact'];
-				$last_contact = strtotime($gcnt['last_contact']);
-				$doprobing = (((time() - $last_contact) > (90 * 86400)) && ((time() - $last_failure) > (90 * 86400)));
-			}
 		} else {
 			$contact['location'] = $contact['location'] ?? '';
 			$contact['about'] = $contact['about'] ?? '';
@@ -638,16 +626,10 @@ class GContact
 			$cnt = DBA::selectFirst('gcontact', ['id', 'network'], $condition, ['order' => ['id']]);
 			if (DBA::isResult($cnt)) {
 				$gcontact_id = $cnt['id'];
-				$doprobing = (empty($cnt['network']) || in_array($cnt['network'], Protocol::FEDERATED));
 			}
 		}
 		DBA::unlock();
 
-		if ($doprobing) {
-			Logger::notice('Probing', ['contact' => $last_contact_str, "failure" => $last_failure_str, "checking" => $contact['url']]);
-			Worker::add(PRIORITY_LOW, 'GProbe', $contact['url']);
-		}
-
 		return $gcontact_id;
 	}
 
@@ -801,7 +783,7 @@ class GContact
 			return;
 		}
 
-		if (!$force && !PortableContact::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) {
+		if (!$force && !GServer::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) {
 			Logger::info("Don't update profile", ['url' => $data['url'], 'updated' => $gcontact['updated']]);
 			return;
 		}

src/Model/GServer.php

@@ -26,6 +26,10 @@ use Friendica\Network\Probe;
  */
 class GServer
 {
+	// Directory types
+	const DT_NONE = 0;
+	const DT_POCO = 1;
+	const DT_MASTODON = 2;
 	/**
 	 * Checks if the given server is reachable
 	 *
@@ -49,6 +53,61 @@ class GServer
 		return self::check($server, $network, $force);
 	}
 
+	/**
+	 * Decides if a server needs to be updated, based upon several date fields
+	 *
+	 * @param date $created      Creation date of that server entry
+	 * @param date $updated      When had the server entry be updated
+	 * @param date $last_failure Last failure when contacting that server
+	 * @param date $last_contact Last time the server had been contacted
+	 *
+	 * @return boolean Does the server record needs an update?
+	 */
+	public static function updateNeeded($created, $updated, $last_failure, $last_contact)
+	{
+		$now = strtotime(DateTimeFormat::utcNow());
+
+		if ($updated > $last_contact) {
+			$contact_time = strtotime($updated);
+		} else {
+			$contact_time = strtotime($last_contact);
+		}
+
+		$failure_time = strtotime($last_failure);
+		$created_time = strtotime($created);
+
+		// If there is no "created" time then use the current time
+		if ($created_time <= 0) {
+			$created_time = $now;
+		}
+
+		// If the last contact was less than 24 hours then don't update
+		if (($now - $contact_time) < (60 * 60 * 24)) {
+			return false;
+		}
+
+		// If the last failure was less than 24 hours then don't update
+		if (($now - $failure_time) < (60 * 60 * 24)) {
+			return false;
+		}
+
+		// If the last contact was less than a week ago and the last failure is older than a week then don't update
+		//if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
+		//	return false;
+
+		// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
+		if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
+			return false;
+		}
+
+		// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
+		if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
+			return false;
+		}
+
+		return true;
+	}
+
 	/**
 	 * Checks the state of the given server.
 	 *
@@ -89,7 +148,7 @@ class GServer
 				$last_failure = DBA::NULL_DATETIME;
 			}
 
-			if (!$force && !PortableContact::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) {
+			if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) {
 				Logger::info('No update needed', ['server' => $server_url]);
 				return ($last_contact >= $last_failure);
 			}
@@ -112,8 +171,24 @@ class GServer
 	 */
 	public static function detect(string $url, string $network = '')
 	{
+		Logger::info('Detect server type', ['server' => $url]);
 		$serverdata = [];
 
+		$original_url = $url;
+
+		// Remove URL content that is not supposed to exist for a server url
+		$urlparts = parse_url($url);
+		unset($urlparts['user']);
+		unset($urlparts['pass']);
+		unset($urlparts['query']);
+		unset($urlparts['fragment']);
+		$url = Network::unparseURL($urlparts);
+
+		// If the URL missmatches, then we mark the old entry as failure
+		if ($url != $original_url) {
+			DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($original_url)]);
+		}
+
 		// When a nodeinfo is present, we don't need to dig further
 		$xrd_timeout = Config::get('system', 'xrd_timeout');
 		$curlResult = Network::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]);
@@ -183,7 +258,10 @@ class GServer
 			$serverdata = $nodeinfo;
 		}
 
+		// Detect the directory type
+		$serverdata['directory-type'] = self::DT_NONE;
 		$serverdata = self::checkPoCo($url, $serverdata);
+		$serverdata = self::checkMastodonDirectory($url, $serverdata);
 
 		// We can't detect the network type. Possibly it is some system that we don't know yet
 		if (empty($serverdata['network'])) {
@@ -740,6 +818,8 @@ class GServer
 	 */
 	private static function checkPoCo(string $url, array $serverdata)
 	{
+		$serverdata['poco'] = '';
+
 		$curlResult = Network::curl($url. '/poco');
 		if (!$curlResult->isSuccess()) {
 			return $serverdata;
@@ -753,9 +833,35 @@ class GServer
 		if (!empty($data['totalResults'])) {
 			$registeredUsers = $serverdata['registered-users'] ?? 0;
 			$serverdata['registered-users'] = max($data['totalResults'], $registeredUsers);
+			$serverdata['directory-type'] = self::DT_POCO;
 			$serverdata['poco'] = $url . '/poco';
-		} else {
+		}
-			$serverdata['poco'] = '';
+
+		return $serverdata;
+	}
+
+	/**
+	 * Checks if the given server does have a Mastodon style directory endpoint.
+	 *
+	 * @param string $url        URL of the given server
+	 * @param array  $serverdata array with server data
+	 *
+	 * @return array server data
+	 */
+	public static function checkMastodonDirectory(string $url, array $serverdata)
+	{
+		$curlResult = Network::curl($url . '/api/v1/directory?limit=1');
+		if (!$curlResult->isSuccess()) {
+			return $serverdata;
+		}
+
+		$data = json_decode($curlResult->getBody(), true);
+		if (empty($data)) {
+			return $serverdata;
+		}
+
+		if (count($data) == 1) {
+			$serverdata['directory-type'] = self::DT_MASTODON;
 		}
 
 		return $serverdata;
@@ -1184,4 +1290,18 @@ class GServer
 		}
 		return $serverdata;
 	}
+
+	/**
+	 * Update the user directory of a given gserver record
+	 *
+	 * @param array $gserver gserver record
+	 */
+	public static function updateDirectory(array $gserver)
+	{
+		/// @todo Add Mastodon API directory
+
+		if (!empty($gserver['poco'])) {
+			PortableContact::discoverSingleServer($gserver['id']);
+		}
+	}
 }

src/Model/Profile.php

@@ -1074,8 +1074,6 @@ class Profile
 
 		Logger::log('Not authenticated. Invoking reverse magic-auth for ' . $my_url, Logger::DEBUG);
 
-		Worker::add(PRIORITY_LOW, 'GProbe', $my_url);
-
 		// Remove the "addr" parameter from the destination. It is later added as separate parameter again.
 		$addr_request = 'addr=' . urlencode($addr);
 		$query = rtrim(str_replace($addr_request, '', $a->query_string), '?&');

src/Protocol/PortableContact.php

@@ -57,7 +57,7 @@ class PortableContact
 	public static function loadWorker($cid, $uid = 0, $zcid = 0, $url = null)
 	{
 		// Call the function "load" via the worker
-		Worker::add(PRIORITY_LOW, "DiscoverPoCo", "load", (int)$cid, (int)$uid, (int)$zcid, $url);
+		Worker::add(PRIORITY_LOW, 'FetchPoCo', (int)$cid, (int)$uid, (int)$zcid, $url);
 	}
 
 	/**
@@ -219,51 +219,6 @@ class PortableContact
 		return(preg_match("=https?://.+/user/\d+=ism", $url, $matches));
 	}
 
-	public static function updateNeeded($created, $updated, $last_failure, $last_contact)
-	{
-		$now = strtotime(DateTimeFormat::utcNow());
-
-		if ($updated > $last_contact) {
-			$contact_time = strtotime($updated);
-		} else {
-			$contact_time = strtotime($last_contact);
-		}
-
-		$failure_time = strtotime($last_failure);
-		$created_time = strtotime($created);
-
-		// If there is no "created" time then use the current time
-		if ($created_time <= 0) {
-			$created_time = $now;
-		}
-
-		// If the last contact was less than 24 hours then don't update
-		if (($now - $contact_time) < (60 * 60 * 24)) {
-			return false;
-		}
-
-		// If the last failure was less than 24 hours then don't update
-		if (($now - $failure_time) < (60 * 60 * 24)) {
-			return false;
-		}
-
-		// If the last contact was less than a week ago and the last failure is older than a week then don't update
-		//if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
-		//	return false;
-
-		// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
-		if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
-			return false;
-		}
-
-		// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
-		if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
-			return false;
-		}
-
-		return true;
-	}
-
 	/**
 	 * @brief Returns a list of all known servers
 	 * @return array List of server urls
@@ -315,7 +270,7 @@ class PortableContact
 
 			if (!DBA::isResult($r)) {
 				Logger::log("Call server check for server ".$server_url, Logger::DEBUG);
-				Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server_url);
+				Worker::add(PRIORITY_LOW, 'UpdateGServer', $server_url);
 			}
 		}
 	}
@@ -340,7 +295,7 @@ class PortableContact
 
 			if (!empty($servers['pods'])) {
 				foreach ($servers['pods'] as $server) {
-					Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", "https://" . $server['host']);
+					Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host']);
 				}
 			}
 		}
@@ -359,7 +314,7 @@ class PortableContact
 
 					foreach ($servers['instances'] as $server) {
 						$url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name'];
-						Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $url);
+						Worker::add(PRIORITY_LOW, 'UpdateGServer', $url);
 					}
 				}
 			}
@@ -470,7 +425,7 @@ class PortableContact
 
 		$last_update = date('c', time() - (60 * 60 * 24 * $requery_days));
 
-		$gservers = q("SELECT `id`, `url`, `nurl`, `network`
+		$gservers = q("SELECT `id`, `url`, `nurl`, `network`, `poco`
 			FROM `gserver`
 			WHERE `last_contact` >= `last_failure`
 			AND `poco` != ''
@@ -488,7 +443,7 @@ class PortableContact
 				}
 
 				Logger::log('Update directory from server ' . $gserver['url'] . ' with ID ' . $gserver['id'], Logger::DEBUG);
-				Worker::add(PRIORITY_LOW, 'DiscoverPoCo', 'update_server_directory', (int) $gserver['id']);
+				Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', $gserver);
 
 				if (!$complete && ( --$no_of_queries == 0)) {
 					break;

src/Worker/Cron.php

@@ -38,11 +38,11 @@ class Cron
 		// Fork the cron jobs in separate parts to avoid problems when one of them is crashing
 		Hook::fork($a->queue['priority'], "cron");
 
-		// run the process to discover global contacts in the background
+		// run the process to update server directories in the background
-		Worker::add(PRIORITY_LOW, "DiscoverPoCo");
+		Worker::add(PRIORITY_LOW, 'UpdateServerDirectories');
 
 		// run the process to update locally stored global contacts in the background
-		Worker::add(PRIORITY_LOW, "DiscoverPoCo", "checkcontact");
+		Worker::add(PRIORITY_LOW, 'UpdateGContacts');
 
 		// Expire and remove user entries
 		Worker::add(PRIORITY_MEDIUM, "CronJobs", "expire_and_remove_users");
@@ -73,9 +73,9 @@ class Cron
 			// update nodeinfo data
 			Worker::add(PRIORITY_LOW, "CronJobs", "nodeinfo");
 
-			Worker::add(PRIORITY_LOW, "DiscoverPoCo", "update_server");
+			Worker::add(PRIORITY_LOW, 'UpdateGServers');
 
-			Worker::add(PRIORITY_LOW, "DiscoverPoCo", "suggestions");
+			Worker::add(PRIORITY_LOW, 'UpdateSuggestions');
 
 			Worker::add(PRIORITY_LOW, 'Expire');
 

src/Worker/DiscoverPoCo.php

@@ -1,305 +0,0 @@
-<?php
-/**
- * @file src/Worker/DiscoverPoCo.php
- */
-namespace Friendica\Worker;
-
-use Friendica\Core\Cache;
-use Friendica\Core\Config;
-use Friendica\Core\Logger;
-use Friendica\Core\Protocol;
-use Friendica\Core\Worker;
-use Friendica\Database\DBA;
-use Friendica\Model\GContact;
-use Friendica\Model\Contact;
-use Friendica\Model\GServer;
-use Friendica\Network\Probe;
-use Friendica\Protocol\PortableContact;
-use Friendica\Util\DateTimeFormat;
-use Friendica\Util\Network;
-use Friendica\Util\Strings;
-
-class DiscoverPoCo
-{
-	/// @todo Clean up this mess of a parameter hell and split it in several classes
-	public static function execute($command = '', $param1 = '', $param2 = '', $param3 = '', $param4 = '')
-	{
-		/*
-		This function can be called in these ways:
-		- dirsearch <search pattern>: Searches for "search pattern" in the directory. "search pattern" is url encoded.
-		- checkcontact: Updates gcontact entries
-		- suggestions: Discover other servers for their contacts.
-		- server <poco url>: Searches for the poco server list. "poco url" is base64 encoded.
-		- update_server: Frequently check the first 250 servers for vitality.
-		- update_server_directory: Discover the given server id for their contacts
-		- PortableContact::load: Load POCO data from a given POCO address
-		- check_profile: Update remote profile data
-		*/
-
-		$search = "";
-		$mode = 0;
-		if ($command == "dirsearch") {
-			$search = urldecode($param1);
-			$mode = 1;
-		} elseif ($command == "checkcontact") {
-			$mode = 2;
-		} elseif ($command == "suggestions") {
-			$mode = 3;
-		} elseif ($command == "server") {
-			$mode = 4;
-		} elseif ($command == "update_server") {
-			$mode = 5;
-		} elseif ($command == "update_server_directory") {
-			$mode = 6;
-		} elseif ($command == "load") {
-			$mode = 7;
-		} elseif ($command == "check_profile") {
-			$mode = 8;
-		} elseif ($command !== "") {
-			Logger::log("Unknown or missing parameter ".$command."\n");
-			return;
-		}
-
-		Logger::log('start '.$search);
-
-		if ($mode == 8) {
-			if ($param1 != "") {
-				GContact::updateFromProbe($param1, true);
-			}
-		} elseif ($mode == 7) {
-			if (!empty($param4)) {
-				$url = $param4;
-			} else {
-				$url = '';
-			}
-			PortableContact::load(intval($param1), intval($param2), intval($param3), $url);
-		} elseif ($mode == 6) {
-			PortableContact::discoverSingleServer(intval($param1));
-		} elseif ($mode == 5) {
-			self::updateServer();
-		} elseif ($mode == 4) {
-			$server_url = $param1;
-			if ($server_url == "") {
-				return;
-			}
-			$server_url = filter_var($server_url, FILTER_SANITIZE_URL);
-			if (substr(Strings::normaliseLink($server_url), 0, 7) != "http://") {
-				return;
-			}
-			$result = "Checking server ".$server_url." - ";
-			$ret = GServer::check($server_url);
-			if ($ret) {
-				$result .= "success";
-			} else {
-				$result .= "failed";
-			}
-			Logger::log($result, Logger::DEBUG);
-		} elseif ($mode == 3) {
-			GContact::updateSuggestions();
-		} elseif (($mode == 2) && Config::get('system', 'poco_completion')) {
-			self::discoverUsers();
-		} elseif (($mode == 1) && ($search != "") && Config::get('system', 'poco_local_search')) {
-			self::discoverDirectory($search);
-			self::gsSearchUser($search);
-		} elseif (($mode == 0) && ($search == "") && (Config::get('system', 'poco_discovery') != PortableContact::DISABLED)) {
-			// Query Friendica and Hubzilla servers for their users
-			PortableContact::discover();
-
-			// Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server)
-			if (!Config::get('system', 'ostatus_disabled')) {
-				GContact::discoverGsUsers();
-			}
-		}
-
-		Logger::log('end '.$search);
-
-		return;
-	}
-
-	/**
-	 * @brief Updates the first 250 servers
-	 *
-	 */
-	private static function updateServer() {
-		$r = q("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()");
-
-		if (!DBA::isResult($r)) {
-			return;
-		}
-
-		$updated = 0;
-
-		foreach ($r AS $server) {
-			if (!PortableContact::updateNeeded($server["created"], "", $server["last_failure"], $server["last_contact"])) {
-				continue;
-			}
-			Logger::log('Update server status for server '.$server["url"], Logger::DEBUG);
-
-			Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server["url"]);
-
-			if (++$updated > 250) {
-				return;
-			}
-		}
-	}
-
-	private static function discoverUsers() {
-		Logger::log("Discover users", Logger::DEBUG);
-
-		$starttime = time();
-
-		$users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact`
-				WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
-					`last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
-					`network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()",
-				DBA::escape(Protocol::DFRN), DBA::escape(Protocol::DIASPORA),
-				DBA::escape(Protocol::OSTATUS), DBA::escape(Protocol::FEED));
-
-		if (!$users) {
-			return;
-		}
-		$checked = 0;
-
-		foreach ($users AS $user) {
-
-			$urlparts = parse_url($user["url"]);
-			if (!isset($urlparts["scheme"])) {
-				DBA::update('gcontact', ['network' => Protocol::PHANTOM],
-					['nurl' => Strings::normaliseLink($user["url"])]);
-				continue;
-			 }
-
-			if (in_array($urlparts["host"], ["twitter.com", "identi.ca"])) {
-				$networks = ["twitter.com" => Protocol::TWITTER, "identi.ca" => Protocol::PUMPIO];
-
-				DBA::update('gcontact', ['network' => $networks[$urlparts["host"]]],
-					['nurl' => Strings::normaliseLink($user["url"])]);
-				continue;
-			}
-
-			$server_url = Contact::getBasepath($user["url"]);
-			$force_update = false;
-
-			if ($user["server_url"] != "") {
-
-				$force_update = (Strings::normaliseLink($user["server_url"]) != Strings::normaliseLink($server_url));
-
-				$server_url = $user["server_url"];
-			}
-
-			if ((($server_url == "") && ($user["network"] == Protocol::FEED)) || $force_update || GServer::check($server_url, $user["network"])) {
-				Logger::log('Check profile '.$user["url"]);
-				Worker::add(PRIORITY_LOW, "DiscoverPoCo", "check_profile", $user["url"]);
-
-				if (++$checked > 100) {
-					return;
-				}
-			} else {
-				DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()],
-					['nurl' => Strings::normaliseLink($user["url"])]);
-			}
-
-			// Quit the loop after 3 minutes
-			if (time() > ($starttime + 180)) {
-				return;
-			}
-		}
-	}
-
-	private static function discoverDirectory($search) {
-
-		$data = Cache::get("dirsearch:".$search);
-		if (!is_null($data)) {
-			// Only search for the same item every 24 hours
-			if (time() < $data + (60 * 60 * 24)) {
-				Logger::log("Already searched for ".$search." in the last 24 hours", Logger::DEBUG);
-				return;
-			}
-		}
-
-		$x = Network::fetchUrl(get_server()."/lsearch?p=1&n=500&search=".urlencode($search));
-		$j = json_decode($x);
-
-		if (!empty($j->results)) {
-			foreach ($j->results as $jj) {
-				// Check if the contact already exists
-				$exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", Strings::normaliseLink($jj->url));
-				if (DBA::isResult($exists)) {
-					Logger::log("Profile ".$jj->url." already exists (".$search.")", Logger::DEBUG);
-
-					if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) &&
-						($exists[0]["updated"] < $exists[0]["last_failure"])) {
-						continue;
-					}
-					// Update the contact
-					GContact::updateFromProbe($jj->url);
-					continue;
-				}
-
-				$server_url = Contact::getBasepath($jj->url);
-				if ($server_url != '') {
-					if (!GServer::check($server_url)) {
-						Logger::log("Friendica server ".$server_url." doesn't answer.", Logger::DEBUG);
-						continue;
-					}
-					Logger::log("Friendica server ".$server_url." seems to be okay.", Logger::DEBUG);
-				}
-
-				$data = Probe::uri($jj->url);
-				if ($data["network"] == Protocol::DFRN) {
-					Logger::log("Profile ".$jj->url." is reachable (".$search.")", Logger::DEBUG);
-					Logger::log("Add profile ".$jj->url." to local directory (".$search.")", Logger::DEBUG);
-
-					if ($jj->tags != "") {
-						$data["keywords"] = $jj->tags;
-					}
-
-					$data["server_url"] = $data["baseurl"];
-
-					GContact::update($data);
-				} else {
-					Logger::log("Profile ".$jj->url." is not responding or no Friendica contact - but network ".$data["network"], Logger::DEBUG);
-				}
-			}
-		}
-		Cache::set("dirsearch:".$search, time(), Cache::DAY);
-	}
-
-	/**
-	 * @brief Search for GNU Social user with gstools.org
-	 *
-	 * @param string $search User name
-	 * @return bool
-	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
-	 * @throws \ImagickException
-	 */
-	private static function gsSearchUser($search) {
-
-		// Currently disabled, since the service isn't available anymore.
-		// It is not removed since I hope that there will be a successor.
-		return false;
-
-		$url = "http://gstools.org/api/users_search/".urlencode($search);
-
-		$curlResult = Network::curl($url);
-		if (!$curlResult->isSuccess()) {
-			return false;
-		}
-
-		$contacts = json_decode($curlResult->getBody());
-
-		if ($contacts->status == 'ERROR') {
-			return false;
-		}
-
-		/// @TODO AS is considered as a notation for constants (as they usually being written all upper-case)
-		/// @TODO find all those and convert to all lower-case which is a keyword then
-		foreach ($contacts->data AS $user) {
-			$contact = Probe::uri($user->site_address."/".$user->name);
-			if ($contact["network"] != Protocol::PHANTOM) {
-				$contact["about"] = $user->description;
-				GContact::update($contact);
-			}
-		}
-	}
-}

src/Worker/FetchPoCo.php

@@ -0,0 +1,24 @@
+<?php
+/**
+ * @file src/Worker/FetchPoCo.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Protocol\PortableContact;
+
+class FetchPoCo
+{
+	/**
+	 * Fetch PortableContacts from a given PoCo server address
+	 *
+	 * @param integer $cid  Contact ID
+	 * @param integer $uid  User ID
+	 * @param integer $zcid Global Contact ID
+	 * @param integer $url  PoCo address that should be polled
+	 */
+	public static function execute($cid, $uid, $zcid, $url)
+	{
+		PortableContact::load($cid, $uid, $zcid, $url);
+	}
+}

src/Worker/GProbe.php

@@ -1,68 +0,0 @@
-<?php
-/**
- * @file src/Worker/GProbe.php
- */
-
-namespace Friendica\Worker;
-
-use Friendica\Core\Cache;
-use Friendica\Core\Logger;
-use Friendica\Core\Protocol;
-use Friendica\Database\DBA;
-use Friendica\Model\GContact;
-use Friendica\Network\Probe;
-use Friendica\Protocol\PortableContact;
-use Friendica\Util\Strings;
-
-class GProbe {
-	public static function execute($url = '')
-	{
-		if (empty($url)) {
-			return;
-		}
-
-		$r = q(
-			"SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1",
-			DBA::escape(Strings::normaliseLink($url))
-		);
-
-		Logger::log("gprobe start for ".Strings::normaliseLink($url), Logger::DEBUG);
-
-		if (!DBA::isResult($r)) {
-			// Is it a DDoS attempt?
-			$urlparts = parse_url($url);
-
-			$result = Cache::get("gprobe:".$urlparts["host"]);
-			if (!is_null($result)) {
-				if (in_array($result["network"], [Protocol::FEED, Protocol::PHANTOM])) {
-					Logger::debug("DDoS attempt detected for " . $urlparts["host"] . " by " . ($_SERVER["REMOTE_ADDR"] ?? ''), ['$_SERVER' => $_SERVER]);
-					return;
-				}
-			}
-
-			$arr = Probe::uri($url);
-
-			if (is_null($result)) {
-				Cache::set("gprobe:".$urlparts["host"], $arr);
-			}
-
-			if (!in_array($arr["network"], [Protocol::FEED, Protocol::PHANTOM])) {
-				GContact::update($arr);
-			}
-
-			$r = q(
-				"SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1",
-				DBA::escape(Strings::normaliseLink($url))
-			);
-		}
-		if (DBA::isResult($r)) {
-			// Check for accessibility and do a poco discovery
-			if (GContact::updateFromProbe($r[0]['url'], true) && ($r[0]["network"] == Protocol::DFRN)) {
-				PortableContact::loadWorker(0, 0, $r[0]['id'], str_replace('/profile/', '/poco/', $r[0]['url']));
-			}
-		}
-
-		Logger::log("gprobe end for ".Strings::normaliseLink($url), Logger::DEBUG);
-		return;
-	}
-}

src/Worker/SearchDirectory.php

@@ -0,0 +1,85 @@
+<?php
+/**
+ * @file src/Worker/SearchDirectory.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Cache;
+use Friendica\Core\Config;
+use Friendica\Core\Logger;
+use Friendica\Core\Protocol;
+use Friendica\Database\DBA;
+use Friendica\Model\GContact;
+use Friendica\Model\Contact;
+use Friendica\Model\GServer;
+use Friendica\Network\Probe;
+use Friendica\Util\Network;
+use Friendica\Util\Strings;
+
+class SearchDirectory
+{
+	// <search pattern>: Searches for "search pattern" in the directory.
+	public static function execute($search)
+	{
+		if (!Config::get('system', 'poco_local_search')) {
+			Logger::info('Local search is not enabled');
+			return;
+		}
+
+		$data = Cache::get('SearchDirectory:' . $search);
+		if (!is_null($data)) {
+			// Only search for the same item every 24 hours
+			if (time() < $data + (60 * 60 * 24)) {
+				Logger::info('Already searched this in the last 24 hours', ['search' => $search]);
+				return;
+			}
+		}
+
+		$x = Network::fetchUrl(get_server() . '/lsearch?p=1&n=500&search=' . urlencode($search));
+		$j = json_decode($x);
+
+		if (!empty($j->results)) {
+			foreach ($j->results as $jj) {
+				// Check if the contact already exists
+				$gcontact = DBA::selectFirst('gcontact', ['id', 'last_contact', 'last_failure', 'updated'], ['nurl' => Strings::normaliseLink($jj->url)]);
+				if (DBA::isResult($gcontact)) {
+					Logger::info('Profile already exists', ['profile' => $jj->url, 'search' => $search]);
+
+					if (($gcontact['last_contact'] < $gcontact['last_failure']) &&
+						($gcontact['updated'] < $gcontact['last_failure'])) {
+						continue;
+					}
+
+					// Update the contact
+					GContact::updateFromProbe($jj->url);
+					continue;
+				}
+
+				$server_url = Contact::getBasepath($jj->url);
+				if ($server_url != '') {
+					if (!GServer::check($server_url)) {
+						Logger::info("Friendica server doesn't answer.", ['server' => $server_url]);
+						continue;
+					}
+					Logger::info('Friendica server seems to be okay.', ['server' => $server_url]);
+				}
+
+				$data = Probe::uri($jj->url);
+				if ($data['network'] == Protocol::DFRN) {
+					Logger::info('Add profile to local directory', ['profile' => $jj->url]);
+
+					if ($jj->tags != '') {
+						$data['keywords'] = $jj->tags;
+					}
+
+					$data['server_url'] = $data['baseurl'];
+
+					GContact::update($data);
+				} else {
+					Logger::info('Profile is not responding or no Friendica contact', ['profile' => $jj->url, 'network' => $data['network']]);
+				}
+			}
+		}
+		Cache::set('SearchDirectory:' . $search, time(), Cache::DAY);
+	}
+}

src/Worker/UpdateContact.php

@@ -13,6 +13,11 @@ use Friendica\Database\DBA;
 
 class UpdateContact
 {
+	/**
+	 * Update contact data via probe
+	 * @param int    $contact_id Contact ID
+	 * @param string $command
+	 */
 	public static function execute($contact_id, $command = '')
 	{
 		$force = ($command == "force");

src/Worker/UpdateGContact.php

@@ -12,6 +12,11 @@ use Friendica\Database\DBA;
 
 class UpdateGContact
 {
+	/**
+	 * Update global contact via probe
+	 * @param string $url     Global contact url
+	 * @param string $command
+	 */
 	public static function execute($url, $command = '')
 	{
 		$force = ($command == "force");

src/Worker/UpdateGContacts.php

@@ -0,0 +1,83 @@
+<?php
+/**
+ * @file src/Worker/UpdateGContacts.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Config;
+use Friendica\Core\Logger;
+use Friendica\Core\Protocol;
+use Friendica\Core\Worker;
+use Friendica\Database\DBA;
+use Friendica\Model\Contact;
+use Friendica\Model\GServer;
+use Friendica\Util\DateTimeFormat;
+use Friendica\Util\Strings;
+
+class UpdateGContacts
+{
+	/**
+	 * Updates global contacts
+	 */
+	public static function execute()
+	{
+		if (!Config::get('system', 'poco_completion')) {
+			return;
+		}
+
+		Logger::info('Update global contacts');
+
+		$starttime = time();
+
+		$contacts = DBA::p("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact`
+				WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
+					`last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
+					`network` IN (?, ?, ?, ?, ?, '') ORDER BY rand()",
+				Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED);
+
+		$checked = 0;
+
+		while ($contact = DBA::fetch($contacts)) {
+			$urlparts = parse_url($contact['url']);
+			if (empty($urlparts['scheme'])) {
+				DBA::update('gcontact', ['network' => Protocol::PHANTOM],
+					['nurl' => Strings::normaliseLink($contact['url'])]);
+				continue;
+			 }
+
+			if (in_array($urlparts['host'], ['twitter.com', 'identi.ca'])) {
+				$networks = ['twitter.com' => Protocol::TWITTER, 'identi.ca' => Protocol::PUMPIO];
+
+				DBA::update('gcontact', ['network' => $networks[$urlparts['host']]],
+					['nurl' => Strings::normaliseLink($contact['url'])]);
+				continue;
+			}
+
+			$server_url = Contact::getBasepath($contact['url']);
+			$force_update = false;
+
+			if (!empty($contact['server_url'])) {
+				$force_update = (Strings::normaliseLink($contact['server_url']) != Strings::normaliseLink($server_url));
+
+				$server_url = $contact['server_url'];
+			}
+
+			if ((empty($server_url) && ($contact['network'] == Protocol::FEED)) || $force_update || GServer::check($server_url, $contact['network'])) {
+				Logger::info('Check profile', ['profile' => $contact['url']]);
+				Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact['url'], 'force');
+
+				if (++$checked > 100) {
+					return;
+				}
+			} else {
+				DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()],
+					['nurl' => Strings::normaliseLink($contact['url'])]);
+			}
+
+			// Quit the loop after 3 minutes
+			if (time() > ($starttime + 180)) {
+				return;
+			}
+		}
+	}
+}

src/Worker/UpdateGServer.php

@@ -0,0 +1,31 @@
+<?php
+/**
+ * @file src/Worker/UpdateGServer.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Model\GServer;
+use Friendica\Util\Strings;
+
+class UpdateGServer
+{
+	/**
+	 * Update the given server
+	 * @param string $server_url Server URL
+	 */
+	public static function execute($server_url)
+	{
+		if (empty($server_url)) {
+			return;
+		}
+
+		$server_url = filter_var($server_url, FILTER_SANITIZE_URL);
+		if (substr(Strings::normaliseLink($server_url), 0, 7) != 'http://') {
+			return;
+		}
+
+		$ret = GServer::check($server_url);
+		Logger::info('Updated gserver', ['url' => $server_url, 'result' => $ret]);
+	}
+}

src/Worker/UpdateGServers.php

@@ -0,0 +1,39 @@
+<?php
+/**
+ * @file src/Worker/UpdateGServers.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Core\Worker;
+use Friendica\Database\DBA;
+use Friendica\Model\GServer;
+
+class UpdateGServers
+{
+	/**
+	 * Updates the first 250 servers
+	 */
+	public static function execute()
+	{
+		$gservers = DBA::p("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()");
+		if (!DBA::isResult($gservers)) {
+			return;
+		}
+
+		$updated = 0;
+
+		while ($gserver = DBA::fetch($gservers)) {
+			if (!GServer::updateNeeded($gserver['created'], '', $gserver['last_failure'], $gserver['last_contact'])) {
+				continue;
+			}
+			Logger::info('Update server status', ['server' => $gserver['url']]);
+
+			Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url']);
+
+			if (++$updated > 250) {
+				return;
+			}
+		}
+	}
+}

src/Worker/UpdateServerDirectories.php

@@ -0,0 +1,31 @@
+<?php
+/**
+ * @file src/Worker/UpdateServerDirectories.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Config;
+use Friendica\Core\Logger;
+use Friendica\Model\GContact;
+use Friendica\Protocol\PortableContact;
+
+class UpdateServerDirectories
+{
+	/**
+	 * Query global servers for their users
+	 */
+	public static function execute()
+	{
+		if (Config::get('system', 'poco_discovery') == PortableContact::DISABLED) {
+			return;
+		}
+
+		// Query Friendica and Hubzilla servers for their users
+		PortableContact::discover();
+
+		// Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server)
+		if (!Config::get('system', 'ostatus_disabled')) {
+			GContact::discoverGsUsers();
+		}
+	}
+}

src/Worker/UpdateServerDirectory.php

@@ -0,0 +1,21 @@
+<?php
+/**
+ * @file src/Worker/UpdateServerDirectory.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Model\GServer;
+
+class UpdateServerDirectory
+{
+	/**
+	 * Query the given server for their users
+	 * @param string $gserver Server URL
+	 */
+	public static function execute($gserver)
+	{
+		GServer::updateDirectory($gserver);
+		return;
+	}
+}

src/Worker/UpdateSuggestions.php

@@ -0,0 +1,19 @@
+<?php
+/**
+ * @file src/Worker/UpdateSuggestions.php
+ */
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Model\GContact;
+
+class UpdateSuggestions
+{
+	/**
+	 * Discover other servers for their contacts.
+	 */
+	public static function execute()
+	{
+		GContact::updateSuggestions();
+	}
+}

static/dbstructure.config.php

@@ -34,7 +34,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1327);
+	define('DB_UPDATE_VERSION', 1328);
 }
 
 return [
@@ -526,6 +526,7 @@ return [
 			"info" => ["type" => "text", "comment" => ""],
 			"register_policy" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => ""],
 			"registered-users" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Number of registered users"],
+			"directory-type" => ["type" => "tinyint", "default" => "0", "comment" => "Type of directory service (Poco, Mastodon)"],
 			"poco" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
 			"noscrape" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
 			"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],