Check and add a server

src/Model/GServer.php

@@ -70,6 +70,22 @@ class GServer
 	const DETECT_NODEINFO_1 = 101;
 	const DETECT_NODEINFO_2 = 102;
 
+	/**
+	 * Check for the existance of a server and adds it in the background if not existant
+	 *
+	 * @param string $url
+	 * @param boolean $only_nodeinfo
+	 * @return void
+	 */
+	public static function add(string $url, bool $only_nodeinfo = false)
+	{
+		if (self::getID($url, false)) {
+			return;
+		}
+
+		Worker::add(PRIORITY_LOW, 'UpdateGServer', $url, $only_nodeinfo);
+	}
+
 	/**
 	 * Get the ID for the given server URL
 	 *
@@ -173,61 +189,6 @@ class GServer
 		return DateTimeFormat::utc('now +3 month');
 	}
 
-	/**
-	 * Decides if a server needs to be updated, based upon several date fields
-	 *
-	 * @param date $created      Creation date of that server entry
-	 * @param date $updated      When had the server entry be updated
-	 * @param date $last_failure Last failure when contacting that server
-	 * @param date $last_contact Last time the server had been contacted
-	 *
-	 * @return boolean Does the server record needs an update?
-	 */
-	public static function updateNeeded($created, $updated, $last_failure, $last_contact)
-	{
-		$now = strtotime(DateTimeFormat::utcNow());
-
-		if ($updated > $last_contact) {
-			$contact_time = strtotime($updated);
-		} else {
-			$contact_time = strtotime($last_contact);
-		}
-
-		$failure_time = strtotime($last_failure);
-		$created_time = strtotime($created);
-
-		// If there is no "created" time then use the current time
-		if ($created_time <= 0) {
-			$created_time = $now;
-		}
-
-		// If the last contact was less than 24 hours then don't update
-		if (($now - $contact_time) < (60 * 60 * 24)) {
-			return false;
-		}
-
-		// If the last failure was less than 24 hours then don't update
-		if (($now - $failure_time) < (60 * 60 * 24)) {
-			return false;
-		}
-
-		// If the last contact was less than a week ago and the last failure is older than a week then don't update
-		//if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
-		//	return false;
-
-		// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
-		if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
-			return false;
-		}
-
-		// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
-		if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
-			return false;
-		}
-
-		return true;
-	}
-
 	/**
 	 * Checks the state of the given server.
 	 *
@@ -241,7 +202,6 @@ class GServer
 	public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false)
 	{
 		$server_url = self::cleanURL($server_url);
-
 		if ($server_url == '') {
 			return false;
 		}
@@ -254,24 +214,11 @@ class GServer
 				DBA::update('gserver', $fields, $condition);
 			}
 
-			$last_contact = $gserver['last_contact'];
+			if (!$force && (strtotime($gserver['next_contact']) > time())) {
-			$last_failure = $gserver['last_failure'];
-
-			// See discussion under https://forum.friendi.ca/display/0b6b25a8135aabc37a5a0f5684081633
-			// It can happen that a zero date is in the database, but storing it again is forbidden.
-			if ($last_contact < DBA::NULL_DATETIME) {
-				$last_contact = DBA::NULL_DATETIME;
-			}
-
-			if ($last_failure < DBA::NULL_DATETIME) {
-				$last_failure = DBA::NULL_DATETIME;
-			}
-
-			if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) {
 				Logger::info('No update needed', ['server' => $server_url]);
-				return ($last_contact >= $last_failure);
+				return (!$gserver['failed']);
 			}
-			Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force, 'Created' => $gserver['created'], 'Failure' => $last_failure, 'Contact' => $last_contact]);
+			Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force]);
 		} else {
 			Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]);
 		}
@@ -1690,7 +1637,7 @@ class GServer
 				if (!empty($data['data']['nodes'])) {
 					foreach ($data['data']['nodes'] as $server) {
 						// Using "only_nodeinfo" since servers that are listed on that page should always have it.
-						Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host'], true);
+						self::add('https://' . $server['host'], true);
 					}
 				}
 			}
@@ -1709,7 +1656,7 @@ class GServer
 
 				foreach ($servers['instances'] as $server) {
 					$url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name'];
-					Worker::add(PRIORITY_LOW, 'UpdateGServer', $url);
+					self::add($url);
 				}
 			}
 		}

src/Worker/UpdateGServer.php

@@ -22,6 +22,7 @@
 namespace Friendica\Worker;
 
 use Friendica\Core\Logger;
+use Friendica\Database\DBA;
 use Friendica\Model\GServer;
 use Friendica\Util\Strings;
 
@@ -32,7 +33,7 @@ class UpdateGServer
 	 * @param string  $server_url    Server URL
 	 * @param boolean $only_nodeinfo Only use nodeinfo for server detection
 	 */
-	public static function execute(string $server_url, bool $only_nodeinfo = false, bool $force = false)
+	public static function execute(string $server_url, bool $only_nodeinfo = false)
 	{
 		if (empty($server_url)) {
 			return;
@@ -40,11 +41,22 @@ class UpdateGServer
 
 		$filtered = filter_var($server_url, FILTER_SANITIZE_URL);
 		if (substr(Strings::normaliseLink($filtered), 0, 7) != 'http://') {
-			GServer::setFailure($filtered);
+			GServer::setFailure($server_url);
 			return;
 		}
 
-		$ret = GServer::check($filtered, '', $force, $only_nodeinfo);
+		if (($filtered != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) {
+			GServer::setFailure($server_url);
+			return;
+		}
+
+		$cleaned = GServer::cleanURL($server_url);
+		if (($cleaned != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) {
+			GServer::setFailure($server_url);
+			return;
+		}
+
+		$ret = GServer::check($filtered, '', true, $only_nodeinfo);
 		Logger::info('Updated gserver', ['url' => $filtered, 'result' => $ret]);
 	}
 }

src/Worker/UpdateGServers.php

@@ -24,33 +24,49 @@ namespace Friendica\Worker;
 use Friendica\Core\Logger;
 use Friendica\Core\Worker;
 use Friendica\Database\DBA;
+use Friendica\DI;
+use Friendica\Util\Strings;
 
 class UpdateGServers
 {
 	/**
-	 * Updates up to 100 servers
+	 * Updates a defined number of servers
 	 */
 	public static function execute()
 	{
+		$update_limit = DI::config()->get('system', 'gserver_update_limit');
+		if (empty($update_limit)) {
+			return;
+		}
+
 		$updating = Worker::countWorkersByCommand('UpdateGServer');
-		$limit = 100 - $updating;
+		$limit = $update_limit - $updating;
 		if ($limit <= 0) {
 			Logger::info('The number of currently running jobs exceed the limit');
 			return;
 		}
 
-		$outdated = DBA::count('gserver', ["`next_contact` < UTC_TIMESTAMP()"]);
 		$total = DBA::count('gserver');
+		$condition = ["`next_contact` < UTC_TIMESTAMP() AND (`nurl` != ? OR `url` != ?)", '', ''];
+		$outdated = DBA::count('gserver', $condition);
 		Logger::info('Server status', ['total' => $total, 'outdated' => $outdated, 'updating' => $limit]);
 
-		$gservers = DBA::select('gserver', ['url'], ["`next_contact` < UTC_TIMESTAMP()"], ['limit' => $limit]);
+		$gservers = DBA::select('gserver', ['url', 'nurl'], $condition, ['limit' => $limit]);
 		if (!DBA::isResult($gservers)) {
 			return;
 		}
 
 		$count = 0;
 		while ($gserver = DBA::fetch($gservers)) {
-			Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false, true);
+			// Sometimes the "nurl" and "url" doesn't seem to fit, see https://forum.friendi.ca/display/ec054ce7-155f-c94d-6159-f50372664245
+			// There are duplicated "url" but not "nurl". So we check both addresses instead of just overwriting them,
+			// since that would mean loosing data.
+			if (!empty($gserver['url'])) {
+				Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url']);
+			}
+			if (!empty($gserver['nurl']) && ($gserver['nurl'] != Strings::normaliseLink($gserver['url']))) {
+				Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['nurl']);
+			}
 			$count++;
 		}
 		DBA::close($gservers);

src/Worker/UpdateServerPeers.php

@@ -22,9 +22,9 @@
 namespace Friendica\Worker;
 
 use Friendica\Core\Logger;
-use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\GServer;
 use Friendica\Util\Strings;
 
 class UpdateServerPeers
@@ -58,7 +58,7 @@ class UpdateServerPeers
 				continue;
 			}
 			// This endpoint doesn't offer the schema. So we assume that it is HTTPS.
-			Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $peer);
+			GServer::add('https://' . $peer);
 			++$added;
 		}
 		Logger::info('Server peer update ended', ['total' => $total, 'added' => $added, 'url' => $url]);

static/defaults.config.php

@@ -254,6 +254,10 @@ return [
 		// This can alternatively be set on a per account basis in the pconfig table.
 		'groupedit_image_limit' => 400,
 
+		// gserver_update_limit (Integer)
+		// How much servers should be checked at a time?
+		'gserver_update_limit' => 100,
+
 		// hsts (Boolean)
 		// Enables the sending of HTTP Strict Transport Security headers.
 		'hsts' => false,