Merge pull request #12688 from annando/blocked-server

News "blocked" field on the gserver table

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2023.03-dev (Giant Rhubarb)
--- DB_UPDATE_VERSION 1510
+-- DB_UPDATE_VERSION 1511
 -- ------------------------------------------
 
 
@@ -34,6 +34,7 @@ CREATE TABLE IF NOT EXISTS `gserver` (
 	`last_poco_query` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
 	`last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Last successful connection request',
 	`last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Last failed connection request',
+	`blocked` boolean COMMENT 'Server is blocked',
 	`failed` boolean COMMENT 'Connection failed',
 	`next_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Next connection request',
 	 PRIMARY KEY(`id`),

doc/database/db_gserver.md

@@ -34,6 +34,7 @@ Fields
 | last_poco_query       |                                                    | datetime         | YES  |     | 0001-01-01 00:00:00 |                |
 | last_contact          | Last successful connection request                 | datetime         | YES  |     | 0001-01-01 00:00:00 |                |
 | last_failure          | Last failed connection request                     | datetime         | YES  |     | 0001-01-01 00:00:00 |                |
+| blocked               | Server is blocked                                  | boolean          | YES  |     | NULL                |                |
 | failed                | Connection failed                                  | boolean          | YES  |     | NULL                |                |
 | next_contact          | Next connection request                            | datetime         | YES  |     | 0001-01-01 00:00:00 |                |
 

src/Console/ServerBlock.php

@@ -24,6 +24,7 @@ namespace Friendica\Console;
 use Asika\SimpleConsole\CommandArgsException;
 use Asika\SimpleConsole\Console;
 use Console_Table;
+use Friendica\Core\Worker;
 use Friendica\Moderation\DomainPatternBlocklist;
 
 /**
@@ -127,6 +128,7 @@ HELP;
 
 		if ($this->blocklist->append($newBlockList)) {
 			$this->out(sprintf("Entries from %s that were not blocked before are now blocked", $filename));
+			Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
 			return 0;
 		} else {
 			$this->out("Couldn't save the block list");
@@ -169,6 +171,7 @@ HELP;
 			} else {
 				$this->out(sprintf("The domain pattern '%s' is now blocked. (Reason: '%s')", $pattern, $reason));
 			}
+			Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
 			return 0;
 		} else {
 			$this->out(sprintf("Couldn't save '%s' as blocked domain pattern", $pattern));
@@ -193,6 +196,7 @@ HELP;
 		if ($result) {
 			if ($result == 2) {
 				$this->out(sprintf("The domain pattern '%s' isn't blocked anymore", $pattern));
+				Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
 				return 0;
 			} else {
 				$this->out(sprintf("The domain pattern '%s' wasn't blocked.", $pattern));

src/Model/GServer.php

@@ -127,6 +127,13 @@ class GServer
 		$gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => Strings::normaliseLink($url)]);
 		if (DBA::isResult($gserver)) {
 			Logger::debug('Got ID for URL', ['id' => $gserver['id'], 'url' => $url, 'callstack' => System::callstack(20)]);
+
+			if (Network::isUrlBlocked($url)) {
+				self::setBlockedById($gserver['id']);
+			} else {
+				self::setUnblockedById($gserver['id']);
+			}
+
 			return $gserver['id'];
 		}
 
@@ -341,6 +348,12 @@ class GServer
 			return false;
 		}
 
+		if (Network::isUrlBlocked($server_url)) {
+			Logger::info('Server is blocked', ['url' => $server_url]);
+			self::setBlockedByUrl($server_url);
+			return false;
+		}
+
 		$gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($server_url)]);
 		if (DBA::isResult($gserver)) {
 			if ($gserver['created'] <= DBA::NULL_DATETIME) {
@@ -370,8 +383,13 @@ class GServer
 	public static function setReachableById(int $gsid, string $network)
 	{
 		$gserver = DBA::selectFirst('gserver', ['url', 'failed', 'next_contact', 'network'], ['id' => $gsid]);
-		if (DBA::isResult($gserver) && $gserver['failed']) {
+		if (!DBA::isResult($gserver)) {
-			$fields = ['failed' => false, 'last_contact' => DateTimeFormat::utcNow()];
+			return;
+		}
+
+		$blocked = Network::isUrlBlocked($gserver['url']);
+		if ($gserver['failed']) {
+			$fields = ['failed' => false, 'blocked' => $blocked, 'last_contact' => DateTimeFormat::utcNow()];
 			if (!empty($network) && !in_array($gserver['network'], Protocol::FEDERATED)) {
 				$fields['network'] = $network;
 			}
@@ -381,6 +399,10 @@ class GServer
 			if (strtotime($gserver['next_contact']) < time()) {
 				UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
 			}
+		} elseif ($blocked) {
+			self::setBlockedById($gsid);
+		} else {
+			self::setUnblockedById($gsid);
 		}
 	}
 
@@ -393,7 +415,7 @@ class GServer
 	{
 		$gserver = DBA::selectFirst('gserver', ['url', 'failed', 'next_contact'], ['id' => $gsid]);
 		if (DBA::isResult($gserver) && !$gserver['failed']) {
-			self::update(['failed' => true, 'last_failure' => DateTimeFormat::utcNow()], ['id' => $gsid]);
+			self::update(['failed' => true, 'blocked' => Network::isUrlBlocked($gserver['url']), 'last_failure' => DateTimeFormat::utcNow()], ['id' => $gsid]);
 			Logger::info('Set failed status for server', ['url' => $gserver['url']]);
 
 			if (strtotime($gserver['next_contact']) < time()) {
@@ -402,6 +424,33 @@ class GServer
 		}
 	}
 
+	public static function setUnblockedById(int $gsid)
+	{
+		$gserver = DBA::selectFirst('gserver', ['url'], ["(`blocked` OR `blocked` IS NULL) AND `id` = ?", $gsid]);
+		if (DBA::isResult($gserver)) {
+			self::update(['blocked' => false], ['id' => $gsid]);
+			Logger::info('Set unblocked status for server', ['url' => $gserver['url']]);
+		}
+	}
+
+	public static function setBlockedById(int $gsid)
+	{
+		$gserver = DBA::selectFirst('gserver', ['url'], ["(NOT `blocked` OR `blocked` IS NULL) AND `id` = ?", $gsid]);
+		if (DBA::isResult($gserver)) {
+			self::update(['blocked' => true, 'failed' => true], ['id' => $gsid]);
+			Logger::info('Set blocked status for server', ['url' => $gserver['url']]);
+		}
+	}
+
+	public static function setBlockedByUrl(string $url)
+	{
+		$gserver = DBA::selectFirst('gserver', ['url', 'id'], ["(NOT `blocked` OR `blocked` IS NULL) AND `nurl` = ?", Strings::normaliseLink($url)]);
+		if (DBA::isResult($gserver)) {
+			self::update(['blocked' => true, 'failed' => true], ['id' => $gserver['id']]);
+			Logger::info('Set blocked status for server', ['url' => $gserver['url']]);
+		}
+	}
+
 	/**
 	 * Set failed server status
 	 *
@@ -412,7 +461,7 @@ class GServer
 		$gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($url)]);
 		if (DBA::isResult($gserver)) {
 			$next_update = self::getNextUpdateDate(false, $gserver['created'], $gserver['last_contact']);
-			self::update(['url' => $url, 'failed' => true, 'last_failure' => DateTimeFormat::utcNow(),
+			self::update(['url' => $url, 'failed' => true, 'blocked' => Network::isUrlBlocked($url), 'last_failure' => DateTimeFormat::utcNow(),
 			'next_contact' => $next_update, 'network' => Protocol::PHANTOM, 'detection-method' => null],
 			['nurl' => Strings::normaliseLink($url)]);
 			Logger::info('Set failed status for existing server', ['url' => $url]);
@@ -491,7 +540,7 @@ class GServer
 	 *
 	 * @return boolean 'true' if server could be detected
 	 */
-	public static function detect(string $url, string $network = '', bool $only_nodeinfo = false): bool
+	private static function detect(string $url, string $network = '', bool $only_nodeinfo = false): bool
 	{
 		Logger::info('Detect server type', ['server' => $url]);
 
@@ -732,9 +781,9 @@ class GServer
 		}
 
 		$serverdata['next_contact'] = self::getNextUpdateDate(true, '', '', in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]));
-
 		$serverdata['last_contact'] = DateTimeFormat::utcNow();
 		$serverdata['failed']       = false;
+		$serverdata['blocked']      = false;
 
 		$gserver = DBA::selectFirst('gserver', ['network'], ['nurl' => Strings::normaliseLink($url)]);
 		if (!DBA::isResult($gserver)) {
@@ -2259,7 +2308,7 @@ class GServer
 		$last_update = date('c', time() - (60 * 60 * 24 * $requery_days));
 
 		$gservers = DBA::select('gserver', ['id', 'url', 'nurl', 'network', 'poco', 'directory-type'],
-			["NOT `failed` AND `directory-type` != ? AND `last_poco_query` < ?", GServer::DT_NONE, $last_update],
+			["NOT `blocked` AND NOT `failed` AND `directory-type` != ? AND `last_poco_query` < ?", GServer::DT_NONE, $last_update],
 			['order' => ['RAND()']]);
 
 		while ($gserver = DBA::fetch($gservers)) {

src/Module/Api/Mastodon/Instance/Peers.php

@@ -42,7 +42,7 @@ class Peers extends BaseApi
 		$return = [];
 
 		// We only select for Friendica and ActivityPub servers, since it is expected to only deliver AP compatible systems here.
-		$instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND NOT `failed` AND NOT `detection-method` IN (?, ?, ?, ?)",
+		$instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND NOT `blocked` AND NOT `failed` AND NOT `detection-method` IN (?, ?, ?, ?)",
 			Protocol::DFRN, Protocol::ACTIVITYPUB,
 			GServer::DETECT_MANUAL, GServer::DETECT_HEADER, GServer::DETECT_BODY, GServer::DETECT_HOST_META]);
 		while ($instance = DBA::fetch($instances)) {

src/Module/Moderation/Blocklist/Server/Add.php

@@ -75,6 +75,8 @@ class Add extends BaseModeration
 		//  Add new item to blocklist
 		$this->blocklist->addPattern($pattern, trim($request['reason']));
 
+		Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
+
 		$this->systemMessages->addInfo($this->t('Server domain pattern added to the blocklist.'));
 
 		if (!empty($request['purge'])) {

src/Module/Moderation/Blocklist/Server/Import.php

@@ -25,6 +25,7 @@ use Friendica\App;
 use Friendica\Core\L10n;
 use Friendica\Core\Renderer;
 use Friendica\Core\Session\Capability\IHandleUserSessions;
+use Friendica\Core\Worker;
 use Friendica\Moderation\DomainPatternBlocklist;
 use Friendica\Module\Response;
 use Friendica\Navigation\SystemMessages;
@@ -95,6 +96,8 @@ class Import extends \Friendica\Module\BaseModeration
 				}
 			}
 
+			Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
+
 			$this->baseUrl->redirect('/moderation/blocklist/server');
 		}
 	}

src/Module/Moderation/Blocklist/Server/Index.php

@@ -25,6 +25,7 @@ use Friendica\App;
 use Friendica\Core\L10n;
 use Friendica\Core\Renderer;
 use Friendica\Core\Session\Capability\IHandleUserSessions;
+use Friendica\Core\Worker;
 use Friendica\Moderation\DomainPatternBlocklist;
 use Friendica\Module\BaseModeration;
 use Friendica\Module\Response;
@@ -70,6 +71,8 @@ class Index extends BaseModeration
 
 		$this->blocklist->set($blocklist);
 
+		Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
+
 		$this->baseUrl->redirect('moderation/blocklist/server');
 	}
 

src/Worker/Cron.php

@@ -85,8 +85,6 @@ class Cron
 
 		// Hourly cron calls
 		if ((DI::keyValue()->get('last_cron_hourly') ?? 0) + 3600 < time()) {
-
-
 			// Update trending tags cache for the community page
 			Tag::setLocalTrendingHashtags(24, 20);
 			Tag::setGlobalTrendingHashtags(24, 20);
@@ -145,6 +143,9 @@ class Cron
 			// Resubscribe to relay servers
 			Relay::reSubscribe();
 
+			// Update "blocked" status of servers
+			Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
+
 			DI::keyValue()->set('last_cron_daily', time());
 		}
 

src/Worker/UpdateBlockedServers.php

@@ -0,0 +1,53 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2023, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Database\DBA;
+use Friendica\Model\GServer;
+use Friendica\Util\Network;
+
+class UpdateBlockedServers
+{
+	/**
+	 * Updates the server blocked status
+	 */
+	public static function execute()
+	{
+		Logger::debug('Update blocked servers - start');
+		$gservers = DBA::select('gserver', ['id', 'url', 'blocked']);
+		while ($gserver = DBA::fetch($gservers)) {
+			$blocked = Network::isUrlBlocked($gserver['url']);
+			if (!is_null($gserver['blocked']) && ($blocked == $gserver['blocked'])) {
+				continue;
+			}
+
+			if ($blocked) {
+				GServer::setBlockedById($gserver['id']);
+			} else {
+				GServer::setUnblockedById($gserver['id']);
+			}
+		}
+		DBA::close($gservers);
+		Logger::debug('Update blocked servers - done');
+	}
+}

src/Worker/UpdateGServer.php

@@ -55,6 +55,7 @@ class UpdateGServer
 
 		// Silently dropping the worker task if the server domain is blocked
 		if (Network::isUrlBlocked($filtered)) {
+			GServer::setBlockedByUrl($filtered);
 			return;
 		}
 
@@ -84,6 +85,7 @@ class UpdateGServer
 	{
 		// Dropping the worker task if the server domain is blocked
 		if (Network::isUrlBlocked($serverUrl)) {
+			GServer::setBlockedByUrl($serverUrl);
 			return 0;
 		}
 

src/Worker/UpdateGServers.php

@@ -27,7 +27,6 @@ use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Strings;
-use GuzzleHttp\Psr7\Uri;
 
 class UpdateGServers
 {
@@ -49,7 +48,7 @@ class UpdateGServers
 		}
 
 		$total = DBA::count('gserver');
-		$condition = ["`next_contact` < ? AND (`nurl` != ? OR `url` != ?)",  DateTimeFormat::utcNow(), '', ''];
+		$condition = ["NOT `blocked` AND `next_contact` < ? AND (`nurl` != ? OR `url` != ?)",  DateTimeFormat::utcNow(), '', ''];
 		$outdated = DBA::count('gserver', $condition);
 		Logger::info('Server status', ['total' => $total, 'outdated' => $outdated, 'updating' => $limit]);
 

src/Worker/UpdateServerPeers.php

@@ -27,6 +27,7 @@ use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\GServer;
 use Friendica\Network\HTTPClient\Client\HttpClientAccept;
+use Friendica\Util\Network;
 use Friendica\Util\Strings;
 
 class UpdateServerPeers
@@ -56,6 +57,11 @@ class UpdateServerPeers
 		$total = 0;
 		$added = 0;
 		foreach ($peers as $peer) {
+			if (Network::isUrlBlocked('http://' . $peer)) {
+				// Ignore blocked systems as soon as possible in the loop to avoid being slowed down by tar pits
+				continue;
+			}
+
 			++$total;
 			if (DBA::exists('gserver', ['nurl' => Strings::normaliseLink('http://' . $peer)])) {
 				// We already know this server

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1510);
+	define('DB_UPDATE_VERSION', 1511);
 }
 
 return [
@@ -89,6 +89,7 @@ return [
 			"last_poco_query" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
 			"last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Last successful connection request"],
 			"last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Last failed connection request"],
+			"blocked" => ["type" => "boolean", "comment" => "Server is blocked"],
 			"failed" => ["type" => "boolean", "comment" => "Connection failed"],
 			"next_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Next connection request"],
 		],