clones/friendica
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

News "blocked" field on the gserver table

This commit is contained in:
Michael 2023-01-17 17:25:19 +00:00
parent d63aca97b8
commit 356a2a9a5a
14 changed files with 140 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
database.sql
View File

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2023.03-dev (Giant Rhubarb)
-- DB_UPDATE_VERSION 1510
-- DB_UPDATE_VERSION 1511
-- ------------------------------------------
@ -34,6 +34,7 @@ CREATE TABLE IF NOT EXISTS `gserver` (
`last_poco_query` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
`last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Last successful connection request',
`last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Last failed connection request',
`blocked` boolean COMMENT 'Server is blocked',
`failed` boolean COMMENT 'Connection failed',
`next_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Next connection request',
PRIMARY KEY(`id`),

1
doc/database/db_gserver.md
View File

@ -34,6 +34,7 @@ Fields
| last_poco_query | | datetime | YES | | 0001-01-01 00:00:00 | |
| last_contact | Last successful connection request | datetime | YES | | 0001-01-01 00:00:00 | |
| last_failure | Last failed connection request | datetime | YES | | 0001-01-01 00:00:00 | |
| blocked | Server is blocked | boolean | YES | | NULL | |
| failed | Connection failed | boolean | YES | | NULL | |
| next_contact | Next connection request | datetime | YES | | 0001-01-01 00:00:00 | |

4
src/Console/ServerBlock.php
View File

@ -24,6 +24,7 @@ namespace Friendica\Console;
use Asika\SimpleConsole\CommandArgsException;
use Asika\SimpleConsole\Console;
use Console_Table;
use Friendica\Core\Worker;
use Friendica\Moderation\DomainPatternBlocklist;
/**
@ -127,6 +128,7 @@ HELP;
if ($this->blocklist->append($newBlockList)) {
$this->out(sprintf("Entries from %s that were not blocked before are now blocked", $filename));
Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
return 0;
} else {
$this->out("Couldn't save the block list");
@ -169,6 +171,7 @@ HELP;
} else {
$this->out(sprintf("The domain pattern '%s' is now blocked. (Reason: '%s')", $pattern, $reason));
}
Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
return 0;
} else {
$this->out(sprintf("Couldn't save '%s' as blocked domain pattern", $pattern));
@ -193,6 +196,7 @@ HELP;
if ($result) {
if ($result == 2) {
$this->out(sprintf("The domain pattern '%s' isn't blocked anymore", $pattern));
Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
return 0;
} else {
$this->out(sprintf("The domain pattern '%s' wasn't blocked.", $pattern));

65
src/Model/GServer.php
View File

@ -127,6 +127,13 @@ class GServer
$gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => Strings::normaliseLink($url)]);
if (DBA::isResult($gserver)) {
Logger::debug('Got ID for URL', ['id' => $gserver['id'], 'url' => $url, 'callstack' => System::callstack(20)]);
if (Network::isUrlBlocked($url)) {
self::setBlockedById($gserver['id']);
} else {
self::setUnblockedById($gserver['id']);
}
return $gserver['id'];
}
@ -341,6 +348,12 @@ class GServer
return false;
}
if (Network::isUrlBlocked($server_url)) {
Logger::info('Server is blocked', ['url' => $server_url]);
self::setBlockedByUrl($server_url);
return false;
}
$gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($server_url)]);
if (DBA::isResult($gserver)) {
if ($gserver['created'] <= DBA::NULL_DATETIME) {
@ -370,8 +383,13 @@ class GServer
public static function setReachableById(int $gsid, string $network)
{
$gserver = DBA::selectFirst('gserver', ['url', 'failed', 'next_contact', 'network'], ['id' => $gsid]);
if (DBA::isResult($gserver) && $gserver['failed']) {
$fields = ['failed' => false, 'last_contact' => DateTimeFormat::utcNow()];
if (!DBA::isResult($gserver)) {
return;
}
$blocked = Network::isUrlBlocked($gserver['url']);
if ($gserver['failed']) {
$fields = ['failed' => false, 'blocked' => $blocked, 'last_contact' => DateTimeFormat::utcNow()];
if (!empty($network) && !in_array($gserver['network'], Protocol::FEDERATED)) {
$fields['network'] = $network;
}
@ -381,6 +399,10 @@ class GServer
if (strtotime($gserver['next_contact']) < time()) {
UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
}
} elseif ($blocked) {
self::setBlockedById($gsid);
} else {
self::setUnblockedById($gsid);
}
}
@ -393,7 +415,7 @@ class GServer
{
$gserver = DBA::selectFirst('gserver', ['url', 'failed', 'next_contact'], ['id' => $gsid]);
if (DBA::isResult($gserver) && !$gserver['failed']) {
self::update(['failed' => true, 'last_failure' => DateTimeFormat::utcNow()], ['id' => $gsid]);
self::update(['failed' => true, 'blocked' => Network::isUrlBlocked($gserver['url']), 'last_failure' => DateTimeFormat::utcNow()], ['id' => $gsid]);
Logger::info('Set failed status for server', ['url' => $gserver['url']]);
if (strtotime($gserver['next_contact']) < time()) {
@ -402,6 +424,33 @@ class GServer
}
}
public static function setUnblockedById(int $gsid)
{
$gserver = DBA::selectFirst('gserver', ['url'], ["(`blocked` OR `blocked` IS NULL) AND `id` = ?", $gsid]);
if (DBA::isResult($gserver)) {
self::update(['blocked' => false], ['id' => $gsid]);
Logger::info('Set unblocked status for server', ['url' => $gserver['url']]);
}
}
public static function setBlockedById(int $gsid)
{
$gserver = DBA::selectFirst('gserver', ['url'], ["(NOT `blocked` OR `blocked` IS NULL) AND `id` = ?", $gsid]);
if (DBA::isResult($gserver)) {
self::update(['blocked' => true, 'failed' => true], ['id' => $gsid]);
Logger::info('Set blocked status for server', ['url' => $gserver['url']]);
}
}
public static function setBlockedByUrl(string $url)
{
$gserver = DBA::selectFirst('gserver', ['url', 'id'], ["(NOT `blocked` OR `blocked` IS NULL) AND `nurl` = ?", Strings::normaliseLink($url)]);
if (DBA::isResult($gserver)) {
self::update(['blocked' => true, 'failed' => true], ['id' => $gserver['id']]);
Logger::info('Set blocked status for server', ['url' => $gserver['url']]);
}
}
/**
* Set failed server status
*
@ -412,7 +461,7 @@ class GServer
$gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($url)]);
if (DBA::isResult($gserver)) {
$next_update = self::getNextUpdateDate(false, $gserver['created'], $gserver['last_contact']);
self::update(['url' => $url, 'failed' => true, 'last_failure' => DateTimeFormat::utcNow(),
self::update(['url' => $url, 'failed' => true, 'blocked' => Network::isUrlBlocked($url), 'last_failure' => DateTimeFormat::utcNow(),
'next_contact' => $next_update, 'network' => Protocol::PHANTOM, 'detection-method' => null],
['nurl' => Strings::normaliseLink($url)]);
Logger::info('Set failed status for existing server', ['url' => $url]);
@ -491,7 +540,7 @@ class GServer
*
* @return boolean 'true' if server could be detected
*/
public static function detect(string $url, string $network = '', bool $only_nodeinfo = false): bool
private static function detect(string $url, string $network = '', bool $only_nodeinfo = false): bool
{
Logger::info('Detect server type', ['server' => $url]);
@ -732,9 +781,9 @@ class GServer
}
$serverdata['next_contact'] = self::getNextUpdateDate(true, '', '', in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]));
$serverdata['last_contact'] = DateTimeFormat::utcNow();
$serverdata['failed'] = false;
$serverdata['failed'] = false;
$serverdata['blocked'] = false;
$gserver = DBA::selectFirst('gserver', ['network'], ['nurl' => Strings::normaliseLink($url)]);
if (!DBA::isResult($gserver)) {
@ -2259,7 +2308,7 @@ class GServer
$last_update = date('c', time() - (60 * 60 * 24 * $requery_days));
$gservers = DBA::select('gserver', ['id', 'url', 'nurl', 'network', 'poco', 'directory-type'],
["NOT `failed` AND `directory-type` != ? AND `last_poco_query` < ?", GServer::DT_NONE, $last_update],
["NOT `blocked` AND NOT `failed` AND `directory-type` != ? AND `last_poco_query` < ?", GServer::DT_NONE, $last_update],
['order' => ['RAND()']]);
while ($gserver = DBA::fetch($gservers)) {

2
src/Module/Api/Mastodon/Instance/Peers.php
View File

@ -42,7 +42,7 @@ class Peers extends BaseApi
$return = [];
// We only select for Friendica and ActivityPub servers, since it is expected to only deliver AP compatible systems here.
$instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND NOT `failed` AND NOT `detection-method` IN (?, ?, ?, ?)",
$instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND NOT `blocked` AND NOT `failed` AND NOT `detection-method` IN (?, ?, ?, ?)",
Protocol::DFRN, Protocol::ACTIVITYPUB,
GServer::DETECT_MANUAL, GServer::DETECT_HEADER, GServer::DETECT_BODY, GServer::DETECT_HOST_META]);
while ($instance = DBA::fetch($instances)) {

2
src/Module/Moderation/Blocklist/Server/Add.php
View File

@ -75,6 +75,8 @@ class Add extends BaseModeration
// Add new item to blocklist
$this->blocklist->addPattern($pattern, trim($request['reason']));
Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
$this->systemMessages->addInfo($this->t('Server domain pattern added to the blocklist.'));
if (!empty($request['purge'])) {

3
src/Module/Moderation/Blocklist/Server/Import.php
View File

@ -25,6 +25,7 @@ use Friendica\App;
use Friendica\Core\L10n;
use Friendica\Core\Renderer;
use Friendica\Core\Session\Capability\IHandleUserSessions;
use Friendica\Core\Worker;
use Friendica\Moderation\DomainPatternBlocklist;
use Friendica\Module\Response;
use Friendica\Navigation\SystemMessages;
@ -95,6 +96,8 @@ class Import extends \Friendica\Module\BaseModeration
}
}
Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
$this->baseUrl->redirect('/moderation/blocklist/server');
}
}

3
src/Module/Moderation/Blocklist/Server/Index.php
View File

@ -25,6 +25,7 @@ use Friendica\App;
use Friendica\Core\L10n;
use Friendica\Core\Renderer;
use Friendica\Core\Session\Capability\IHandleUserSessions;
use Friendica\Core\Worker;
use Friendica\Moderation\DomainPatternBlocklist;
use Friendica\Module\BaseModeration;
use Friendica\Module\Response;
@ -70,6 +71,8 @@ class Index extends BaseModeration
$this->blocklist->set($blocklist);
Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
$this->baseUrl->redirect('moderation/blocklist/server');
}

5
src/Worker/Cron.php
View File

@ -85,8 +85,6 @@ class Cron
// Hourly cron calls
if ((DI::keyValue()->get('last_cron_hourly') ?? 0) + 3600 < time()) {
// Update trending tags cache for the community page
Tag::setLocalTrendingHashtags(24, 20);
Tag::setGlobalTrendingHashtags(24, 20);
@ -145,6 +143,9 @@ class Cron
// Resubscribe to relay servers
Relay::reSubscribe();
// Update "blocked" status of servers
Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers');
DI::keyValue()->set('last_cron_daily', time());
}

53
src/Worker/UpdateBlockedServers.php Normal file
View File

@ -0,0 +1,53 @@
<?php
/**
* @copyright Copyright (C) 2010-2023, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Database\DBA;
use Friendica\Model\GServer;
use Friendica\Util\Network;
class UpdateBlockedServers
{
/**
* Updates the server blocked status
*/
public static function execute()
{
Logger::debug('Update blocked servers - start');
$gservers = DBA::select('gserver', ['id', 'url', 'blocked']);
while ($gserver = DBA::fetch($gservers)) {
$blocked = Network::isUrlBlocked($gserver['url']);
if (!is_null($gserver['blocked']) && ($blocked == $gserver['blocked'])) {
continue;
}
if ($blocked) {
GServer::setBlockedById($gserver['id']);
} else {
GServer::setUnblockedById($gserver['id']);
}
}
DBA::close($gservers);
Logger::debug('Update blocked servers - done');
}
}

2
src/Worker/UpdateGServer.php
View File

@ -55,6 +55,7 @@ class UpdateGServer
// Silently dropping the worker task if the server domain is blocked
if (Network::isUrlBlocked($filtered)) {
GServer::setBlockedByUrl($filtered);
return;
}
@ -84,6 +85,7 @@ class UpdateGServer
{
// Dropping the worker task if the server domain is blocked
if (Network::isUrlBlocked($serverUrl)) {
GServer::setBlockedByUrl($serverUrl);
return 0;
}

3
src/Worker/UpdateGServers.php
View File

@ -27,7 +27,6 @@ use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Strings;
use GuzzleHttp\Psr7\Uri;
class UpdateGServers
{
@ -49,7 +48,7 @@ class UpdateGServers
}
$total = DBA::count('gserver');
$condition = ["`next_contact` < ? AND (`nurl` != ? OR `url` != ?)", DateTimeFormat::utcNow(), '', ''];
$condition = ["NOT `blocked` AND `next_contact` < ? AND (`nurl` != ? OR `url` != ?)", DateTimeFormat::utcNow(), '', ''];
$outdated = DBA::count('gserver', $condition);
Logger::info('Server status', ['total' => $total, 'outdated' => $outdated, 'updating' => $limit]);

6
src/Worker/UpdateServerPeers.php
View File

@ -27,6 +27,7 @@ use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\GServer;
use Friendica\Network\HTTPClient\Client\HttpClientAccept;
use Friendica\Util\Network;
use Friendica\Util\Strings;
class UpdateServerPeers
@ -56,6 +57,11 @@ class UpdateServerPeers
$total = 0;
$added = 0;
foreach ($peers as $peer) {
if (Network::isUrlBlocked('http://' . $peer)) {
// Ignore blocked systems as soon as possible in the loop to avoid being slowed down by tar pits
continue;
}
++$total;
if (DBA::exists('gserver', ['nurl' => Strings::normaliseLink('http://' . $peer)])) {
// We already know this server

3
static/dbstructure.config.php
View File

@ -55,7 +55,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1510);
define('DB_UPDATE_VERSION', 1511);
}
return [
@ -89,6 +89,7 @@ return [
"last_poco_query" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
"last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Last successful connection request"],
"last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Last failed connection request"],
"blocked" => ["type" => "boolean", "comment" => "Server is blocked"],
"failed" => ["type" => "boolean", "comment" => "Connection failed"],
"next_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Next connection request"],
],