clones/friendica
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Check and add a server

This commit is contained in:
Michael 2020-12-03 22:32:51 +00:00
parent 8cbe2f025b
commit 8d999f54d0
5 changed files with 63 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
src/Model/GServer.php
View File

@ -70,6 +70,22 @@ class GServer
const DETECT_NODEINFO_1 = 101;
const DETECT_NODEINFO_2 = 102;
/**
* Check for the existance of a server and adds it in the background if not existant
*
* @param string $url
* @param boolean $only_nodeinfo
* @return void
*/
public static function add(string $url, bool $only_nodeinfo = false)
{
if (self::getID($url, false)) {
return;
}
Worker::add(PRIORITY_LOW, 'UpdateGServer', $url, $only_nodeinfo);
}
/**
* Get the ID for the given server URL
*
@ -173,61 +189,6 @@ class GServer
return DateTimeFormat::utc('now +3 month');
}
/**
* Decides if a server needs to be updated, based upon several date fields
*
* @param date $created Creation date of that server entry
* @param date $updated When had the server entry be updated
* @param date $last_failure Last failure when contacting that server
* @param date $last_contact Last time the server had been contacted
*
* @return boolean Does the server record needs an update?
*/
public static function updateNeeded($created, $updated, $last_failure, $last_contact)
{
$now = strtotime(DateTimeFormat::utcNow());
if ($updated > $last_contact) {
$contact_time = strtotime($updated);
} else {
$contact_time = strtotime($last_contact);
}
$failure_time = strtotime($last_failure);
$created_time = strtotime($created);
// If there is no "created" time then use the current time
if ($created_time <= 0) {
$created_time = $now;
}
// If the last contact was less than 24 hours then don't update
if (($now - $contact_time) < (60 * 60 * 24)) {
return false;
}
// If the last failure was less than 24 hours then don't update
if (($now - $failure_time) < (60 * 60 * 24)) {
return false;
}
// If the last contact was less than a week ago and the last failure is older than a week then don't update
//if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
// return false;
// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
return false;
}
// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
return false;
}
return true;
}
/**
* Checks the state of the given server.
*
@ -241,7 +202,6 @@ class GServer
public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false)
{
$server_url = self::cleanURL($server_url);
if ($server_url == '') {
return false;
}
@ -254,24 +214,11 @@ class GServer
DBA::update('gserver', $fields, $condition);
}
$last_contact = $gserver['last_contact'];
$last_failure = $gserver['last_failure'];
// See discussion under https://forum.friendi.ca/display/0b6b25a8135aabc37a5a0f5684081633
// It can happen that a zero date is in the database, but storing it again is forbidden.
if ($last_contact < DBA::NULL_DATETIME) {
$last_contact = DBA::NULL_DATETIME;
}
if ($last_failure < DBA::NULL_DATETIME) {
$last_failure = DBA::NULL_DATETIME;
}
if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) {
if (!$force && (strtotime($gserver['next_contact']) > time())) {
Logger::info('No update needed', ['server' => $server_url]);
return ($last_contact >= $last_failure);
return (!$gserver['failed']);
}
Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force, 'Created' => $gserver['created'], 'Failure' => $last_failure, 'Contact' => $last_contact]);
Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force]);
} else {
Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]);
}
@ -1690,7 +1637,7 @@ class GServer
if (!empty($data['data']['nodes'])) {
foreach ($data['data']['nodes'] as $server) {
// Using "only_nodeinfo" since servers that are listed on that page should always have it.
Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host'], true);
self::add('https://' . $server['host'], true);
}
}
}
@ -1709,7 +1656,7 @@ class GServer
foreach ($servers['instances'] as $server) {
$url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name'];
Worker::add(PRIORITY_LOW, 'UpdateGServer', $url);
self::add($url);
}
}
}

18
src/Worker/UpdateGServer.php
View File

@ -22,6 +22,7 @@
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Database\DBA;
use Friendica\Model\GServer;
use Friendica\Util\Strings;
@ -32,7 +33,7 @@ class UpdateGServer
* @param string $server_url Server URL
* @param boolean $only_nodeinfo Only use nodeinfo for server detection
*/
public static function execute(string $server_url, bool $only_nodeinfo = false, bool $force = false)
public static function execute(string $server_url, bool $only_nodeinfo = false)
{
if (empty($server_url)) {
return;
@ -40,11 +41,22 @@ class UpdateGServer
$filtered = filter_var($server_url, FILTER_SANITIZE_URL);
if (substr(Strings::normaliseLink($filtered), 0, 7) != 'http://') {
GServer::setFailure($filtered);
GServer::setFailure($server_url);
return;
}
$ret = GServer::check($filtered, '', $force, $only_nodeinfo);
if (($filtered != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) {
GServer::setFailure($server_url);
return;
}
$cleaned = GServer::cleanURL($server_url);
if (($cleaned != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) {
GServer::setFailure($server_url);
return;
}
$ret = GServer::check($filtered, '', true, $only_nodeinfo);
Logger::info('Updated gserver', ['url' => $filtered, 'result' => $ret]);
}
}

26
src/Worker/UpdateGServers.php
View File

@ -24,33 +24,49 @@ namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Util\Strings;
class UpdateGServers
{
/**
* Updates up to 100 servers
* Updates a defined number of servers
*/
public static function execute()
{
$update_limit = DI::config()->get('system', 'gserver_update_limit');
if (empty($update_limit)) {
return;
}
$updating = Worker::countWorkersByCommand('UpdateGServer');
$limit = 100 - $updating;
$limit = $update_limit - $updating;
if ($limit <= 0) {
Logger::info('The number of currently running jobs exceed the limit');
return;
}
$outdated = DBA::count('gserver', ["`next_contact` < UTC_TIMESTAMP()"]);
$total = DBA::count('gserver');
$condition = ["`next_contact` < UTC_TIMESTAMP() AND (`nurl` != ? OR `url` != ?)", '', ''];
$outdated = DBA::count('gserver', $condition);
Logger::info('Server status', ['total' => $total, 'outdated' => $outdated, 'updating' => $limit]);
$gservers = DBA::select('gserver', ['url'], ["`next_contact` < UTC_TIMESTAMP()"], ['limit' => $limit]);
$gservers = DBA::select('gserver', ['url', 'nurl'], $condition, ['limit' => $limit]);
if (!DBA::isResult($gservers)) {
return;
}
$count = 0;
while ($gserver = DBA::fetch($gservers)) {
Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false, true);
// Sometimes the "nurl" and "url" doesn't seem to fit, see https://forum.friendi.ca/display/ec054ce7-155f-c94d-6159-f50372664245
// There are duplicated "url" but not "nurl". So we check both addresses instead of just overwriting them,
// since that would mean loosing data.
if (!empty($gserver['url'])) {
Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url']);
}
if (!empty($gserver['nurl']) && ($gserver['nurl'] != Strings::normaliseLink($gserver['url']))) {
Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['nurl']);
}
$count++;
}
DBA::close($gservers);

4
src/Worker/UpdateServerPeers.php
View File

@ -22,9 +22,9 @@
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\GServer;
use Friendica\Util\Strings;
class UpdateServerPeers
@ -58,7 +58,7 @@ class UpdateServerPeers
continue;
}
// This endpoint doesn't offer the schema. So we assume that it is HTTPS.
Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $peer);
GServer::add('https://' . $peer);
++$added;
}
Logger::info('Server peer update ended', ['total' => $total, 'added' => $added, 'url' => $url]);

4
static/defaults.config.php
View File

@ -254,6 +254,10 @@ return [
// This can alternatively be set on a per account basis in the pconfig table.
'groupedit_image_limit' => 400,
// gserver_update_limit (Integer)
// How much servers should be checked at a time?
'gserver_update_limit' => 100,
// hsts (Boolean)
// Enables the sending of HTTP Strict Transport Security headers.
'hsts' => false,