Merge pull request #11742 from annando/server-detection

Improved server detection

src/Console/Relay.php

@@ -91,8 +91,8 @@ HELP;
 
 		if ((count($this->args) == 1) && ($this->getArgument(0) == 'list')) {
 			$contacts = $this->dba->select('apcontact', ['url'],
-			["`type` = ? AND `url` IN (SELECT `url` FROM `contact` WHERE `uid` = ? AND `rel` = ?)",
+			["`type` IN (?, ?) AND `url` IN (SELECT `url` FROM `contact` WHERE `uid` = ? AND `rel` = ?)",
-				'Application', 0, Contact::FRIEND]);
+				'Application', 'Service', 0, Contact::FRIEND]);
 			while ($contact = $this->dba->fetch($contacts)) {
 				$this->out($contact['url']);
 			}
@@ -108,7 +108,7 @@ HELP;
 			$actor = $this->getArgument(1);
 
 			$apcontact = APContact::getByURL($actor);
-			if (empty($apcontact) || ($apcontact['type'] != 'Application')) {
+			if (empty($apcontact) || !in_array($apcontact['type'], ['Application', 'Service'])) {
 				$this->out($actor . ' is no relay actor');
 				return 1;
 			}

src/Core/Worker/Cron.php

@@ -25,9 +25,11 @@ use Friendica\Core\Logger;
 use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\Contact;
 use Friendica\Model\Post;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Util\DateTimeFormat;
+use Friendica\Util\Strings;
 
 /**
  * Contains the class for jobs that are executed in an interval
@@ -179,4 +181,24 @@ class Cron
 			}
 		}
 	}
+
+	/**
+	 * Add missing "intro" records.
+	 *
+	 * @return void
+	 */
+	private static function addIntros()
+	{
+		$contacts = DBA::p("SELECT `uid`, `id`, `created` FROM `contact` WHERE `rel` = ? AND `pending` AND NOT EXISTS (SELECT `id` FROM `intro` WHERE `contact-id` = `contact`.`id`)", Contact::FOLLOWER);
+		while ($contact = DBA::fetch($contacts)) {
+			$fields = [
+				'uid'        => $contact['uid'],
+				'contact-id' => $contact['id'],
+				'datetime'   => $contact['created'],
+				'hash'       => Strings::getRandomHex()
+			];
+			Logger::notice('Adding missing intro', ['fields' => $fields]);
+			DBA::insert('intro', $fields);
+		}
+	}
 }

src/Model/APContact.php

@@ -165,6 +165,8 @@ class APContact
 				return $fetched_contact;
 			}
 			$url = $apcontact['url'];
+		} elseif (empty(parse_url($url, PHP_URL_PATH))) {
+			$apcontact['baseurl'] = $url;
 		}
 
 		// Detect multiple fast repeating request to the same address

src/Module/Admin/Federation.php

@@ -40,14 +40,17 @@ class Federation extends BaseAdmin
 			'friendica'    => ['name' => 'Friendica', 'color' => '#ffc018'], // orange from the logo
 			'birdsitelive' => ['name' => 'BirdsiteLIVE', 'color' => '#1b6ec2'], // Color from the page
 			'bookwyrm'     => ['name' => 'BookWyrm', 'color' => '#00d1b2'], // Color from the page
+			'castopod'     => ['name' => 'Castopod', 'color' => '#00564a'], // Background color from the page
 			'diaspora'     => ['name' => 'Diaspora', 'color' => '#a1a1a1'], // logo is black and white, makes a gray
 			'funkwhale'    => ['name' => 'Funkwhale', 'color' => '#4082B4'], // From the homepage
 			'gnusocial'    => ['name' => 'GNU Social/Statusnet', 'color' => '#a22430'], // dark red from the logo
 			'gotosocial'   => ['name' => 'GoToSocial', 'color' => '#df8958'], // Some color from their mascot
 			'hometown'     => ['name' => 'Hometown', 'color' => '#1f70c1'], // Color from the Patreon page
+			'honk'         => ['name' => 'Honk', 'color' => '##0d0d0d'], // Background color from the page
 			'hubzilla'     => ['name' => 'Hubzilla/Red Matrix', 'color' => '#43488a'], // blue from the logo
 			'lemmy'        => ['name' => 'Lemmy', 'color' => '#00c853'], // Green from the page
 			'mastodon'     => ['name' => 'Mastodon', 'color' => '#1a9df9'], // blue from the Mastodon logo
+			'microblog'    => ['name' => 'Microblog', 'color' => '#fdb52b'], // Color from the page
 			'misskey'      => ['name' => 'Misskey', 'color' => '#ccfefd'], // Font color of the homepage
 			'mobilizon'    => ['name' => 'Mobilizon', 'color' => '#ffd599'], // Background color of parts of the homepage
 			'nextcloud'    => ['name' => 'Nextcloud', 'color' => '#1cafff'], // Logo color
@@ -83,7 +86,8 @@ class Federation extends BaseAdmin
 			SUM(IFNULL(`active-month-users`, `active-week-users`)) AS `month`,
 			SUM(IFNULL(`active-halfyear-users`, `active-week-users`)) AS `halfyear`, `platform`,
 			ANY_VALUE(`network`) AS `network`, MAX(`version`) AS `version`
-			FROM `gserver` WHERE NOT `failed` AND `detection-method` != ? AND NOT `network` IN (?, ?) GROUP BY `platform`", GServer::DETECT_MANUAL, Protocol::PHANTOM, Protocol::FEED);
+			FROM `gserver` WHERE NOT `failed` AND `platform` != ? AND `detection-method` != ? AND NOT `network` IN (?, ?) GROUP BY `platform`",
+				'', GServer::DETECT_MANUAL, Protocol::PHANTOM, Protocol::FEED);
 		while ($gserver = DBA::fetch($gservers)) {
 			$total    += $gserver['total'];
 			$users    += $gserver['users'];
@@ -100,7 +104,7 @@ class Federation extends BaseAdmin
 
 				if (in_array($gserver['platform'], ['Red Matrix', 'redmatrix', 'red'])) {
 					$version['version'] = 'Red ' . $version['version'];
-				} elseif (in_array($gserver['platform'], ['osada', 'mistpark', 'roadhouse', 'zap'])) {
+				} elseif (in_array($gserver['platform'], ['osada', 'mistpark', 'roadhouse', 'zap', 'macgirvin', 'mkultra'])) {
 					$version['version'] = $gserver['platform'] . ' ' . $version['version'];
 				} elseif (in_array($gserver['platform'], ['activityrelay', 'pub-relay', 'selective-relay', 'aoderelay'])) {
 					$version['version'] = $gserver['platform'] . '-' . $version['version'];
@@ -116,7 +120,7 @@ class Federation extends BaseAdmin
 				$platform = 'friendica';
 			} elseif (in_array($platform, ['red matrix', 'redmatrix', 'red'])) {
 				$platform = 'hubzilla';
-			} elseif (in_array($platform, ['mistpark', 'osada', 'roadhouse', 'zap'])) {
+			} elseif (in_array($platform, ['osada', 'mistpark', 'roadhouse', 'zap', 'macgirvin', 'mkultra'])) {
 				$platform = 'mistpark';
 			} elseif(stristr($platform, 'pleroma')) {
 				$platform = 'pleroma';

src/Module/Api/Mastodon/Instance/Peers.php

@@ -24,6 +24,7 @@ namespace Friendica\Module\Api\Mastodon\Instance;
 use Friendica\Core\Protocol;
 use Friendica\Core\System;
 use Friendica\Database\DBA;
+use Friendica\Model\GServer;
 use Friendica\Module\BaseApi;
 use Friendica\Network\HTTPException;
 use Friendica\Util\Network;
@@ -41,7 +42,9 @@ class Peers extends BaseApi
 		$return = [];
 
 		// We only select for Friendica and ActivityPub servers, since it is expected to only deliver AP compatible systems here.
-		$instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND NOT `failed`", Protocol::DFRN, Protocol::ACTIVITYPUB]);
+		$instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND NOT `failed` AND NOT `detection-method` IN (?, ?, ?, ?)",
+			Protocol::DFRN, Protocol::ACTIVITYPUB,
+			GServer::DETECT_MANUAL, GServer::DETECT_HEADER, GServer::DETECT_BODY, GServer::DETECT_HOST_META]);
 		while ($instance = DBA::fetch($instances)) {
 			$urldata = parse_url($instance['url']);
 			unset($urldata['scheme']);

src/Network/HTTPClient/Client/HttpClient.php

@@ -68,6 +68,13 @@ class HttpClient implements ICanSendHttpRequests
 		$this->profiler->startRecording('network');
 		$this->logger->debug('Request start.', ['url' => $url, 'method' => $method]);
 
+		$host = parse_url($url, PHP_URL_HOST);
+		if(!filter_var($host, FILTER_VALIDATE_IP) && !@dns_get_record($host . '.', DNS_A + DNS_AAAA)) {
+			$this->logger->debug('URL cannot be resolved.', ['url' => $url, 'callstack' => System::callstack(20)]);
+			$this->profiler->stopRecording();
+			return CurlResult::createErrorCurl($url);
+		}
+
 		if (Network::isLocalLink($url)) {
 			$this->logger->info('Local link', ['url' => $url, 'callstack' => System::callstack(20)]);
 		}
@@ -125,6 +132,10 @@ class HttpClient implements ICanSendHttpRequests
 			$conf[RequestOptions::TIMEOUT] = $opts[HttpClientOptions::TIMEOUT];
 		}
 
+		if (isset($opts[HttpClientOptions::VERIFY])) {
+			$conf[RequestOptions::VERIFY] = $opts[HttpClientOptions::VERIFY];
+		}
+
 		if (!empty($opts[HttpClientOptions::BODY])) {
 			$conf[RequestOptions::BODY] = $opts[HttpClientOptions::BODY];
 		}

src/Network/HTTPClient/Client/HttpClientOptions.php

@@ -52,6 +52,12 @@ class HttpClientOptions
 	 * content_length: (int) maximum File content length
 	 */
 	const CONTENT_LENGTH = 'content_length';
+
+	/**
+	 * verify: (bool|string, default=true) Describes the SSL certificate
+	 */
+	const VERIFY = 'verify';
+
 	/**
 	 * body: (mixed) Setting the body for sending data
 	 */

src/Network/HTTPClient/Response/CurlResult.php

@@ -167,6 +167,10 @@ class CurlResult implements ICanHandleHttpResponses
 			$this->isSuccess = false;
 		}
 
+		if (empty($this->returnCode) && empty($this->header) && empty($this->body)) {
+			$this->isSuccess = false;
+		}
+
 		if (!$this->isSuccess) {
 			Logger::debug('debug', ['info' => $this->info]);
 		}

src/Util/Network.php

@@ -25,6 +25,8 @@ use Friendica\Core\Hook;
 use Friendica\Core\Logger;
 use Friendica\DI;
 use Friendica\Model\Contact;
+use Friendica\Network\HTTPClient\Client\HttpClientAccept;
+use Friendica\Network\HTTPClient\Client\HttpClientOptions;
 use Friendica\Network\HTTPException\NotModifiedException;
 use GuzzleHttp\Psr7\Uri;
 
@@ -66,14 +68,31 @@ class Network
 			$url = 'http://' . $url;
 		}
 
-		/// @TODO Really suppress function outcomes? Why not find them + debug them?
+		$xrd_timeout = DI::config()->get('system', 'xrd_timeout');
-		$h = @parse_url($url);
+		$host = parse_url($url, PHP_URL_HOST);
 
-		if (!empty($h['host']) && (@dns_get_record($h['host'], DNS_A + DNS_CNAME) || filter_var($h['host'], FILTER_VALIDATE_IP))) {
+		if (empty($host) || !(filter_var($host, FILTER_VALIDATE_IP) || @dns_get_record($host . '.', DNS_A + DNS_AAAA))) {
-			return $url;
+			return false;
 		}
 
+		if (in_array(parse_url($url, PHP_URL_SCHEME), ['https', 'http'])) {
+			$options = [HttpClientOptions::VERIFY => true, HttpClientOptions::TIMEOUT => $xrd_timeout];
+			$curlResult = DI::httpClient()->head($url, $options);
+	
+			// Workaround for systems that can't handle a HEAD request. Don't retry on timeouts.
+			if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() >= 400) && !in_array($curlResult->getReturnCode(), [408, 504])) {
+				$curlResult = DI::httpClient()->get($url, HttpClientAccept::DEFAULT, $options);
+			}
+	
+			if (!$curlResult->isSuccess()) {
+				Logger::notice('Url not reachable', ['host' => $host, 'url' => $url]);
 				return false;
+			} elseif ($curlResult->isRedirectUrl()) {
+				$url = $curlResult->getRedirectUrl();
+			}
+		}
+
+		return $url;
 	}
 
 	/**
@@ -95,7 +114,7 @@ class Network
 		$h = substr($addr, strpos($addr, '@') + 1);
 
 		// Concerning the @ see here: https://stackoverflow.com/questions/36280957/dns-get-record-a-temporary-server-error-occurred
-		if ($h && (@dns_get_record($h, DNS_A + DNS_MX) || filter_var($h, FILTER_VALIDATE_IP))) {
+		if ($h && (@dns_get_record($h, DNS_A + DNS_AAAA + DNS_MX) || filter_var($h, FILTER_VALIDATE_IP))) {
 			return true;
 		}
 		if ($h && @dns_get_record($h, DNS_CNAME + DNS_MX)) {

src/Util/ParseUrl.php

@@ -57,15 +57,22 @@ class ParseUrl
 	 * Fetch the content type of the given url
 	 * @param string $url    URL of the page
 	 * @param string $accept content-type to accept
+	 * @param int    $timeout
 	 * @return array content type
 	 */
-	public static function getContentType(string $url, string $accept = HttpClientAccept::DEFAULT): array
+	public static function getContentType(string $url, string $accept = HttpClientAccept::DEFAULT, int $timeout = 0): array
 	{
-		$curlResult = DI::httpClient()->head($url, [HttpClientOptions::ACCEPT_CONTENT => $accept]);
+		if (!empty($timeout)) {
+			$options = [HttpClientOptions::TIMEOUT => $timeout];
+		} else {
+			$options = [];
+		}
 
-		// Workaround for systems that can't handle a HEAD request
+		$curlResult = DI::httpClient()->head($url, array_merge([HttpClientOptions::ACCEPT_CONTENT => $accept], $options));
-		if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() == 405)) {
+
-			$curlResult = DI::httpClient()->get($url, $accept, [HttpClientOptions::CONTENT_LENGTH => 1000000]);
+		// Workaround for systems that can't handle a HEAD request. Don't retry on timeouts.
+		if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() >= 400) && !in_array($curlResult->getReturnCode(), [408, 504])) {
+			$curlResult = DI::httpClient()->get($url, $accept, array_merge([HttpClientOptions::CONTENT_LENGTH => 1000000], $options));
 		}
 
 		if (!$curlResult->isSuccess()) {

static/platforms.config.php

@@ -0,0 +1,145 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2022, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+$platforms = [
+	'AlphaChat' => 'alphachat',
+	'AV Arcade' => 'av-arcade',
+	'BaseKit' => 'basekit',
+	'BBEdit' => 'bbedit',
+	'Big Cartel' => 'big-cartel',
+	'Blogger' => 'blogger',
+	'Bloom' => 'bloom',
+	'Bludit' => 'bludit',
+	'BunnyPress' => 'bunnypress',
+	'Contao Open Source CMS' => 'contao',
+	'castopod' => 'castopod',
+	'cgit' => 'cgit',
+	'Chevereto' => 'chevereto',
+	'ClanSphere' => 'clansphere',
+	'Claudia Content' => 'claudia-content',
+	'diaspora' => 'diaspora',
+	'DIMV' => 'dimv',
+	'Discourse' => 'discourse',
+	'Divi' => 'divi',
+	'Django-cypress' => 'django-cypress',
+	'Docutils' => 'docutils',
+	'DokuWiki' => 'dokuwiki',
+	'Drupal' => 'drupal',
+	'Element' => 'element',
+	'Eleventy' => 'eleventy',
+	'filerun' => 'filerun',
+	'FlatPress' => 'flatpress',
+	'Gatsby' => 'gatsby',
+	'Ghost' => 'ghost',
+	'gitweb' => 'gitweb',
+	'gnusocial' => 'gnusocial',
+	'Government Site Builder' => 'government-site-builder',
+	'GravCMS' => 'gravcms',
+	'grocy' => 'grocy',
+	'Gruta' => 'gruta',
+	'hakyll' => 'hakyll',
+	'HedgeDoc - Collaborative markdown notes' => 'hedgedoc',
+	'Hello, world. https://github.com/mimecuvalo/helloworld' => 'helloworld',
+	'Hexo' => 'hexo',
+	'Hugo' => 'hugo',
+	'ian' => 'ian',
+	'InterRed' => 'interred',
+	'Ikiwiki' => 'ikiwiki',
+	'Jekyll' => 'jekyll',
+	'Joomla!' => 'joomla',
+	'KeyHelp' => 'keyhelp',
+	'Known https://withknown.com' => 'known',
+	'KONTEXT-CMS (c) WARENFORM [www.warenform.net]' => 'kontext-cms',
+	'ktistec' => 'ktistec',
+	'lemoncurry' => 'lemoncurry',
+	'LibreOffice' => 'libreoffice',
+	'Magazine News Byte' => 'magazine-news-byte',
+	'Magnet' => 'magnet',
+	'mastodon' => 'mastodon',
+	'Mattermost' => 'mattermost',
+	'MediaWiki' => 'mediawiki',
+	'Medium' => 'medium',
+	'microblog' => 'microblog',
+	'Misskey' => 'misskey',
+	'Mobirise' => 'mobirise',
+	'Movable Type Pro' => 'movable-type',
+	'Movim' => 'movim',
+	'MyArcadePlugin Pro' => 'my-arcade-plugin',
+	'Nanoc' => 'nanoc',
+	'nb3 system v1.6' => 'nb3',
+	'Nikola (getnikola.com)' => 'nikola',
+	'Odoo' => 'odoo',
+	'Org mode' => 'org-mode',
+	'Org-mode' => 'org-mode',
+	'Org Mode' => 'org-mode',
+	'Osclass' => 'osclass',
+	'pamphlets/vinyl-press' => 'pamphlets',
+	'peertube' => 'peertube',
+	'phpMyFAQ' => 'phpmyfaq',
+	'Pelican' => 'pelican',
+	'pH7CMS,' => 'ph7cms',
+	'Polr' => 'polr',
+	'Publii Open-Source CMS for Static Site' => 'publii',
+	'pygruta' => 'pygruta',
+	'Riot' => 'riot',
+	'Satelito' => 'satelito',
+	'Scully' => 'scully',
+	'Sedo' => 'sedo',
+	'sitebaker' => 'sitebaker',
+	'SitePad' => 'sitepad',
+	'SMAR' => 'smar',
+	'SPIP' => 'spip',
+	'STUDIO' => 'studio',
+	'Synology - Synology DiskStation' => 'synology',
+	'Thufie' => 'thufie',
+	'TiddlyWiki' => 'tiddlywiki',
+	'Typesetter CMS' => 'typesetter',
+	'TYPO3 CMS' => 'typo3',
+	'Tumblr' => 'tumblr',
+	'Typecho' => 'typecho',
+	'http://txt2tags.org' => 'txt2tags',
+	'UISP' => 'uisp',
+	'VuePress' => 'vuepress',
+	'webEdition CMS' => 'webedition',
+	'Webflow' => 'webflow',
+	'WikkaWiki' => 'wikkawiki',
+	'Wix.com' => 'wix.com',
+	'WordPress.com' => 'wordpress',
+	'WordPress' => 'wordpress',
+	'Write.as' => 'write.as',
+	'XAG/CMS' => 'xagcms',
+	'Zim' => 'zim',
+];
+
+$ap_platforms = [
+	'honk' => 'honk',
+	'PeerTube' => 'peertube',
+];
+
+$dfrn_platforms = [
+	'Friendika' => 'friendika',
+	'Friendica' => 'friendica',
+];
+
+$zap_platforms = [
+	'hubzilla' => 'hubzilla',
+	'osada' => 'osada',
+];