diff --git a/src/Model/APContact.php b/src/Model/APContact.php index 55dd756638..9759876fab 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -28,6 +28,7 @@ use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\Database\DBStructure; use Friendica\DI; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPException; use Friendica\Network\Probe; use Friendica\Protocol\ActivityNamespace; @@ -71,10 +72,10 @@ class APContact $data = ['addr' => $addr]; $template = 'https://' . $addr_parts[1] . '/.well-known/webfinger?resource=acct:' . urlencode($addr); - $webfinger = Probe::webfinger(str_replace('{uri}', urlencode($addr), $template), 'application/jrd+json'); + $webfinger = Probe::webfinger(str_replace('{uri}', urlencode($addr), $template), HttpClient::ACCEPT_JRD_JSON); if (empty($webfinger['links'])) { $template = 'http://' . $addr_parts[1] . '/.well-known/webfinger?resource=acct:' . urlencode($addr); - $webfinger = Probe::webfinger(str_replace('{uri}', urlencode($addr), $template), 'application/jrd+json'); + $webfinger = Probe::webfinger(str_replace('{uri}', urlencode($addr), $template), HttpClient::ACCEPT_JRD_JSON); if (empty($webfinger['links'])) { return []; } diff --git a/src/Model/Contact.php b/src/Model/Contact.php index e1152b4081..2c88e798de 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -38,10 +38,6 @@ use Friendica\DI; use Friendica\Network\HTTPException; use Friendica\Network\Probe; use Friendica\Protocol\Activity; -use Friendica\Protocol\ActivityPub; -use Friendica\Protocol\Diaspora; -use Friendica\Protocol\OStatus; -use Friendica\Protocol\Salmon; use Friendica\Util\DateTimeFormat; use Friendica\Util\Images; use Friendica\Util\Network; diff --git a/src/Model/GServer.php b/src/Model/GServer.php index b739c74208..3cb4dc5f12 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -34,6 +34,7 @@ use Friendica\DI; use Friendica\Module\Register; use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Network\HTTPClient\Capability\ICanHandleHttpResponses; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Protocol\Relay; use Friendica\Util\DateTimeFormat; use Friendica\Util\Network; @@ -343,7 +344,7 @@ class GServer // When a nodeinfo is present, we don't need to dig further $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = DI::httpClient()->get($url . '/.well-known/nodeinfo', [HttpClientOptions::TIMEOUT => $xrd_timeout]); + $curlResult = DI::httpClient()->get($url . '/.well-known/nodeinfo', [HttpClientOptions::TIMEOUT => $xrd_timeout, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if ($curlResult->isTimeout()) { self::setFailure($url); return false; @@ -351,7 +352,7 @@ class GServer // On a redirect follow the new host but mark the old one as failure if ($curlResult->isSuccess() && !empty($curlResult->getRedirectUrl()) && (parse_url($url, PHP_URL_HOST) != parse_url($curlResult->getRedirectUrl(), PHP_URL_HOST))) { - $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout]); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_HTML]); if (!empty($curlResult->getRedirectUrl()) && parse_url($url, PHP_URL_HOST) != parse_url($curlResult->getRedirectUrl(), PHP_URL_HOST)) { Logger::info('Found redirect. Mark old entry as failure', ['old' => $url, 'new' => $curlResult->getRedirectUrl()]); self::setFailure($url); @@ -393,7 +394,7 @@ class GServer $basedata = ['detection-method' => self::DETECT_MANUAL]; } - $curlResult = DI::httpClient()->get($baseurl, [HttpClientOptions::TIMEOUT => $xrd_timeout]); + $curlResult = DI::httpClient()->get($baseurl, [HttpClientOptions::TIMEOUT => $xrd_timeout, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_HTML]); if ($curlResult->isSuccess()) { if (!empty($curlResult->getRedirectUrl()) && (parse_url($baseurl, PHP_URL_HOST) != parse_url($curlResult->getRedirectUrl(), PHP_URL_HOST))) { Logger::info('Found redirect. Mark old entry as failure', ['old' => $url, 'new' => $curlResult->getRedirectUrl()]); @@ -417,7 +418,7 @@ class GServer // When the base path doesn't seem to contain a social network we try the complete path. // Most detectable system have to be installed in the root directory. // We checked the base to avoid false positives. - $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout]); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_HTML]); if ($curlResult->isSuccess()) { $urldata = self::analyseRootHeader($curlResult, $serverdata); $urldata = self::analyseRootBody($curlResult, $urldata, $url); @@ -444,7 +445,13 @@ class GServer // All following checks are done for systems that always have got a "host-meta" endpoint. // With this check we don't have to waste time and ressources for dead systems. // Also this hopefully prevents us from receiving abuse messages. - if (empty($serverdata['network']) && !self::validHostMeta($url)) { + $validHostMeta = self::validHostMeta($url); + + if (empty($serverdata['network']) && !$validHostMeta) { + $serverdata = self::detectFromContacts($url, $serverdata); + } + + if (empty($serverdata['network']) && !$validHostMeta) { self::setFailure($url); return false; } @@ -581,7 +588,7 @@ class GServer { Logger::info('Discover relay data', ['server' => $server_url]); - $curlResult = DI::httpClient()->get($server_url . '/.well-known/x-social-relay'); + $curlResult = DI::httpClient()->get($server_url . '/.well-known/x-social-relay', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { return; } @@ -676,7 +683,7 @@ class GServer */ private static function fetchStatistics(string $url) { - $curlResult = DI::httpClient()->get($url . '/statistics.json'); + $curlResult = DI::httpClient()->get($url . '/statistics.json', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { return []; } @@ -802,8 +809,7 @@ class GServer */ private static function parseNodeinfo1(string $nodeinfo_url) { - $curlResult = DI::httpClient()->get($nodeinfo_url); - + $curlResult = DI::httpClient()->get($nodeinfo_url, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { return []; } @@ -896,7 +902,7 @@ class GServer */ private static function parseNodeinfo2(string $nodeinfo_url) { - $curlResult = DI::httpClient()->get($nodeinfo_url); + $curlResult = DI::httpClient()->get($nodeinfo_url, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { return []; } @@ -991,7 +997,7 @@ class GServer */ private static function fetchSiteinfo(string $url, array $serverdata) { - $curlResult = DI::httpClient()->get($url . '/siteinfo.json'); + $curlResult = DI::httpClient()->get($url . '/siteinfo.json', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1076,7 +1082,7 @@ class GServer private static function validHostMeta(string $url) { $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = DI::httpClient()->get($url . '/.well-known/host-meta', [HttpClientOptions::TIMEOUT => $xrd_timeout]); + $curlResult = DI::httpClient()->get($url . '/.well-known/host-meta', [HttpClientOptions::TIMEOUT => $xrd_timeout, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_XRD_XML]); if (!$curlResult->isSuccess()) { return false; } @@ -1168,7 +1174,7 @@ class GServer { $serverdata['poco'] = ''; - $curlResult = DI::httpClient()->get($url . '/poco'); + $curlResult = DI::httpClient()->get($url . '/poco', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1198,7 +1204,7 @@ class GServer */ public static function checkMastodonDirectory(string $url, array $serverdata) { - $curlResult = DI::httpClient()->get($url . '/api/v1/directory?limit=1'); + $curlResult = DI::httpClient()->get($url . '/api/v1/directory?limit=1', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1225,8 +1231,7 @@ class GServer */ private static function detectPeertube(string $url, array $serverdata) { - $curlResult = DI::httpClient()->get($url . '/api/v1/config'); - + $curlResult = DI::httpClient()->get($url . '/api/v1/config', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1273,8 +1278,7 @@ class GServer */ private static function detectNextcloud(string $url, array $serverdata) { - $curlResult = DI::httpClient()->get($url . '/status.php'); - + $curlResult = DI::httpClient()->get($url . '/status.php', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1298,8 +1302,7 @@ class GServer } private static function fetchWeeklyUsage(string $url, array $serverdata) { - $curlResult = DI::httpClient()->get($url . '/api/v1/instance/activity'); - + $curlResult = DI::httpClient()->get($url . '/api/v1/instance/activity', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1328,6 +1331,34 @@ class GServer return $serverdata; } + + /** + * Detects the server network type from contacts of that server + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + private static function detectFromContacts(string $url, array $serverdata) + { + $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => Strings::normaliseLink($url)]); + if (empty($gserver)) { + return $serverdata; + } + + $contact = Contact::selectFirst(['id'], ['uid' => 0, 'failed' => false, 'gsid' => $gserver['id']]); + + // Via probing we can be sure that the server is responding + if (Contact::updateFromProbe($contact['id'])) { + $contact = Contact::selectFirst(['network', 'failed'], ['id' => $contact['id']]); + if (!$contact['failed'] && in_array($contact['network'], Protocol::FEDERATED)) { + $serverdata['network'] = $contact['network']; + } + } + + return $serverdata; + } /** * Detects data from a given server url if it was a mastodon alike system @@ -1339,8 +1370,7 @@ class GServer */ private static function detectMastodonAlikes(string $url, array $serverdata) { - $curlResult = DI::httpClient()->get($url . '/api/v1/instance'); - + $curlResult = DI::httpClient()->get($url . '/api/v1/instance', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1405,7 +1435,7 @@ class GServer */ private static function detectHubzilla(string $url, array $serverdata) { - $curlResult = DI::httpClient()->get($url . '/api/statusnet/config.json'); + $curlResult = DI::httpClient()->get($url . '/api/statusnet/config.json', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1502,7 +1532,7 @@ class GServer */ private static function detectPumpIO(string $url, array $serverdata) { - $curlResult = DI::httpClient()->get($url . '/.well-known/host-meta.json'); + $curlResult = DI::httpClient()->get($url . '/.well-known/host-meta.json', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1553,7 +1583,7 @@ class GServer private static function detectGNUSocial(string $url, array $serverdata) { // Test for GNU Social - $curlResult = DI::httpClient()->get($url . '/api/gnusocial/version.json'); + $curlResult = DI::httpClient()->get($url . '/api/gnusocial/version.json', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { $serverdata['platform'] = 'gnusocial'; @@ -1571,7 +1601,7 @@ class GServer } // Test for Statusnet - $curlResult = DI::httpClient()->get($url . '/api/statusnet/version.json'); + $curlResult = DI::httpClient()->get($url . '/api/statusnet/version.json', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { @@ -1607,9 +1637,11 @@ class GServer */ private static function detectFriendica(string $url, array $serverdata) { - $curlResult = DI::httpClient()->get($url . '/friendica/json'); + // There is a bug in some versions of Friendica that will return an ActivityStream actor when the content type "application/json" is requested. + // Because of this me must not use ACCEPT_JSON here. + $curlResult = DI::httpClient()->get($url . '/friendica/json', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_DEFAULT]); if (!$curlResult->isSuccess()) { - $curlResult = DI::httpClient()->get($url . '/friendika/json'); + $curlResult = DI::httpClient()->get($url . '/friendika/json', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_DEFAULT]); $friendika = true; $platform = 'Friendika'; } else { @@ -1931,8 +1963,7 @@ class GServer if (!empty($accesstoken)) { $api = 'https://instances.social/api/1.0/instances/list?count=0'; - $curlResult = DI::httpClient()->get($api, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $accesstoken]]]); - + $curlResult = DI::httpClient()->get($api, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $accesstoken]], HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if ($curlResult->isSuccess()) { $servers = json_decode($curlResult->getBody(), true); diff --git a/src/Network/HTTPClient/Client/HttpClient.php b/src/Network/HTTPClient/Client/HttpClient.php index ae34defc41..1f68397624 100644 --- a/src/Network/HTTPClient/Client/HttpClient.php +++ b/src/Network/HTTPClient/Client/HttpClient.php @@ -44,7 +44,17 @@ use Psr\Log\LoggerInterface; class HttpClient implements ICanSendHttpRequests { /** @var string Default value for "Accept" header */ - const DEFAULT_ACCEPT = '*/*'; + const ACCEPT_DEFAULT = '*/*'; + const ACCEPT_ATOM_XML = 'application/atom+xml,text/xml;q=0.9,*/*;q=0.8'; + const ACCEPT_FEED_XML = 'application/atom+xml,application/rss+xml;q=0.9,application/rdf+xml;q=0.8,text/xml;q=0.7,*/*;q=0.6'; + const ACCEPT_HTML = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'; + const ACCEPT_IMAGE = 'image/png,image/jpeg,image/gif,image/*;q=0.9,*/*;q=0.8'; + const ACCEPT_JRD_JSON = 'application/jrd+json,application/json;q=0.9'; + const ACCEPT_JSON = 'application/json,*/*;q=0.9'; + const ACCEPT_JSON_AS = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"'; + const ACCEPT_RSS_XML = 'application/rss+xml,text/xml;q=0.9,*/*;q=0.8'; + const ACCEPT_VIDEO = 'video/mp4,video/*;q=0.9,*/*;q=0.8'; + const ACCEPT_XRD_XML = 'application/xrd+xml,text/xml;q=0.9,*/*;q=0.8'; /** @var LoggerInterface */ private $logger; @@ -144,7 +154,7 @@ class HttpClient implements ICanSendHttpRequests }; if (empty($conf[HttpClientOptions::HEADERS]['Accept'])) { - $conf[HttpClientOptions::HEADERS]['Accept'] = static::DEFAULT_ACCEPT; + $conf[HttpClientOptions::HEADERS]['Accept'] = static::ACCEPT_DEFAULT; } try { diff --git a/src/Network/Probe.php b/src/Network/Probe.php index a6987d0a64..810885323d 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -34,6 +34,7 @@ use Friendica\Model\Contact; use Friendica\Model\GServer; use Friendica\Model\Profile; use Friendica\Model\User; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Protocol\ActivityNamespace; use Friendica\Protocol\ActivityPub; @@ -530,7 +531,7 @@ class Probe $addr = $nick . '@' . $host; } - $webfinger = self::getWebfinger($parts['scheme'] . '://' . $host . self::WEBFINGER, 'application/jrd+json', $uri, $addr); + $webfinger = self::getWebfinger($parts['scheme'] . '://' . $host . self::WEBFINGER, HttpClient::ACCEPT_JRD_JSON, $uri, $addr); if (empty($webfinger)) { $lrdd = self::hostMeta($host); } @@ -544,7 +545,7 @@ class Probe $addr = $nick . '@' . $host; } - $webfinger = self::getWebfinger($parts['scheme'] . '://' . $host . self::WEBFINGER, 'application/jrd+json', $uri, $addr); + $webfinger = self::getWebfinger($parts['scheme'] . '://' . $host . self::WEBFINGER, HttpClient::ACCEPT_JRD_JSON, $uri, $addr); if (empty($webfinger)) { $lrdd = self::hostMeta($host); } @@ -562,13 +563,13 @@ class Probe $nick = substr($uri, 0, strpos($uri, '@')); $addr = $uri; - $webfinger = self::getWebfinger('https://' . $host . self::WEBFINGER, 'application/jrd+json', $uri, $addr); + $webfinger = self::getWebfinger('https://' . $host . self::WEBFINGER, HttpClient::ACCEPT_JRD_JSON, $uri, $addr); if (self::$istimeout) { return []; } if (empty($webfinger)) { - $webfinger = self::getWebfinger('http://' . $host . self::WEBFINGER, 'application/jrd+json', $uri, $addr); + $webfinger = self::getWebfinger('http://' . $host . self::WEBFINGER, HttpClient::ACCEPT_JRD_JSON, $uri, $addr); if (self::$istimeout) { return []; } diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index 68f6cb1dd7..faad39dde5 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -28,6 +28,7 @@ use Friendica\DI; use Friendica\Model\APContact; use Friendica\Model\Contact; use Friendica\Model\User; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPClient\Client\HttpClientOptions; /** @@ -415,7 +416,7 @@ class HTTPSignature * @return \Friendica\Network\HTTPClient\Capability\ICanHandleHttpResponses CurlResult * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function fetchRaw($request, $uid = 0, $opts = ['accept_content' => ['application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"']]) + public static function fetchRaw($request, $uid = 0, $opts = ['accept_content' => [HttpClient::ACCEPT_JSON_AS]]) { $header = []; diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index fbc00e1d58..40295f0c9d 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -30,6 +30,7 @@ use Friendica\Model\Contact; use Friendica\Model\Item; use Friendica\Model\Post; use Friendica\Model\User; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; @@ -154,7 +155,7 @@ class OnePoll } $cookiejar = tempnam(System::getTempPath(), 'cookiejar-onepoll-'); - $curlResult = DI::httpClient()->get($contact['poll'], [HttpClientOptions::COOKIEJAR => $cookiejar]); + $curlResult = DI::httpClient()->get($contact['poll'], [HttpClientOptions::COOKIEJAR => $cookiejar], [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_FEED_XML]); unlink($cookiejar); if ($curlResult->isTimeout()) { @@ -173,7 +174,7 @@ class OnePoll return false; } - Logger::notice('Consume feed of contact', ['id' => $contact['id'], 'url' => $contact['poll']]); + Logger::notice('Consume feed of contact', ['id' => $contact['id'], 'url' => $contact['poll'], 'Content-Type' => $curlResult->getHeader('Content-Type')]); return !empty(Feed::import($xml, $importer, $contact)); }