From c2105f93d1edf4f9d1b566cc4f69005ae335de81 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Thu, 21 May 2020 00:23:14 -0400 Subject: [PATCH 1/4] Remove duplicate curl call in Network\Probe::getFeedLink - Add page body argument instead - Expand method scope to allow tests --- src/Network/Probe.php | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/Network/Probe.php b/src/Network/Probe.php index edcb9f4c34..65ce67d6bf 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -1756,25 +1756,20 @@ class Probe } /** - * Check page for feed link + * Checks HTML page for RSS feed link * - * @param string $url Page link - * - * @return string feed link + * @param string $url Page link + * @param string $body Page body string + * @return string|false Feed link or false if body was invalid HTML document */ - private static function getFeedLink($url) + public static function getFeedLink(string $url, string $body) { - $curlResult = Network::curl($url); - if (!$curlResult->isSuccess()) { - return false; - } - $doc = new DOMDocument(); - if (!@$doc->loadHTML($curlResult->getBody())) { + if (!@$doc->loadHTML($body)) { return false; } - $xpath = new DomXPath($doc); + $xpath = new DOMXPath($doc); //$feeds = $xpath->query("/html/head/link[@type='application/rss+xml']"); $feeds = $xpath->query("/html/head/link[@type='application/rss+xml' and @rel='alternate']"); @@ -1826,7 +1821,7 @@ class Probe return false; } - $feed_url = self::getFeedLink($url); + $feed_url = self::getFeedLink($url, $feed); if (!$feed_url) { return false; From 3ef987e4e16f802dfabf7a87ac13bf3c828b066e Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Thu, 21 May 2020 00:24:23 -0400 Subject: [PATCH 2/4] Add new Network\Probe::ensureAbsoluteLinkFromHTMLDoc method - Add supports for relative URL in href attributes to probe feed URL --- src/Network/Probe.php | 74 +++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 65ce67d6bf..1a7f578673 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -1771,30 +1771,72 @@ class Probe $xpath = new DOMXPath($doc); - //$feeds = $xpath->query("/html/head/link[@type='application/rss+xml']"); - $feeds = $xpath->query("/html/head/link[@type='application/rss+xml' and @rel='alternate']"); - if (!is_object($feeds)) { - return false; + $feedUrl = $xpath->evaluate('string(/html/head/link[@type="application/rss+xml" and @rel="alternate"]/@href)'); + + $feedUrl = $feedUrl ? self::ensureAbsoluteLinkFromHTMLDoc($feedUrl, $url, $xpath) : ''; + + return $feedUrl; + } + + /** + * Return an absolute URL in the context of a HTML document retrieved from the provided URL. + * + * Loosely based on RFC 1808 + * + * @see https://tools.ietf.org/html/rfc1808 + * + * @param string $href The potential relative href found in the HTML document + * @param string $base The HTML document URL + * @param DOMXPath $xpath The HTML document XPath + * @return string + */ + private static function ensureAbsoluteLinkFromHTMLDoc(string $href, string $base, DOMXPath $xpath) + { + if (filter_var($href, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED | FILTER_FLAG_HOST_REQUIRED)) { + return $href; } - if ($feeds->length == 0) { - return false; - } + $base = $xpath->evaluate('string(/html/head/base/@href)') ?: $base; - $feed_url = ""; + $baseParts = parse_url($base); - foreach ($feeds as $feed) { - $attr = []; - foreach ($feed->attributes as $attribute) { - $attr[$attribute->name] = trim($attribute->value); + // Naked domain case (scheme://basehost) + $path = $baseParts['path'] ?? '/'; + + // Remove the filename part of the path if it exists (/base/path/file) + $path = implode('/', array_slice(explode('/', $path), 0, -1)); + + $hrefParts = parse_url($href); + + // Root path case (/path) including relative scheme case (//host/path) + if ($hrefParts['path'] && $hrefParts['path'][0] == '/') { + $path = $hrefParts['path']; + } else { + $path = $path . '/' . $hrefParts['path']; + + // Resolve arbitrary relative path + // Lifted from https://www.php.net/manual/en/function.realpath.php#84012 + $parts = array_filter(explode('/', $path), 'strlen'); + $absolutes = array(); + foreach ($parts as $part) { + if ('.' == $part) continue; + if ('..' == $part) { + array_pop($absolutes); + } else { + $absolutes[] = $part; + } } - if (empty($feed_url) && !empty($attr['href'])) { - $feed_url = $attr["href"]; - } + $path = '/' . implode('/', $absolutes); } - return $feed_url; + // Relative scheme case (//host/path) + $baseParts['host'] = $hrefParts['host'] ?? $baseParts['host']; + $baseParts['path'] = $path; + unset($baseParts['query']); + unset($baseParts['fragment']); + + return Network::unparseURL($baseParts); } /** From 782218f781343969b797ec00b7e5f397cb6a10ed Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Thu, 21 May 2020 00:24:46 -0400 Subject: [PATCH 3/4] Add tests for Network\Probe::getFeedLink --- tests/src/Network/ProbeTest.php | 108 ++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/src/Network/ProbeTest.php diff --git a/tests/src/Network/ProbeTest.php b/tests/src/Network/ProbeTest.php new file mode 100644 index 0000000000..2dfc0e2ac8 --- /dev/null +++ b/tests/src/Network/ProbeTest.php @@ -0,0 +1,108 @@ + + + + Example Blog + + + + +

Hello World!

+ +'; + + const TEMPLATEBASE = ' + + + + Example Blog + + + + + +

Hello World!

+ +'; + + const EXPECTED = [ + 'https://example.org/path/to/blog/index.php' => [ + 'index.xml' => 'https://example.org/path/to/blog/index.xml', + './index.xml' => 'https://example.org/path/to/blog/index.xml', + '../index.xml' => 'https://example.org/path/to/index.xml', + '/index.xml' => 'https://example.org/index.xml', + '//example.com/index.xml' => 'https://example.com/index.xml', + ], + 'https://example.org/path/to/blog/' => [ + 'index.xml' => 'https://example.org/path/to/blog/index.xml', + './index.xml' => 'https://example.org/path/to/blog/index.xml', + '../index.xml' => 'https://example.org/path/to/index.xml', + '/index.xml' => 'https://example.org/index.xml', + '//example.com/index.xml' => 'https://example.com/index.xml', + ], + 'https://example.org/blog/' => [ + 'index.xml' => 'https://example.org/blog/index.xml', + './index.xml' => 'https://example.org/blog/index.xml', + '../index.xml' => 'https://example.org/index.xml', + '/index.xml' => 'https://example.org/index.xml', + '//example.com/index.xml' => 'https://example.com/index.xml', + ], + 'https://example.org' => [ + 'index.xml' => 'https://example.org/index.xml', + './index.xml' => 'https://example.org/index.xml', + '../index.xml' => 'https://example.org/index.xml', + '/index.xml' => 'https://example.org/index.xml', + '//example.com/index.xml' => 'https://example.com/index.xml', + ], + ]; + + private function replaceMacros($template, $vars) + { + foreach ($vars as $var => $value) { + $template = str_replace('{{' . $var . '}}', $value, $template); + } + + return $template; + } + + /** + * @small + */ + public function testGetFeedLinkNoBase() + { + foreach (self::EXPECTED as $url => $hrefs) { + foreach ($hrefs as $href => $expected) { + $body = $this->replaceMacros(self::TEMPLATENOBASE, ['$link' => $href]); + + $feedLink = Probe::getFeedLink($url, $body); + + $this->assertEquals($expected, $feedLink, 'base url = ' . $url . ' | href = ' . $href); + } + } + } + + /** + * @small + */ + public function testGetFeedLinkBase() + { + foreach (self::EXPECTED as $url => $hrefs) { + foreach ($hrefs as $href => $expected) { + $body = $this->replaceMacros(self::TEMPLATEBASE, ['$url' => $url, '$link' => $href]); + + $feedLink = Probe::getFeedLink('http://example.com', $body); + + $this->assertEquals($expected, $feedLink, 'base url = ' . $url . ' | href = ' . $href); + } + } + } +} From 04e8d5be2c59ad5ea41e9bb4a240911e7652e269 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Thu, 21 May 2020 02:27:33 -0400 Subject: [PATCH 4/4] Remove deprecated/implicit filter_var() flags in Network\Probe::ensureAbsoluteLinkFromHTMLDoc - Suppresses a test breaking notice message in PHP 7.4 --- src/Network/Probe.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 1a7f578673..d5b3e0be8a 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -1792,7 +1792,7 @@ class Probe */ private static function ensureAbsoluteLinkFromHTMLDoc(string $href, string $base, DOMXPath $xpath) { - if (filter_var($href, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED | FILTER_FLAG_HOST_REQUIRED)) { + if (filter_var($href, FILTER_VALIDATE_URL)) { return $href; }