From 0733ae6aae2bee02335613125bd0ac290b3a4859 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Mon, 18 Nov 2019 13:29:27 +0100
Subject: [PATCH] Improve the look of fetched feeds and the BBCode processing
 of attachments

---
 src/Content/Text/BBCode.php              | 97 ++++++++++++------------
 src/Protocol/ActivityPub/Transmitter.php |  5 +-
 src/Protocol/Feed.php                    | 18 ++++-
 3 files changed, 68 insertions(+), 52 deletions(-)

diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php
index cf3eb93714..c2b54821aa 100644
--- a/src/Content/Text/BBCode.php
+++ b/src/Content/Text/BBCode.php
@@ -388,21 +388,22 @@ class BBCode extends BaseObject
 	/**
 	 * Remove [attachment] BBCode and replaces it with a regular [url]
 	 *
-	 * @param string $body
+	 * @param string  $body
+	 * @param boolean $no_link_desc No link description
 	 *
 	 * @return string with replaced body
 	 */
-	public static function removeAttachment($body)
+	public static function removeAttachment($body, $no_link_desc = false)
 	{
 		return preg_replace_callback("/\[attachment (.*)\](.*?)\[\/attachment\]/ism",
-			function ($match) {
+			function ($match) use ($no_link_desc) {
 				$attach_data = self::getAttachmentData($match[0]);
 				if (empty($attach_data['url'])) {
 					return $match[0];
-				} elseif (empty($attach_data['title'])) {
-					return '[url]' . $attach_data['url'] . '[/url]';
+				} elseif (empty($attach_data['title']) || $no_link_desc) {
+					return '[url]' . $attach_data['url'] . "[/url]\n";
 				} else {
-					return '[url=' . $attach_data['url'] . ']' . $attach_data['title'] . '[/url]';
+					return '[url=' . $attach_data['url'] . ']' . $attach_data['title'] . "[/url]\n";
 				}
 		}, $body);
 	}
@@ -622,53 +623,44 @@ class BBCode extends BaseObject
 		}
 
 		$return = '';
-		if (in_array($simplehtml, [7, 9])) {
-			// Only add the link when it isn't already part of the body
-			if (substr_count($text, $data['url']) == 1) {
-				$return = self::convertUrlForActivityPub($data['url']);
+		try {
+			if ($tryoembed && OEmbed::isAllowedURL($data['url'])) {
+				$return = OEmbed::getHTML($data['url'], $data['title']);
+			} else {
+				throw new Exception('OEmbed is disabled for this attachment.');
 			}
-		} elseif (($simplehtml != 4) && ($simplehtml != 0)) {
-			$return = sprintf('<a href="%s" target="_blank">%s</a><br>', $data['url'], $data['title']);
-		} else {
-			try {
-				if ($tryoembed && OEmbed::isAllowedURL($data['url'])) {
-					$return = OEmbed::getHTML($data['url'], $data['title']);
+		} catch (Exception $e) {
+			$data['title'] = ($data['title'] ?? '') ?: $data['url'];
+
+			if ($simplehtml != 4) {
+				$return = sprintf('<div class="type-%s">', $data['type']);
+			}
+
+			if (!empty($data['title']) && !empty($data['url'])) {
+				if (!empty($data['image']) && empty($data['text']) && ($data['type'] == 'photo')) {
+					$return .= sprintf('<a href="%s" target="_blank"><img src="%s" alt="" title="%s" class="attachment-image" /></a>', $data['url'], self::proxyUrl($data['image'], $simplehtml), $data['title']);
 				} else {
-					throw new Exception('OEmbed is disabled for this attachment.');
-				}
-			} catch (Exception $e) {
-				$data['title'] = ($data['title'] ?? '') ?: $data['url'];
-
-				if ($simplehtml != 4) {
-					$return = sprintf('<div class="type-%s">', $data['type']);
-				}
-
-				if (!empty($data['title']) && !empty($data['url'])) {
-					if (!empty($data['image']) && empty($data['text']) && ($data['type'] == 'photo')) {
-						$return .= sprintf('<a href="%s" target="_blank"><img src="%s" alt="" title="%s" class="attachment-image" /></a>', $data['url'], self::proxyUrl($data['image'], $simplehtml), $data['title']);
-					} else {
-						if (!empty($data['image'])) {
-							$return .= sprintf('<a href="%s" target="_blank"><img src="%s" alt="" title="%s" class="attachment-image" /></a><br />', $data['url'], self::proxyUrl($data['image'], $simplehtml), $data['title']);
-						} elseif (!empty($data['preview'])) {
-							$return .= sprintf('<a href="%s" target="_blank"><img src="%s" alt="" title="%s" class="attachment-preview" /></a><br />', $data['url'], self::proxyUrl($data['preview'], $simplehtml), $data['title']);
-						}
-						$return .= sprintf('<h4><a href="%s">%s</a></h4>', $data['url'], $data['title']);
+					if (!empty($data['image'])) {
+						$return .= sprintf('<a href="%s" target="_blank"><img src="%s" alt="" title="%s" class="attachment-image" /></a><br />', $data['url'], self::proxyUrl($data['image'], $simplehtml), $data['title']);
+					} elseif (!empty($data['preview'])) {
+						$return .= sprintf('<a href="%s" target="_blank"><img src="%s" alt="" title="%s" class="attachment-preview" /></a><br />', $data['url'], self::proxyUrl($data['preview'], $simplehtml), $data['title']);
 					}
+					$return .= sprintf('<h4><a href="%s">%s</a></h4>', $data['url'], $data['title']);
 				}
+			}
 
-				if (!empty($data['description']) && $data['description'] != $data['title']) {
-					// Sanitize the HTML by converting it to BBCode
-					$bbcode = HTML::toBBCode($data['description']);
-					$return .= sprintf('<blockquote>%s</blockquote>', trim(self::convert($bbcode)));
-				}
+			if (!empty($data['description']) && $data['description'] != $data['title']) {
+				// Sanitize the HTML by converting it to BBCode
+				$bbcode = HTML::toBBCode($data['description']);
+				$return .= sprintf('<blockquote>%s</blockquote>', trim(self::convert($bbcode)));
+			}
 
-				if (!empty($data['url'])) {
-					$return .= sprintf('<sup><a href="%s">%s</a></sup>', $data['url'], parse_url($data['url'], PHP_URL_HOST));
-				}
+			if (!empty($data['url'])) {
+				$return .= sprintf('<sup><a href="%s">%s</a></sup>', $data['url'], parse_url($data['url'], PHP_URL_HOST));
+			}
 
-				if ($simplehtml != 4) {
-					$return .= '</div>';
-				}
+			if ($simplehtml != 4) {
+				$return .= '</div>';
 			}
 		}
 
@@ -1244,7 +1236,7 @@ class BBCode extends BaseObject
 	 * - 5: Unused
 	 * - 6: Unused
 	 * - 7: Used for dfrn, OStatus
-	 * - 8: Used for WP backlink text setting
+	 * - 8: Used for Twitter, WP backlink text setting
 	 * - 9: ActivityPub
 	 *
 	 * @param string $text
@@ -1380,8 +1372,15 @@ class BBCode extends BaseObject
 			} while ($oldtext != $text);
 		}
 
+		/// @todo Have a closer look at the different html modes
 		// Handle attached links or videos
-		$text = self::convertAttachment($text, $simple_html, $try_oembed);
+		if (in_array($simple_html, [9])) {
+			$text = self::removeAttachment($text);
+		} elseif (!in_array($simple_html, [0, 4])) {
+			$text = self::removeAttachment($text, true);
+		} else {
+			$text = self::convertAttachment($text, $simple_html, $try_oembed);
+		}
 
 		// leave open the posibility of [map=something]
 		// this is replaced in Item::prepareBody() which has knowledge of the item location
@@ -1771,7 +1770,7 @@ class BBCode extends BaseObject
 		 * - #[url=<anything>]<term>[/url]
 		 * - [url=<anything>]#<term>[/url]
 		 */
-		$text = preg_replace_callback("/(?:#\[url\=.*?\]|\[url\=.*?\]#)(.*?)\[\/url\]/ism", function($matches) {
+		$text = preg_replace_callback("/(?:#\[url\=[^\[\]]*\]|\[url\=[^\[\]]*\]#)(.*?)\[\/url\]/ism", function($matches) {
 			return '#<a href="'
 				. System::baseUrl()	. '/search?tag=' . rawurlencode($matches[1])
 				. '" class="tag" rel="tag" title="' . XML::escape($matches[1]) . '">'
diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php
index 8fff3fec21..99c3503b2d 100644
--- a/src/Protocol/ActivityPub/Transmitter.php
+++ b/src/Protocol/ActivityPub/Transmitter.php
@@ -1020,6 +1020,9 @@ class Transmitter
 	{
 		$attachments = [];
 
+		// Currently deactivated, since it creates side effects on Mastodon and Pleroma.
+		// It will be reactivated, once this cleared.
+		/*
 		$attach_data = BBCode::getAttachmentData($item['body']);
 		if (!empty($attach_data['url'])) {
 			$attachment = ['type' => 'Page',
@@ -1047,7 +1050,7 @@ class Transmitter
 
 			$attachments[] = $attachment;
 		}
-
+		*/
 		$arr = explode('[/attach],', $item['attach']);
 		if (count($arr)) {
 			foreach ($arr as $r) {
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index b7e7ce9201..0c9501387c 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -15,6 +15,7 @@ use Friendica\Core\System;
 use Friendica\Database\DBA;
 use Friendica\Model\Item;
 use Friendica\Protocol\ActivityNamespace;
+use Friendica\Util\ParseUrl;
 use Friendica\Util\Network;
 use Friendica\Util\XML;
 
@@ -399,7 +400,7 @@ class Feed {
 
 				// Remove a possible link to the item itself
 				$item["body"] = str_replace($item["plink"], '', $item["body"]);
-				$item["body"] = preg_replace('/\[url\=\](\w+.*?)\[\/url\]/i', '', $item["body"]);
+				$item["body"] = trim(preg_replace('/\[url\=\](\w+.*?)\[\/url\]/i', '', $item["body"]));
 
 				// Replace the content when the title is longer than the body
 				$replace = (strlen($item["title"]) > strlen($item["body"]));
@@ -415,8 +416,21 @@ class Feed {
 				}
 
 				if ($replace) {
-					$item["body"] = $item["title"];
+					$item["body"] = trim($item["title"]);
 				}
+
+			        $data = ParseUrl::getSiteinfoCached($item['plink'], true);
+				if (!empty($data['text']) && !empty($data['title']) && (mb_strlen($item['body']) < mb_strlen($data['text']))) {
+					// When the fetched page info text is longer than the body, we do try to enhance the body
+					if ((strpos($data['title'], $item['body']) === false) && (strpos($data['text'], $item['body']) === false)) {
+						// The body is not part of the fetched page info title or page info text. So we add the text to the body
+						$item['body'] .= "\n\n" . $data['text'];
+					} else {
+						// Else we replace the body with the page info text
+						$item['body'] = $data['text'];
+					}
+				}
+
 				// We always strip the title since it will be added in the page information
 				$item["title"] = "";
 				$item["body"] = $item["body"].add_page_info($item["plink"], false, $preview, ($contact["fetch_further_information"] == 2), $contact["ffi_keyword_blacklist"]);