New table for attached media files

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2020.12-dev (Red Hot Poker)
--- DB_UPDATE_VERSION 1370
+-- DB_UPDATE_VERSION 1372
 -- ------------------------------------------
 
 
@@ -775,6 +775,7 @@ CREATE TABLE IF NOT EXISTS `item-content` (
 	`title` varchar(255) NOT NULL DEFAULT '' COMMENT 'item title',
 	`content-warning` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`body` mediumtext COMMENT 'item body content',
+	`raw-body` mediumtext COMMENT 'Body without embedded media links',
 	`location` varchar(255) NOT NULL DEFAULT '' COMMENT 'text location where this item originated',
 	`coord` varchar(255) NOT NULL DEFAULT '' COMMENT 'longitude/latitude pair representing location where this item originated',
 	`language` text COMMENT 'Language information about this post',
@@ -1064,6 +1065,27 @@ CREATE TABLE IF NOT EXISTS `post-delivery-data` (
 	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items';
 
+--
+-- TABLE post-media
+--
+CREATE TABLE IF NOT EXISTS `post-media` (
+	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
+	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
+	`url` varbinary(511) NOT NULL COMMENT 'Media URL',
+	`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Media type',
+	`mimetype` varchar(60) COMMENT '',
+	`height` smallint unsigned COMMENT 'Height of the media',
+	`width` smallint unsigned COMMENT 'Width of the media',
+	`size` int unsigned COMMENT 'Media size',
+	`preview` varbinary(255) COMMENT 'Preview URL',
+	`preview-height` smallint unsigned COMMENT 'Height of the preview picture',
+	`preview-width` smallint unsigned COMMENT 'Width of the preview picture',
+	`description` text COMMENT '',
+	 PRIMARY KEY(`id`),
+	 UNIQUE INDEX `uri-id-url` (`uri-id`,`url`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Attached media';
+
 --
 -- TABLE post-tag
 --
@@ -1390,7 +1412,7 @@ CREATE TABLE IF NOT EXISTS `workerqueue` (
 	 PRIMARY KEY(`id`),
 	 INDEX `done_parameter` (`done`,`parameter`(64)),
 	 INDEX `done_executed` (`done`,`executed`),
-	 INDEX `done_priority_created` (`done`,`priority`,`created`),
+	 INDEX `done_priority_retrial_created` (`done`,`priority`,`retrial`,`created`),
 	 INDEX `done_priority_next_try` (`done`,`priority`,`next_try`),
 	 INDEX `done_pid_next_try` (`done`,`pid`,`next_try`),
 	 INDEX `done_pid_retrial` (`done`,`pid`,`retrial`),

src/Model/Item.php

@@ -104,7 +104,7 @@ class Item
 			'object-type', 'object', 'target-type', 'target', 'plink'];
 
 	// Field list for "item-content" table that is not present in the "item" table
-	const CONTENT_FIELDLIST = ['language'];
+	const CONTENT_FIELDLIST = ['language', 'raw-body'];
 
 	// All fields in the item table
 	const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent',
@@ -1678,6 +1678,7 @@ class Item
 		$item['deny_gid']      = trim($item['deny_gid'] ?? '');
 		$item['private']       = intval($item['private'] ?? self::PUBLIC);
 		$item['body']          = trim($item['body'] ?? '');
+		$item['raw-body']      = trim($item['raw-body'] ?? $item['body']);
 		$item['attach']        = trim($item['attach'] ?? '');
 		$item['app']           = trim($item['app'] ?? '');
 		$item['origin']        = intval($item['origin'] ?? 0);
@@ -1816,6 +1817,10 @@ class Item
 			self::setOwnerforResharedItem($item);
 		}
 
+		// Remove all media attachments from the body and store them in the post-media table
+		$item['raw-body'] = Post\Media::addAttachmentsFromBody($item['uri-id'], $item['raw-body']);
+		$item['raw-body'] = self::setHashtags($item['raw-body']);
+
 		// Check for hashtags in the body and repair or add hashtag links
 		$item['body'] = self::setHashtags($item['body']);
 

src/Model/Post/Media.php

@@ -0,0 +1,172 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use Friendica\Core\Logger;
+use Friendica\Core\System;
+use Friendica\Database\DBA;
+use Friendica\Util\Images;
+
+/**
+ * Class Media
+ *
+ * This Model class handles media interactions.
+ * This tables stores medias (images, videos, audio files) related to posts.
+ */
+class Media
+{
+    const UNKNOWN = 0;
+    const IMAGE   = 1;
+    const VIDEO   = 2;
+	const AUDIO   = 3;
+	const TORRENT = 16;
+
+	/**
+	 * Insert a post-media record
+	 *
+	 * @param array $media
+	 * @return void
+	 */
+	public static function insert(array $media)
+	{
+		if (empty($media['url']) || empty($media['uri-id'])) {
+			return;
+		}
+
+		if (DBA::exists('post-media', ['uri-id' => $media['uri-id'], 'url' => $media['url']])) {
+			Logger::info('Media already exists', ['uri-id' => $media['uri-id'], 'url' => $media['url'], 'callstack' => System::callstack()]);
+			return;
+		}
+
+		$fields = ['type', 'mimetype', 'height', 'width', 'size', 'preview', 'preview-height', 'preview-width', 'description'];
+		foreach ($fields as $field) {
+			if (empty($media[$field])) {
+				unset($media[$field]);
+			}
+		}
+
+		if ($media['type'] == self::IMAGE) {
+			$imagedata = Images::getInfoFromURLCached($media['url']);
+			if (!empty($imagedata)) {
+				$media['mimetype'] = $imagedata['mime'];
+				$media['size'] = $imagedata['size'];
+				$media['width'] = $imagedata[0];
+				$media['height'] = $imagedata[1];
+			}
+			if (!empty($media['preview'])) {
+				$imagedata = Images::getInfoFromURLCached($media['preview']);
+				if (!empty($imagedata)) {
+					$media['preview-width'] = $imagedata[0];
+					$media['preview-height'] = $imagedata[1];
+				}
+			}
+		}
+
+		$result = DBA::insert('post-media', $media, true);
+		Logger::info('Stored media', ['result' => $result, 'media' => $media, 'callstack' => System::callstack()]);
+	}
+
+	/**
+	 * Tests for path patterns that are usef for picture links in Friendica
+	 *
+	 * @param string $page    Link to the image page
+	 * @param string $preview Preview picture
+	 * @return boolean
+	 */
+	private static function isPictureLink(string $page, string $preview)
+	{
+		return preg_match('#/photos/.*/image/#ism', $page) && preg_match('#/photo/.*-1\.#ism', $preview);
+	}
+
+	/**
+	 * Add media links and remove them from the body
+	 *
+	 * @param integer $uriid
+	 * @param string $body
+	 * @return string Body without media links
+	 */
+	public static function addAttachmentsFromBody(int $uriid, string $body)
+	{
+		// Simplify image codes
+		$body = preg_replace("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", '[img]$3[/img]', $body);
+
+		$attachments = [];
+		if (preg_match_all("#\[url=([^\]]+?)\]\s*\[img=([^\[\]]*)\]([^\[\]]*)\[\/img\]\s*\[/url\]#ism", $body, $pictures, PREG_SET_ORDER)) {
+			foreach ($pictures as $picture) {
+				if (!self::isPictureLink($picture[1], $picture[2])) {
+					continue;
+				}
+				$body = str_replace($picture[0], '', $body);
+				$image = str_replace('-1.', '-0.', $picture[2]);
+				$attachments[] = ['uri-id' => $uriid, 'type' => self::IMAGE, 'url' => $image,
+					'preview' => $picture[2], 'description' => $picture[3]];
+			}
+		}
+
+		if (preg_match_all("/\[img=([^\[\]]*)\]([^\[\]]*)\[\/img\]/Usi", $body, $pictures, PREG_SET_ORDER)) {
+			foreach ($pictures as $picture) {
+				$body = str_replace($picture[0], '', $body);
+				$attachments[] = ['uri-id' => $uriid, 'type' => self::IMAGE, 'url' => $picture[1], 'description' => $picture[2]];
+			}
+		}
+
+		if (preg_match_all("#\[url=([^\]]+?)\]\s*\[img\]([^\[]+?)\[/img\]\s*\[/url\]#ism", $body, $pictures, PREG_SET_ORDER)) {
+			foreach ($pictures as $picture) {
+				if (!self::isPictureLink($picture[1], $picture[2])) {
+					continue;
+				}
+				$body = str_replace($picture[0], '', $body);
+				$image = str_replace('-1.', '-0.', $picture[2]);
+				$attachments[] = ['uri-id' => $uriid, 'type' => self::IMAGE, 'url' => $image,
+					'preview' => $picture[2], 'description' => null];
+			}
+		}
+
+		if (preg_match_all("/\[img\]([^\[\]]*)\[\/img\]/ism", $body, $pictures, PREG_SET_ORDER)) {
+			foreach ($pictures as $picture) {
+				$body = str_replace($picture[0], '', $body);
+				$attachments[] = ['uri-id' => $uriid, 'type' => self::IMAGE, 'url' => $picture[1]];
+			}
+		}
+
+		/// @todo audio + video
+		if (preg_match_all("/\[audio\]([^\[\]]*)\[\/audio\]/ism", $body, $audios, PREG_SET_ORDER)) {
+			foreach ($audios as $audio) {
+				$body = str_replace($audio[0], '', $body);
+				$attachments[] = ['uri-id' => $uriid, 'type' => self::AUDIO, 'url' => $audio[1]];
+			}
+		}
+
+		if (preg_match_all("/\[video\]([^\[\]]*)\[\/video\]/ism", $body, $videos, PREG_SET_ORDER)) {
+			foreach ($videos as $video) {
+				$body = str_replace($video[0], '', $body);
+				$attachments[] = ['uri-id' => $uriid, 'type' => self::VIDEO, 'url' => $video[1]];
+			}
+		}
+
+		foreach ($attachments as $attachment) {
+			self::insert($attachment);
+		}
+
+		return trim($body);
+	}
+}

src/Protocol/ActivityPub/Processor.php

@@ -37,6 +37,7 @@ use Friendica\Model\ItemURI;
 use Friendica\Model\Mail;
 use Friendica\Model\Tag;
 use Friendica\Model\User;
+use Friendica\Model\Post;
 use Friendica\Protocol\Activity;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Protocol\Relay;
@@ -81,6 +82,45 @@ class Processor
 		return $body;
 	}
 
+	/**
+	 * Store attached media files in the post-media table
+	 *
+	 * @param int $uriid
+	 * @param array $attachment
+	 * @return void
+	 */
+	private static function storeAttachment(int $uriid, array $attachment)
+	{
+		if (empty($attachment['url'])) {
+			return;
+		}
+
+		$data = ['uri-id' => $uriid];
+
+		$filetype = strtolower(substr($attachment['mediaType'], 0, strpos($attachment['mediaType'], '/')));
+		if ($filetype == 'image') {
+			$data['type'] = Post\Media::IMAGE;
+		} elseif ($filetype == 'video') {
+			$data['type'] = Post\Media::VIDEO;
+		} elseif ($filetype == 'audio') {
+			$data['type'] = Post\Media::AUDIO;
+		} elseif (in_array($attachment['mediaType'], ['application/x-bittorrent', 'application/x-bittorrent;x-scheme-handler/magnet'])) {
+			$data['type'] = Post\Media::TORRENT;
+		} else {
+			Logger::info('Unknown type', ['attachment' => $attachment]);
+			return;
+		}
+
+		$data['url'] = $attachment['url'];
+		$data['mimetype'] = $attachment['mediaType'];
+		$data['height'] = $attachment['height'] ?? null;
+		$data['size'] = $attachment['size'] ?? null;
+		$data['preview'] = $attachment['image'] ?? null;
+		$data['description'] = $attachment['name'] ?? null;
+
+		Post\Media::insert($data);
+	}
+
 	/**
 	 * Add attachment data to the item array
 	 *
@@ -95,6 +135,8 @@ class Processor
 			return $item;
 		}
 
+		$item['attach'] = '';
+
 		foreach ($activity['attachments'] as $attach) {
 			switch ($attach['type']) {
 				case 'link':
@@ -110,6 +152,8 @@ class Processor
 					$item['body'] = PageInfo::appendDataToBody($item['body'], $data);
 					break;
 				default:
+					self::storeAttachment($item['uri-id'], $attach);
+
 					$filetype = strtolower(substr($attach['mediaType'], 0, strpos($attach['mediaType'], '/')));
 					if ($filetype == 'image') {
 						if (!empty($activity['source']) && strpos($activity['source'], $attach['url'])) {
@@ -146,13 +190,13 @@ class Processor
 
 						$item['body'] .= "\n[video]" . $attach['url'] . '[/video]';
 					} else {
-						if (!empty($item["attach"])) {
+						if (!empty($item['attach'])) {
-							$item["attach"] .= ',';
+							$item['attach'] .= ',';
 						} else {
-							$item["attach"] = '';
+							$item['attach'] = '';
 						}
 
-						$item["attach"] .= '[attach]href="' . $attach['url'] . '" length="' . ($attach['length'] ?? '0') . '" type="' . $attach['mediaType'] . '" title="' . ($attach['name'] ?? '') . '"[/attach]';
+						$item['attach'] .= '[attach]href="' . $attach['url'] . '" length="' . ($attach['length'] ?? '0') . '" type="' . $attach['mediaType'] . '" title="' . ($attach['name'] ?? '') . '"[/attach]';
 					}
 			}
 		}
@@ -180,6 +224,9 @@ class Processor
 		$item['edited'] = DateTimeFormat::utc($activity['updated']);
 
 		$item = self::processContent($activity, $item);
+
+		$item = self::constructAttachList($activity, $item);
+
 		if (empty($item)) {
 			return;
 		}
@@ -403,9 +450,6 @@ class Processor
 	{
 		$item['title'] = HTML::toBBCode($activity['name']);
 
-		if (!empty($activity['source'])) {
-			$item['body'] = $activity['source'];
-		} else {
 		$content = HTML::toBBCode($activity['content']);
 
 		if (!empty($activity['emojis'])) {
@@ -414,6 +458,10 @@ class Processor
 
 		$content = self::convertMentions($content);
 
+		if (!empty($activity['source'])) {
+			$item['body'] = $activity['source'];
+			$item['raw-body'] = $content;
+		} else {
 			if (empty($activity['directmessage']) && ($item['thr-parent'] != $item['uri']) && ($item['gravity'] == GRAVITY_COMMENT)) {
 				$item_private = !in_array(0, $activity['item_receiver']);
 				$parent = Item::selectFirst(['id', 'uri-id', 'private', 'author-link', 'alias'], ['uri' => $item['thr-parent']]);
@@ -429,7 +477,7 @@ class Processor
 				$content = self::removeImplicitMentionsFromBody($content, $parent);
 			}
 			$item['content-warning'] = HTML::toBBCode($activity['summary']);
-			$item['body'] = $content;
+			$item['raw-body'] = $item['body'] = $content;
 		}
 
 		self::storeFromBody($item);

src/Protocol/ActivityPub/Receiver.php

@@ -1231,24 +1231,36 @@ class Receiver
 			$filetype = strtolower(substr($mediatype, 0, strpos($mediatype, '/')));
 
 			if ($filetype == 'audio') {
-				$attachments[$filetype] = ['type' => $mediatype, 'url' => $href];
+				$attachments[$filetype] = ['type' => $mediatype, 'url' => $href, 'height' => null, 'size' => null];
 			} elseif ($filetype == 'video') {
 				$height = (int)JsonLD::fetchElement($url, 'as:height', '@value');
+				$size = (int)JsonLD::fetchElement($url, 'pt:size', '@value');
 
-				// We save bandwidth by using a moderate height
+				// We save bandwidth by using a moderate height (alt least 480 pixel height)
 				// Peertube normally uses these heights: 240, 360, 480, 720, 1080
 				if (!empty($attachments[$filetype]['height']) &&
-					(($height > 480) || $height < $attachments[$filetype]['height'])) {
+					($height > $attachments[$filetype]['height']) && ($attachments[$filetype]['height'] >= 480)) {
 					continue;
 				}
 
-				$attachments[$filetype] = ['type' => $mediatype, 'url' => $href, 'height' => $height];
+				$attachments[$filetype] = ['type' => $mediatype, 'url' => $href, 'height' => $height, 'size' => $size];
+			} elseif (in_array($mediatype, ['application/x-bittorrent', 'application/x-bittorrent;x-scheme-handler/magnet'])) {
+				$height = (int)JsonLD::fetchElement($url, 'as:height', '@value');
+
+				// For Torrent links we always store the highest resolution
+				if (!empty($attachments[$mediatype]['height']) && ($height < $attachments[$mediatype]['height'])) {
+					continue;
+				}
+
+				$attachments[$mediatype] = ['type' => $mediatype, 'url' => $href, 'height' => $height, 'size' => null];
 			}
 		}
 
 		foreach ($attachments as $type => $attachment) {
 			$object_data['attachments'][] = ['type' => $type,
 				'mediaType' => $attachment['type'],
+				'height' => $attachment['height'],
+				'size' => $attachment['size'],
 				'name' => '',
 				'url' => $attachment['url']];
 		}

src/Protocol/Diaspora.php

@@ -2810,6 +2810,26 @@ class Diaspora
 		return Relay::isSolicitedPost($tags, $body, $contact['id'], $url, Protocol::DIASPORA);
 	}
 
+	/**
+	 * Store an attached photo in the post-media table
+	 *
+	 * @param int $uriid
+	 * @param object $photo
+	 * @return void
+	 */
+	private static function storePhoto(int $uriid, $photo)
+	{
+		$data = [];
+		$data['uri-id'] = $uriid;
+		$data['type'] = Post\Media::IMAGE;
+		$data['url'] = XML::unescape($photo->remote_photo_path) . XML::unescape($photo->remote_photo_name);
+		$data['height'] = (int)XML::unescape($photo->height ?? 0);
+		$data['width'] = (int)XML::unescape($photo->width ?? 0);
+		$data['description'] = XML::unescape($photo->text ?? '');
+
+		Post\Media::insert($data);
+	}
+
 	/**
 	 * Receives status messages
 	 *
@@ -2847,13 +2867,18 @@ class Diaspora
 			}
 		}
 
-		$body = Markdown::toBBCode($text);
+		$raw_body = $body = Markdown::toBBCode($text);
 
 		$datarray = [];
 
+		$datarray["guid"] = $guid;
+		$datarray["uri"] = $datarray["parent-uri"] = self::getUriFromGuid($author, $guid);
+		$datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]);
+
 		// Attach embedded pictures to the body
 		if ($data->photo) {
 			foreach ($data->photo as $photo) {
+				self::storePhoto($datarray['uri-id'], $photo);
 				$body = "[img]".XML::unescape($photo->remote_photo_path).
 					XML::unescape($photo->remote_photo_name)."[/img]\n".$body;
 			}
@@ -2887,10 +2912,6 @@ class Diaspora
 		$datarray["owner-link"] = $datarray["author-link"];
 		$datarray["owner-id"] = $datarray["author-id"];
 
-		$datarray["guid"] = $guid;
-		$datarray["uri"] = $datarray["parent-uri"] = self::getUriFromGuid($author, $guid);
-		$datarray['uri-id'] = ItemURI::insert(['uri' => $datarray['uri'], 'guid' => $datarray['guid']]);
-
 		$datarray["verb"] = Activity::POST;
 		$datarray["gravity"] = GRAVITY_PARENT;
 
@@ -2904,6 +2925,7 @@ class Diaspora
 		}
 
 		$datarray["body"] = self::replacePeopleGuid($body, $contact["url"]);
+		$datarray["raw-body"] = self::replacePeopleGuid($raw_body, $contact["url"]);
 
 		self::storeMentions($datarray['uri-id'], $text);
 		Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]);

static/dbstructure.config.php

@@ -54,7 +54,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1371);
+	define('DB_UPDATE_VERSION', 1372);
 }
 
 return [
@@ -843,6 +843,7 @@ return [
 			"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
 			"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
 			"body" => ["type" => "mediumtext", "comment" => "item body content"],
+			"raw-body" => ["type" => "mediumtext", "comment" => "Body without embedded media links"],
 			"location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"],
 			"coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"],
 			"language" => ["type" => "text", "comment" => "Language information about this post"],
@@ -1133,6 +1134,27 @@ return [
 			"PRIMARY" => ["uri-id"],
 		]
 	],
+	"post-media" => [
+		"comment" => "Attached media",
+		"fields" => [
+			"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"],
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+			"url" => ["type" => "varbinary(511)", "not null" => "1", "comment" => "Media URL"],
+			"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Media type"],
+			"mimetype" => ["type" => "varchar(60)", "comment" => ""],
+			"height" => ["type" => "smallint unsigned", "comment" => "Height of the media"],
+			"width" => ["type" => "smallint unsigned", "comment" => "Width of the media"],
+			"size" => ["type" => "int unsigned", "comment" => "Media size"],
+			"preview" => ["type" => "varbinary(255)", "comment" => "Preview URL"],
+			"preview-height" => ["type" => "smallint unsigned", "comment" => "Height of the preview picture"],
+			"preview-width" => ["type" => "smallint unsigned", "comment" => "Width of the preview picture"],
+			"description" => ["type" => "text", "comment" => ""],
+		],
+		"indexes" => [
+			"PRIMARY" => ["id"],
+			"uri-id-url" => ["UNIQUE", "uri-id", "url"],
+		]
+	],
 	"post-tag" => [
 		"comment" => "post relation to tags",
 		"fields" => [