Issue 13845: Support "sensitive" attribute

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2024.03-dev (Yellow Archangel)
--- DB_UPDATE_VERSION 1550
+-- DB_UPDATE_VERSION 1552
 -- ------------------------------------------
 
 
@@ -1283,6 +1283,7 @@ CREATE TABLE IF NOT EXISTS `post-content` (
 	`location` varchar(255) NOT NULL DEFAULT '' COMMENT 'text location where this item originated',
 	`coord` varchar(255) NOT NULL DEFAULT '' COMMENT 'longitude/latitude pair representing location where this item originated',
 	`language` text COMMENT 'Language information about this post',
+	`sensitive` boolean COMMENT 'If true, this post contains sensitive content',
 	`app` varchar(255) NOT NULL DEFAULT '' COMMENT 'application which generated this item',
 	`rendered-hash` varchar(32) NOT NULL DEFAULT '' COMMENT '',
 	`rendered-html` mediumtext COMMENT 'item.body converted to html',
@@ -2163,6 +2164,7 @@ CREATE VIEW `post-user-view` AS SELECT
 	`post-content`.`plink` AS `plink`,
 	`post-content`.`location` AS `location`,
 	`post-content`.`coord` AS `coord`,
+	`post-content`.`sensitive` AS `sensitive`,
 	`post-content`.`app` AS `app`,
 	`post-content`.`object-type` AS `object-type`,
 	`post-content`.`object` AS `object`,
@@ -2347,6 +2349,7 @@ CREATE VIEW `post-thread-user-view` AS SELECT
 	`post-content`.`plink` AS `plink`,
 	`post-content`.`location` AS `location`,
 	`post-content`.`coord` AS `coord`,
+	`post-content`.`sensitive` AS `sensitive`,
 	`post-content`.`app` AS `app`,
 	`post-content`.`object-type` AS `object-type`,
 	`post-content`.`object` AS `object`,
@@ -2517,6 +2520,7 @@ CREATE VIEW `post-view` AS SELECT
 	`post-content`.`plink` AS `plink`,
 	`post-content`.`location` AS `location`,
 	`post-content`.`coord` AS `coord`,
+	`post-content`.`sensitive` AS `sensitive`,
 	`post-content`.`app` AS `app`,
 	`post-content`.`object-type` AS `object-type`,
 	`post-content`.`object` AS `object`,
@@ -2663,6 +2667,7 @@ CREATE VIEW `post-thread-view` AS SELECT
 	`post-content`.`plink` AS `plink`,
 	`post-content`.`location` AS `location`,
 	`post-content`.`coord` AS `coord`,
+	`post-content`.`sensitive` AS `sensitive`,
 	`post-content`.`app` AS `app`,
 	`post-content`.`object-type` AS `object-type`,
 	`post-content`.`object` AS `object`,

doc/database/db_post-content.md

@@ -17,6 +17,7 @@ Fields
 | location        | text location where this item originated                                                                                  | varchar(255)   | NO   |     |         |       |
 | coord           | longitude/latitude pair representing location where this item originated                                                  | varchar(255)   | NO   |     |         |       |
 | language        | Language information about this post                                                                                      | text           | YES  |     | NULL    |       |
+| sensitive       | If true, this post contains sensitive content                                                                             | boolean        | YES  |     | NULL    |       |
 | app             | application which generated this item                                                                                     | varchar(255)   | NO   |     |         |       |
 | rendered-hash   |                                                                                                                           | varchar(32)    | NO   |     |         |       |
 | rendered-html   | item.body converted to html                                                                                               | mediumtext     | YES  |     | NULL    |       |

src/Factory/Api/Mastodon/Status.php

@@ -112,7 +112,7 @@ class Status extends BaseFactory
 	{
 		$fields = ['uri-id', 'uid', 'author-id', 'causer-id', 'author-uri-id', 'author-link', 'causer-uri-id', 'post-reason', 'starred', 'app', 'title', 'body', 'raw-body', 'content-warning', 'question-id',
 			'created', 'edited', 'commented', 'received', 'changed', 'network', 'thr-parent-id', 'parent-author-id', 'language', 'uri', 'plink', 'private', 'vid', 'gravity', 'featured', 'has-media', 'quote-uri-id',
-			'delivery_queue_count', 'delivery_queue_done','delivery_queue_failed', 'allow_cid', 'deny_cid', 'allow_gid', 'deny_gid'];
+			'delivery_queue_count', 'delivery_queue_done','delivery_queue_failed', 'allow_cid', 'deny_cid', 'allow_gid', 'deny_gid', 'sensitive'];
 		$item = Post::selectFirst($fields, ['uri-id' => $uriId, 'uid' => [0, $uid]], ['order' => ['uid' => true]]);
 		if (!$item) {
 			$mail = DBA::selectFirst('mail', ['id'], ['uri-id' => $uriId, 'uid' => $uid]);
@@ -217,7 +217,7 @@ class Status extends BaseFactory
 			$item['featured']
 		);
 
-		$sensitive   = $this->dba->exists('tag-view', ['uri-id' => $uriId, 'name' => 'nsfw', 'type' => TagModel::HASHTAG]);
+		$sensitive   = (bool)$item['sensitive'];
 		$application = new \Friendica\Object\Api\Mastodon\Application($item['app'] ?: ContactSelector::networkToName($item['network'], $item['author-link']));
 
 		$mentions    = $this->mstdnMentionFactory->createFromUriId($uriId)->getArrayCopy();

src/Model/Item.php

@@ -99,7 +99,7 @@ class Item
 		'uid', 'id', 'parent', 'guid', 'network', 'gravity',
 		'uri-id', 'uri', 'thr-parent-id', 'thr-parent', 'parent-uri-id', 'parent-uri', 'conversation',
 		'commented', 'created', 'edited', 'received', 'verb', 'object-type', 'postopts', 'plink',
-		'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language',
+		'wall', 'private', 'starred', 'origin', 'parent-origin', 'title', 'body', 'language', 'sensitive',
 		'content-warning', 'location', 'coord', 'app', 'rendered-hash', 'rendered-html', 'object',
 		'quote-uri', 'quote-uri-id', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'mention', 'global',
 		'author-id', 'author-link', 'author-alias', 'author-name', 'author-avatar', 'author-network', 'author-updated', 'author-gsid', 'author-baseurl', 'author-addr', 'author-uri-id',
@@ -1280,6 +1280,15 @@ class Item
 			}
 		}
 
+		// Store tags from the body if this hadn't been handled previously in the protocol classes
+		if (!Tag::existsForPost($item['uri-id'])) {
+			Tag::storeFromBody($item['uri-id'], $item['body']);
+		}
+
+		if (in_array($item['gravity'], [self::GRAVITY_PARENT, self::GRAVITY_COMMENT]) && (!isset($item['sensitive']) || is_null($item['sensitive']))) {
+			$item['sensitive'] = Tag::existsTagForPost($item['uri-id'], 'nsfw');
+		}
+
 		$item['language'] = self::getLanguage($item);
 
 		$inserted = Post::insert($item['uri-id'], $item);
@@ -1319,11 +1328,6 @@ class Item
 			Post\DeliveryData::insert($item['uri-id'], $delivery_data);
 		}
 
-		// Store tags from the body if this hadn't been handled previously in the protocol classes
-		if (!Tag::existsForPost($item['uri-id'])) {
-			Tag::storeFromBody($item['uri-id'], $item['body']);
-		}
-
 		$condition = ['uri-id' => $item['uri-id'], 'uid' => $item['uid']];
 		if (Post::exists($condition)) {
 			Logger::notice('Item is already inserted - aborting', $condition);
@@ -3467,7 +3471,7 @@ class Item
 		}
 
 		if (!empty($sharedSplitAttachments)) {
-			$s = self::addGallery($s, $sharedSplitAttachments['visual']);
+			$s = self::addGallery($s, $sharedSplitAttachments['visual'], (bool)$item['sensitive']);
 			$s = self::addVisualAttachments($sharedSplitAttachments['visual'], $shared_item, $s, true);
 			$s = self::addLinkAttachment($shared_uri_id ?: $item['uri-id'], $sharedSplitAttachments, $body, $s, true, $quote_shared_links);
 			$s = self::addNonVisualAttachments($sharedSplitAttachments['additional'], $item, $s, true);
@@ -3480,7 +3484,7 @@ class Item
 			$s = substr($s, 0, $pos);
 		}
 
-		$s = self::addGallery($s, $itemSplitAttachments['visual']);
+		$s = self::addGallery($s, $itemSplitAttachments['visual'], (bool)$item['sensitive']);
 		$s = self::addVisualAttachments($itemSplitAttachments['visual'], $item, $s, false);
 		$s = self::addLinkAttachment($item['uri-id'], $itemSplitAttachments, $body, $s, false, $shared_links);
 		$s = self::addNonVisualAttachments($itemSplitAttachments['additional'], $item, $s, false);
@@ -3516,9 +3520,10 @@ class Item
 	 *
 	 * @param string     $s
 	 * @param PostMedias $PostMedias
+	 * @param bool       $sensitive
 	 * @return string
 	 */
-	private static function addGallery(string $s, PostMedias $PostMedias): string
+	private static function addGallery(string $s, PostMedias $PostMedias, bool $sensitive): string
 	{
 		foreach ($PostMedias as $PostMedia) {
 			if (!$PostMedia->preview || ($PostMedia->type !== Post\Media::IMAGE)) {
@@ -3528,9 +3533,10 @@ class Item
 			if ($PostMedia->hasDimensions()) {
 				$pattern = '#<a href="' . preg_quote($PostMedia->url) . '">(.*?)"></a>#';
 
-				$s = preg_replace_callback($pattern, function () use ($PostMedia) {
+				$s = preg_replace_callback($pattern, function () use ($PostMedia, $sensitive) {
 					return Renderer::replaceMacros(Renderer::getMarkupTemplate('content/image/single_with_height_allocation.tpl'), [
 						'$image' => $PostMedia,
+						'$sensitive' => $sensitive,
 						'$allocated_height' => $PostMedia->getAllocatedHeight(),
 						'$allocated_max_width' => ($PostMedia->previewWidth ?? $PostMedia->width) . 'px',
 					]);

src/Model/Post/SearchIndex.php

@@ -53,7 +53,7 @@ class SearchIndex
 			'uri-id'     => $uri_id,
 			'owner-id'   => $item['owner-id'],
 			'media-type' => Engagement::getMediaType($uri_id),
-			'language'   => !empty($item['language']) ? (array_key_first(json_decode($item['language'], true)) ?? L10n::UNDETERMINED_LANGUAGE) : L10n::UNDETERMINED_LANGUAGE,
+			'language'   => substr(!empty($item['language']) ? (array_key_first(json_decode($item['language'], true)) ?? L10n::UNDETERMINED_LANGUAGE) : L10n::UNDETERMINED_LANGUAGE, 0, 2),
 			'searchtext' => Post\Engagement::getSearchTextForUriId($uri_id, $refresh),
 			'size'       => Engagement::getContentSize($item),
 			'created'    => $item['created'],

src/Model/Tag.php

@@ -378,6 +378,18 @@ class Tag
 		return DBA::exists('post-tag', ['uri-id' => $uriId, 'type' => [self::HASHTAG, self::MENTION, self::EXCLUSIVE_MENTION, self::IMPLICIT_MENTION]]);
 	}
 
+	/**
+	 * Check for a given hashtag on a given post
+	 *
+	 * @param integer $uriId
+	 * @param string $tag
+	 * @return boolean
+	 */
+	public static function existsTagForPost(int $uriId, string $tag): bool
+	{
+		return DBA::exists('tag-view', ['uri-id' => $uriId, 'type' => self::HASHTAG, 'name' => $tag]);
+	}
+
 	/**
 	 * Remove tag/mention
 	 *

src/Protocol/ActivityPub/Processor.php

@@ -466,6 +466,7 @@ class Processor
 		}
 
 		$item['uri'] = $activity['id'];
+		$item['sensitive'] = $activity['sensitive'];
 
 		if (empty($activity['published']) || empty($activity['updated'])) {
 			DI::logger()->notice('published or updated keys are empty for activity', ['activity' => $activity]);

src/Protocol/ActivityPub/Transmitter.php

@@ -1697,18 +1697,6 @@ class Transmitter
 		});
 	}
 
-	/**
-	 * Returns if the post contains sensitive content ("nsfw")
-	 *
-	 * @param integer $uri_id URI id
-	 * @return boolean Whether URI id was found
-	 * @throws \Exception
-	 */
-	private static function isSensitive(int $uri_id): bool
-	{
-		return DBA::exists('tag-view', ['uri-id' => $uri_id, 'name' => 'nsfw', 'type' => Tag::HASHTAG]);
-	}
-
 	/**
 	 * Creates event data
 	 *
@@ -1812,7 +1800,7 @@ class Transmitter
 		} else {
 			$data['attributedTo'] = $item['author-link'];
 		}
-		$data['sensitive'] = self::isSensitive($item['uri-id']);
+		$data['sensitive'] = (bool)$item['sensitive'];
 
 		if (!empty($item['conversation']) && ($item['conversation'] != './')) {
 			$data['conversation'] = $data['context'] = $item['conversation'];

static/dbstructure.config.php

@@ -56,7 +56,7 @@ use Friendica\Database\DBA;
 
 // This file is required several times during the test in DbaDefinition which justifies this condition
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1550);
+	define('DB_UPDATE_VERSION', 1552);
 }
 
 return [
@@ -1307,6 +1307,7 @@ return [
 			"location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"],
 			"coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"],
 			"language" => ["type" => "text", "comment" => "Language information about this post"],
+			"sensitive" => ["type" => "boolean", "comment" => "If true, this post contains sensitive content"],
 			"app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"],
 			"rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""],
 			"rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"],

static/dbview.config.php

@@ -196,6 +196,7 @@
 			"plink" => ["post-content", "plink"],
 			"location" => ["post-content", "location"],
 			"coord" => ["post-content", "coord"],
+			"sensitive" => ["post-content", "sensitive"],
 			"app" => ["post-content", "app"],
 			"object-type" => ["post-content", "object-type"],
 			"object" => ["post-content", "object"],
@@ -378,6 +379,7 @@
 			"plink" => ["post-content", "plink"],
 			"location" => ["post-content", "location"],
 			"coord" => ["post-content", "coord"],
+			"sensitive" => ["post-content", "sensitive"],
 			"app" => ["post-content", "app"],
 			"object-type" => ["post-content", "object-type"],
 			"object" => ["post-content", "object"],
@@ -546,6 +548,7 @@
 			"plink" => ["post-content", "plink"],
 			"location" => ["post-content", "location"],
 			"coord" => ["post-content", "coord"],
+			"sensitive" => ["post-content", "sensitive"],
 			"app" => ["post-content", "app"],
 			"object-type" => ["post-content", "object-type"],
 			"object" => ["post-content", "object"],
@@ -690,6 +693,7 @@
 			"plink" => ["post-content", "plink"],
 			"location" => ["post-content", "location"],
 			"coord" => ["post-content", "coord"],
+			"sensitive" => ["post-content", "sensitive"],
 			"app" => ["post-content", "app"],
 			"object-type" => ["post-content", "object-type"],
 			"object" => ["post-content", "object"],

update.php

@@ -1422,3 +1422,10 @@ function pre_update_1550()
 	}
 	return Update::SUCCESS;
 }
+
+function update_1552()
+{
+	DBA::e("UPDATE `post-content` INNER JOIN `post-tag` ON `post-tag`.`uri-id` = `post-content`.`uri-id` INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid` SET `sensitive` = ? WHERE `name` = ?", true, 'nsfw');
+
+	return Update::SUCCESS;
+}