Merge pull request #13869 from annando/engagement-searchindex

Unify searchindex table with engagement table

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2024.03-dev (Yellow Archangel)
--- DB_UPDATE_VERSION 1549
+-- DB_UPDATE_VERSION 1550
 -- ------------------------------------------
 
 
@@ -1346,7 +1346,7 @@ CREATE TABLE IF NOT EXISTS `post-engagement` (
 	`owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner',
 	`contact-type` tinyint NOT NULL DEFAULT 0 COMMENT 'Person, organisation, news, community, relay',
 	`media-type` tinyint NOT NULL DEFAULT 0 COMMENT 'Type of media in a bit array (1 = image, 2 = video, 4 = audio',
-	`language` varchar(128) COMMENT 'Language information about this post',
+	`language` char(2) COMMENT 'Language information about this post in the ISO 639-1 format',
 	`searchtext` mediumtext COMMENT 'Simplified text for the full text search',
 	`size` int unsigned COMMENT 'Body size',
 	`created` datetime COMMENT '',
@@ -1467,14 +1467,19 @@ CREATE TABLE IF NOT EXISTS `post-question-option` (
 --
 CREATE TABLE IF NOT EXISTS `post-searchindex` (
 	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
-	`network` char(4) COMMENT '',
+	`owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner',
-	`private` tinyint unsigned COMMENT '0=public, 1=private, 2=unlisted',
+	`media-type` tinyint NOT NULL DEFAULT 0 COMMENT 'Type of media in a bit array (1 = image, 2 = video, 4 = audio',
+	`language` char(2) COMMENT 'Language information about this post in the ISO 639-1 format',
 	`searchtext` mediumtext COMMENT 'Simplified text for the full text search',
+	`size` int unsigned COMMENT 'Body size',
 	`created` datetime COMMENT '',
+	`restricted` boolean NOT NULL DEFAULT '0' COMMENT 'If true, this post is either unlisted or not from a federated network',
 	 PRIMARY KEY(`uri-id`),
+	 INDEX `owner-id` (`owner-id`),
 	 INDEX `created` (`created`),
 	 FULLTEXT INDEX `searchtext` (`searchtext`),
-	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`owner-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts';
 
 --

doc/Channels.md

@@ -71,6 +71,10 @@ Alternatives are presented with "|".
     * source:service | source:news - The posts originates from a service account. This source type is often used to mark bot accounts.
     * source:application | source:relay - The post is created by an application. This is most likely unused in the fediverse for post creation.
 * tag - Use "tag:tagname" to search for a specific tag.
+* media - With this keyword you can search for attached media.
+    * media:image | media:photo | media:picture - The post contains an image
+    * media:video - The post contains a video
+    * media:audio - The post contains audio
 * network | net - Use this to include or exclude some networks from your channel.
     * network:apub | network:activitypub - ActivityPub (Used by the systems in the Fediverse)
     * network:dfrn | network:friendica - Legacy Friendica protocol. Nowayday Friendica mostly uses ActivityPub.

doc/database/db_post-engagement.md

@@ -12,7 +12,7 @@ Fields
 | owner-id     | Item owner                                                            | int unsigned       | NO   |     | 0       |       |
 | contact-type | Person, organisation, news, community, relay                          | tinyint            | NO   |     | 0       |       |
 | media-type   | Type of media in a bit array (1 = image, 2 = video, 4 = audio         | tinyint            | NO   |     | 0       |       |
-| language     | Language information about this post                                  | varchar(128)       | YES  |     | NULL    |       |
+| language     | Language information about this post in the ISO 639-1 format          | char(2)            | YES  |     | NULL    |       |
 | searchtext   | Simplified text for the full text search                              | mediumtext         | YES  |     | NULL    |       |
 | size         | Body size                                                             | int unsigned       | YES  |     | NULL    |       |
 | created      |                                                                       | datetime           | YES  |     | NULL    |       |

doc/database/db_post-searchindex.md

@@ -6,13 +6,16 @@ Content for all posts
 Fields
 ------
 
-| Field      | Description                                               | Type             | Null | Key | Default | Extra |
+| Field      | Description                                                           | Type         | Null | Key | Default | Extra |
-| ---------- | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- |
+| ---------- | --------------------------------------------------------------------- | ------------ | ---- | --- | ------- | ----- |
-| uri-id     | Id of the item-uri table entry that contains the item uri | int unsigned     | NO   | PRI | NULL    |       |
+| uri-id     | Id of the item-uri table entry that contains the item uri             | int unsigned | NO   | PRI | NULL    |       |
-| network    |                                                           | char(4)          | YES  |     | NULL    |       |
+| owner-id   | Item owner                                                            | int unsigned | NO   |     | 0       |       |
-| private    | 0=public, 1=private, 2=unlisted                           | tinyint unsigned | YES  |     | NULL    |       |
+| media-type | Type of media in a bit array (1 = image, 2 = video, 4 = audio         | tinyint      | NO   |     | 0       |       |
-| searchtext | Simplified text for the full text search                  | mediumtext       | YES  |     | NULL    |       |
+| language   | Language information about this post in the ISO 639-1 format          | char(2)      | YES  |     | NULL    |       |
-| created    |                                                           | datetime         | YES  |     | NULL    |       |
+| searchtext | Simplified text for the full text search                              | mediumtext   | YES  |     | NULL    |       |
+| size       | Body size                                                             | int unsigned | YES  |     | NULL    |       |
+| created    |                                                                       | datetime     | YES  |     | NULL    |       |
+| restricted | If true, this post is either unlisted or not from a federated network | boolean      | NO   |     | 0       |       |
 
 Indexes
 ------------
@@ -20,6 +23,7 @@ Indexes
 | Name       | Fields               |
 | ---------- | -------------------- |
 | PRIMARY    | uri-id               |
+| owner-id   | owner-id             |
 | created    | created              |
 | searchtext | FULLTEXT, searchtext |
 
@@ -29,5 +33,6 @@ Foreign Keys
 | Field | Target Table | Target Field |
 |-------|--------------|--------------|
 | uri-id | [item-uri](help/database/db_item-uri) | id |
+| owner-id | [contact](help/database/db_contact) | id |
 
 Return to [database documentation](help/database)

src/Database/PostUpdate.php

@@ -52,7 +52,7 @@ class PostUpdate
 	// Needed for the helper function to read from the legacy term table
 	const OBJECT_TYPE_POST  = 1;
 
-	const VERSION = 1547;
+	const VERSION = 1550;
 
 	/**
 	 * Calls the post update functions
@@ -128,7 +128,7 @@ class PostUpdate
 		if (!self::update1544()) {
 			return false;
 		}
-		if (!self::update1547()) {
+		if (!self::update1550()) {
 			return false;
 		}
 		return true;
@@ -1369,14 +1369,24 @@ class PostUpdate
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
-	private static function update1547()
+	private static function update1550()
 	{
 		// Was the script completed?
-		if (DI::keyValue()->get('post_update_version') >= 1547) {
+		if (DI::keyValue()->get('post_update_version') >= 1550) {
 			return true;
 		}
 
-		$id = (int)(DI::keyValue()->get('post_update_version_1547_id') ?? 0);
+		$engagements = DBA::select('post-engagement', ['uri-id'], ["`language` IS NULL"], ['order' => ['uri-id' => true], 'limit' => 1000]);
+		while ($engagement = DBA::fetch($engagements)) {
+			$item = Post::selectFirst([], ['uri-id' => $engagement['uri-id']]);
+			if (empty($item)) {
+				continue;
+			}
+			Post\Engagement::storeFromItem($item);
+		}
+		DBA::close($engagements);
+
+		$id = (int)(DI::keyValue()->get('post_update_version_1550_id') ?? 0);
 		if ($id == 0) {
 			$post = Post::selectFirstPost(['uri-id'], [], ['order' => ['uri-id' => true]]);
 			$id = (int)($post['uri-id'] ?? 0);
@@ -1393,7 +1403,7 @@ class PostUpdate
 			DBA::mergeConditions($condition, ["`created` > ?", $limit]);
 		}
 
-		$posts = Post::selectPosts(['uri-id', 'network', 'private', 'created'], $condition, ['order' => ['uri-id' => true], 'limit' => 1000]);
+		$posts = Post::selectPosts(['uri-id', 'created'], $condition, ['order' => ['uri-id' => true], 'limit' => 1000]);
 
 		if (DBA::errorNo() != 0) {
 			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
@@ -1402,17 +1412,17 @@ class PostUpdate
 
 		while ($post = Post::fetch($posts)) {
 			$id = $post['uri-id'];
-			Post\SearchIndex::insert($post['uri-id'], $post['network'], $post['private'], $post['created'], true);
+			Post\SearchIndex::insert($post['uri-id'], $post['created'], true);
 			++$rows;
 		}
 		DBA::close($posts);
 
-		DI::keyValue()->set('post_update_version_1547_id', $id);
+		DI::keyValue()->set('post_update_version_1550_id', $id);
 
 		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
 
 		if ($rows <= 100) {
-			DI::keyValue()->set('post_update_version', 1547);
+			DI::keyValue()->set('post_update_version', 1550);
 			Logger::info('Done');
 			return true;
 		}

src/Model/Item.php

@@ -1450,7 +1450,7 @@ class Item
 			$engagement_uri_id = Post\Engagement::storeFromItem($posted_item);
 
 			if (in_array($posted_item['gravity'], [self::GRAVITY_PARENT, self::GRAVITY_COMMENT])) {
-				Post\SearchIndex::insert($posted_item['uri-id'], $posted_item['network'], $posted_item['private'], $posted_item['created']);
+				Post\SearchIndex::insert($posted_item['uri-id'], $posted_item['created']);
 			} elseif ($posted_item['verb'] == Activity::ANNOUNCE) {
 				Post\SearchIndex::update($posted_item['thr-parent-id']);
 			}

src/Model/Post/Content.php

@@ -110,9 +110,9 @@ class Content
 	{
 		$search = Post\Engagement::escapeKeywords($search);
 		if ($uid != 0) {
-			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) and (private = ? OR  `uri-id` in (SELECT `uri-id` FROM `post-user` where `uid` = ?))", $search, Item::PUBLIC, $uid];
+			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND (NOT `restricted` OR `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ?))", $search, $uid];
 		} else {
-			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) and private = ?", $search, Item::PUBLIC];
+			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND NOT `restricted`", $search];
 		}
 
 		if (!empty($last_uriid)) {
@@ -139,9 +139,9 @@ class Content
 	{
 		$search = Post\Engagement::escapeKeywords($search);
 		if ($uid != 0) {
-			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) and (private = ? OR  `uri-id` in (SELECT `uri-id` FROM `post-user` where `uid` = ?))", $search, Item::PUBLIC, $uid];
+			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND (NOT `restricted` OR `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ?))", $search, $uid];
 		} else {
-			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) and private = ?", $search, Item::PUBLIC];
+			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND NOT `restricted", $search];
 		}
 		return DBA::count('post-searchindex', $condition);
 	}

src/Model/Post/Engagement.php

@@ -22,6 +22,7 @@
 namespace Friendica\Model\Post;
 
 use Friendica\Content\Text\BBCode;
+use Friendica\Core\L10n;
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
 use Friendica\Database\DBA;
@@ -38,10 +39,12 @@ use Friendica\Util\DateTimeFormat;
 
 class Engagement
 {
-	const KEYWORDS     = ['source', 'server', 'from', 'to', 'group', 'application', 'tag', 'network', 'platform', 'visibility', 'language'];
+	const KEYWORDS     = ['source', 'server', 'from', 'to', 'group', 'application', 'tag', 'network', 'platform', 'visibility', 'language', 'media'];
 	const SHORTCUTS    = ['lang' => 'language', 'net' => 'network', 'relay' => 'application'];
 	const ALTERNATIVES = ['source:news' => 'source:service', 'source:relay' => 'source:application',
-		'network:activitypub' => 'network:apub', 'network:friendica' => 'network:dfrn', 'network:diaspora' => 'network:dspr', 'network:ostatus' => 'network:stat',
+		'media:picture' => 'media:image', 'media:photo' => 'media:image',
+		'network:activitypub' => 'network:apub', 'network:friendica' => 'network:dfrn',
+		'network:diaspora' => 'network:dspr', 'network:ostatus' => 'network:stat',
 		'network:discourse' => 'network:dscs', 'network:tumblr' => 'network:tmbl', 'network:bluesky' => 'network:bsky'];
 
 	/**
@@ -92,10 +95,10 @@ class Engagement
 			$store = !empty($mediatype);
 		}
 
-		$searchtext = self::getSearchTextForItem($parent);
+		$searchtext = self::getSearchTextForItem($parent, $mediatype);
+		$language   = !empty($parent['language']) ? (array_key_first(json_decode($parent['language'], true)) ?? L10n::UNDETERMINED_LANGUAGE) : L10n::UNDETERMINED_LANGUAGE;
 		if (!$store) {
-			$language = !empty($parent['language']) ? (array_key_first(json_decode($parent['language'], true)) ?? '') : '';
+			$store = DI::userDefinedChannel()->match($searchtext, $language);
-			$store    = DI::userDefinedChannel()->match($searchtext, $language);
 		}
 
 		$engagement = [
@@ -103,7 +106,7 @@ class Engagement
 			'owner-id'     => $parent['owner-id'],
 			'contact-type' => $parent['contact-contact-type'],
 			'media-type'   => $mediatype,
-			'language'     => $parent['language'],
+			'language'     => $language,
 			'searchtext'   => $searchtext,
 			'size'         => self::getContentSize($parent),
 			'created'      => $parent['created'],
@@ -130,7 +133,7 @@ class Engagement
 		return ($ret && !$exists) ? $engagement['uri-id'] : 0;
 	}
 
-	private static function getContentSize(array $item): int 
+	public static function getContentSize(array $item): int 
 	{
 		$body = ' ' . $item['title'] . ' ' . $item['content-warning'] . ' ' . $item['body'];
 		$body = BBCode::removeAttachment($body);
@@ -171,7 +174,7 @@ class Engagement
 			}
 		}
 
-		return self::getSearchText($item, $receivers, $tags);
+		return self::getSearchText($item, $receivers, $tags, 0);
 	}
 
 	public static function getSearchTextForUriId(int $uri_id, bool $refresh = false): string
@@ -189,17 +192,18 @@ class Engagement
 		if (empty($post['uri-id'])) {
 			return '';
 		}
-		return self::getSearchTextForItem($post);
+		$mediatype = self::getMediaType($uri_id);
+		return self::getSearchTextForItem($post, $mediatype);
 	}
 
-	private static function getSearchTextForItem(array $item): string
+	private static function getSearchTextForItem(array $item, int $mediatype): string
 	{
 		$receivers = array_column(Tag::getByURIId($item['uri-id'], [Tag::MENTION, Tag::IMPLICIT_MENTION, Tag::EXCLUSIVE_MENTION, Tag::AUDIENCE]), 'url');
 		$tags      = array_column(Tag::getByURIId($item['uri-id'], [Tag::HASHTAG]), 'name');
-		return self::getSearchText($item, $receivers, $tags);
+		return self::getSearchText($item, $receivers, $tags, $mediatype);
 	}
 
-	private static function getSearchText(array $item, array $receivers, array $tags): string
+	private static function getSearchText(array $item, array $receivers, array $tags, int $mediatype): string
 	{
 		$body = '[nosmile]network_' . $item['network'];
 
@@ -285,6 +289,18 @@ class Engagement
 			$body .= ' language_' . array_key_first($languages);
 		}
 
+		if ($mediatype & 1) {
+			$body .= ' media_image';
+		}
+
+		if ($mediatype & 2) {
+			$body .= ' media_video';
+		}
+
+		if ($mediatype & 4) {
+			$body .= ' media_audio';
+		}
+
 		$body .= ' ' . $item['title'] . ' ' . $item['content-warning'] . ' ' . $item['body'];
 
 		return BBCode::toSearchText($body, $item['uri-id']);
@@ -315,7 +331,7 @@ class Engagement
 		return $text;
 	}
 
-	private static function getMediaType(int $uri_id): int
+	public static function getMediaType(int $uri_id): int
 	{
 		$media = Post\Media::getByURIId($uri_id);
 		$type  = 0;

src/Model/Post/SearchIndex.php

@@ -21,10 +21,13 @@
 
 namespace Friendica\Model\Post;
 
+use Friendica\Core\L10n;
 use Friendica\Core\Logger;
+use Friendica\Core\Protocol;
 use Friendica\Database\Database;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\Item;
 use Friendica\Model\Post;
 use Friendica\Util\DateTimeFormat;
 
@@ -34,24 +37,27 @@ class SearchIndex
 	 * Insert a post-searchindex entry
 	 *
 	 * @param int $uri_id
-	 * @param string $network
-	 * @param int $private
 	 * @param string $created
 	 * @param bool $refresh
 	 */
-	public static function insert(int $uri_id, string $network, int $private, string $created, bool $refresh = false)
+	public static function insert(int $uri_id, string $created, bool $refresh = false)
 	{
 		$limit = self::searchAgeDateLimit();
 		if (!empty($limit) && (strtotime($created) < strtotime($limit))) {
 			return;
 		}
 
+		$item = Post::selectFirstPost(['created', 'owner-id', 'private', 'language', 'network', 'title', 'content-warning', 'body'], ['uri-id' => $uri_id]);
+
 		$search = [
 			'uri-id'     => $uri_id,
-			'network'    => $network,
+			'owner-id'   => $item['owner-id'],
-			'private'    => $private,
+			'media-type' => Engagement::getMediaType($uri_id),
-			'created'    => $created,
+			'language'   => !empty($item['language']) ? (array_key_first(json_decode($item['language'], true)) ?? L10n::UNDETERMINED_LANGUAGE) : L10n::UNDETERMINED_LANGUAGE,
 			'searchtext' => Post\Engagement::getSearchTextForUriId($uri_id, $refresh),
+			'size'       => Engagement::getContentSize($item),
+			'created'    => $item['created'],
+			'restricted' => !in_array($item['network'], Protocol::FEDERATED) || ($item['private'] != Item::PUBLIC),
 		];
 		return DBA::insert('post-searchindex', $search, Database::INSERT_UPDATE);
 	}

src/Module/Api/Mastodon/Search.php

@@ -154,7 +154,7 @@ class Search extends BaseApi
 			$table = 'tag-search-view';
 		} else {
 			$q = Post\Engagement::escapeKeywords($q);
-			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) and (private = ? OR  `uri-id` in (SELECT `uri-id` FROM `post-user` where `uid` = ?))", $q, Item::PUBLIC, $uid];
+			$condition = ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE) AND (NOT `restricted` OR `uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ?))", $q, $uid];
 			$table = 'post-searchindex';
 		}
 

src/Module/Conversation/Timeline.php

@@ -324,7 +324,7 @@ class Timeline extends BaseModule
 		} elseif ($this->selectedTab == ChannelEntity::AUDIO) {
 			$condition = ["`media-type` & ?", 4];
 		} elseif ($this->selectedTab == ChannelEntity::LANGUAGE) {
-			$condition = ["JSON_EXTRACT(JSON_KEYS(language), '$[0]') = ?", User::getLanguageCode($uid)];
+			$condition = ["`language` = ?", User::getLanguageCode($uid)];
 		} elseif (is_numeric($this->selectedTab)) {
 			$condition = $this->getUserChannelConditions($this->selectedTab, $uid);
 		}
@@ -412,19 +412,50 @@ class Timeline extends BaseModule
 		}
 
 		if (!empty($channel->fullTextSearch)) {
-			$condition = DBA::mergeConditions($condition, ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE)", Engagement::escapeKeywords($channel->fullTextSearch)]);
+			if (!empty($channel->includeTags)) {
-		}
+				$additional = self:: addIncludeTags($channel->includeTags);
+			} else {
+				$additional = '';
+			}
 
-		if (!empty($channel->includeTags)) {
+			if (!empty($channel->excludeTags)) {
-			$search       = explode(',', mb_strtolower($channel->includeTags));
+				foreach (explode(',', mb_strtolower($channel->excludeTags)) as $tag) {
-			$placeholders = substr(str_repeat("?, ", count($search)), 0, -2);
+					$additional .= ' -tag:' . $tag;
-			$condition    = DBA::mergeConditions($condition, array_merge(["`uri-id` IN (SELECT `uri-id` FROM `post-tag` INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid` WHERE `post-tag`.`type` = 1 AND `name` IN (" . $placeholders . "))"], $search));
+				}
-		}
+			}
 
-		if (!empty($channel->excludeTags)) {
+			if (!empty($channel->mediaType)) {
-			$search       = explode(',', mb_strtolower($channel->excludeTags));
+				$additional .= self::addMediaTerms($channel->mediaType);
-			$placeholders = substr(str_repeat("?, ", count($search)), 0, -2);
+			}
-			$condition    = DBA::mergeConditions($condition, array_merge(["NOT `uri-id` IN (SELECT `uri-id` FROM `post-tag` INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid` WHERE `post-tag`.`type` = 1 AND `name` IN (" . $placeholders . "))"], $search));
+
+			$additional .= self::addLanguageSearchTerms($uid, $channel->languages);
+
+			if ($additional) {
+				$searchterms = '+(' . trim($channel->fullTextSearch) . ')' . $additional;
+			} else {
+				$searchterms = $channel->fullTextSearch;
+			}
+
+			$condition = DBA::mergeConditions($condition, ["MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE)", Engagement::escapeKeywords($searchterms)]);
+		} else {
+			if (!empty($channel->includeTags)) {
+				$search       = explode(',', mb_strtolower($channel->includeTags));
+				$placeholders = substr(str_repeat("?, ", count($search)), 0, -2);
+				$condition    = DBA::mergeConditions($condition, array_merge(["`uri-id` IN (SELECT `uri-id` FROM `post-tag` INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid` WHERE `post-tag`.`type` = 1 AND `name` IN (" . $placeholders . "))"], $search));
+			}
+	
+			if (!empty($channel->excludeTags)) {
+				$search       = explode(',', mb_strtolower($channel->excludeTags));
+				$placeholders = substr(str_repeat("?, ", count($search)), 0, -2);
+				$condition    = DBA::mergeConditions($condition, array_merge(["NOT `uri-id` IN (SELECT `uri-id` FROM `post-tag` INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid` WHERE `post-tag`.`type` = 1 AND `name` IN (" . $placeholders . "))"], $search));
+			}
+
+			if (!empty($channel->mediaType)) {
+				$condition = DBA::mergeConditions($condition, ["`media-type` & ?", $channel->mediaType]);
+			}
+	
+			// For "addLanguageCondition" to work, the condition must not be empty
+			$condition = $this->addLanguageCondition($uid, $condition ?: ["true"], $channel->languages);
 		}
 
 		if (!is_null($channel->minSize)) {
@@ -435,24 +466,67 @@ class Timeline extends BaseModule
 			$condition = DBA::mergeConditions($condition, ["`size` <= ?", $channel->maxSize]);
 		}
 
-		if (!empty($channel->mediaType)) {
+		return $condition;
-			$condition = DBA::mergeConditions($condition, ["`media-type` & ?", $channel->mediaType]);
+	}
+
+	private function addIncludeTags(string $includeTags): string
+	{
+		$tagterms = '';
+		foreach (explode(',', mb_strtolower($includeTags)) as $tag) {
+			$tagterms .= ' tag:' . $tag;
 		}
 
-		// For "addLanguageCondition" to work, the condition must not be empty
+		if ($tagterms) {
-		$condition = $this->addLanguageCondition($uid, $condition ?: ["true"], $channel->languages);
+			return ' +(' . trim($tagterms) . ')';
+		} else {
+			return '';
+		}
+	}
 
-		return $condition;
+	private function addMediaTerms(int $mediaType): string
+	{
+		$mediaterms = '';
+		if ($mediaType & 1) {
+			$mediaterms .= ' media:image';
+		}
+
+		if ($mediaType & 2) {
+			$mediaterms .= ' media:video';
+		}
+
+		if ($mediaType & 4) {
+			$mediaterms .= ' media:audio';
+		}
+
+		if ($mediaterms) {
+			return ' +(' . trim($mediaterms) . ')';
+		} else {
+			return '';
+		}
+	}
+
+	private function addLanguageSearchTerms(int $uid, $languages = null): string
+	{
+		$langterms = '';
+		foreach ($languages ?: User::getWantedLanguages($uid) as $language) {
+			$langterms .= ' language:' . $language;
+		}
+
+		if ($langterms) {
+			return ' +(' . trim($langterms) . ')';
+		} else {
+			return '';
+		}
 	}
 
 	private function addLanguageCondition(int $uid, array $condition, $languages = null): array
 	{
 		$conditions = [];
-		$languages  = $languages ?: User::getWantedLanguages($uid);
+		foreach ($languages ?: User::getWantedLanguages($uid) as $language) {
-		foreach ($languages as $language) {
+			$conditions[] = "`language` = ?";
-			$conditions[] = "JSON_EXTRACT(JSON_KEYS(language), '$[0]') = ?";
 			$condition[]  = $language;
 		}
+
 		if (!empty($conditions)) {
 			$condition[0] .= " AND (" . implode(' OR ', $conditions) . ")";
 		}

static/dbstructure.config.php

@@ -56,7 +56,7 @@ use Friendica\Database\DBA;
 
 // This file is required several times during the test in DbaDefinition which justifies this condition
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1549);
+	define('DB_UPDATE_VERSION', 1550);
 }
 
 return [
@@ -1245,7 +1245,7 @@ return [
 	"post-activity" => [
 		"comment" => "Original remote activity",
 		"fields" => [
-			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1",  "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
 			"activity" => ["type" => "mediumtext", "comment" => "Original activity"],
 			"received" => ["type" => "datetime", "comment" => ""],
 		],
@@ -1256,7 +1256,7 @@ return [
 	"post-category" => [
 		"comment" => "post relation to categories",
 		"fields" => [
-			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1",  "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
 			"uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "foreign" => ["user" => "uid"], "comment" => "User id"],
 			"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""],
 			"tid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "foreign" => ["tag" => "id", "on delete" => "restrict"], "comment" => ""],
@@ -1363,11 +1363,11 @@ return [
 	"post-engagement" => [
 		"comment" => "Engagement data per post",
 		"fields" => [
-			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1",  "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
 			"owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id"], "comment" => "Item owner"],
 			"contact-type" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Person, organisation, news, community, relay"],
 			"media-type" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Type of media in a bit array (1 = image, 2 = video, 4 = audio"],
-			"language" => ["type" => "varchar(128)", "comment" => "Language information about this post"],
+			"language" => ["type" => "char(2)", "comment" => "Language information about this post in the ISO 639-1 format"],
 			"searchtext" => ["type" => "mediumtext", "comment" => "Simplified text for the full text search"],
 			"size" => ["type" => "int unsigned", "comment" => "Body size"],
 			"created" => ["type" => "datetime", "comment" => ""],
@@ -1486,13 +1486,17 @@ return [
 		"comment" => "Content for all posts",
 		"fields" => [
 			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
-			"network" => ["type" => "char(4)", "comment" => ""],
+			"owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id"], "comment" => "Item owner"],
-			"private" => ["type" => "tinyint unsigned", "comment" => "0=public, 1=private, 2=unlisted"],
+			"media-type" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Type of media in a bit array (1 = image, 2 = video, 4 = audio"],
+			"language" => ["type" => "char(2)", "comment" => "Language information about this post in the ISO 639-1 format"],
 			"searchtext" => ["type" => "mediumtext", "comment" => "Simplified text for the full text search"],
+			"size" => ["type" => "int unsigned", "comment" => "Body size"],
 			"created" => ["type" => "datetime", "comment" => ""],
+			"restricted" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "If true, this post is either unlisted or not from a federated network"],
 		],
 		"indexes" => [
 			"PRIMARY" => ["uri-id"],
+			"owner-id" => ["owner-id"],
 			"created" => ["created"],
 			"searchtext" => ["FULLTEXT", "searchtext"],
 		]

update.php

@@ -1411,3 +1411,14 @@ function update_1539()
 
 	return Update::SUCCESS;
 }
+
+function pre_update_1550()
+{
+	if (DBStructure::existsTable('post-engagement') && DBStructure::existsColumn('post-engagement', ['language'])) {
+		DBA::e("ALTER TABLE `post-engagement` DROP `language`");
+	}
+	if (DBStructure::existsTable('post-searchindex') && DBStructure::existsColumn('post-searchindex', ['network'])) {
+		DBA::e("ALTER TABLE `post-searchindex` DROP `network`, DROP `private`");
+	}
+	return Update::SUCCESS;
+}