Merge pull request #13806 from annando/channel-relay

New user account type "Channel Relay"

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2024.03-dev (Yellow Archangel)
--- DB_UPDATE_VERSION 1545
+-- DB_UPDATE_VERSION 1546
 -- ------------------------------------------
 
 
@@ -505,6 +505,8 @@ CREATE TABLE IF NOT EXISTS `channel` (
 	`full-text-search` varchar(1023) COMMENT 'Full text search pattern, see https://mariadb.com/kb/en/full-text-index-overview/#in-boolean-mode',
 	`media-type` smallint unsigned COMMENT 'Filtered media types',
 	`languages` mediumtext COMMENT 'Desired languages',
+	`publish` boolean COMMENT 'publish channel content',
+	`valid` boolean COMMENT 'Set, when the full-text-search is valid',
 	 PRIMARY KEY(`id`),
 	 INDEX `uid` (`uid`),
 	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
@@ -1343,7 +1345,7 @@ CREATE TABLE IF NOT EXISTS `post-engagement` (
 	`owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner',
 	`contact-type` tinyint NOT NULL DEFAULT 0 COMMENT 'Person, organisation, news, community, relay',
 	`media-type` tinyint NOT NULL DEFAULT 0 COMMENT 'Type of media in a bit array (1 = image, 2 = video, 4 = audio',
-	`language` varbinary(128) COMMENT 'Language information about this post',
+	`language` varchar(128) COMMENT 'Language information about this post',
 	`searchtext` mediumtext COMMENT 'Simplified text for the full text search',
 	`created` datetime COMMENT '',
 	`restricted` boolean NOT NULL DEFAULT '0' COMMENT 'If true, this post is either unlisted or not from a federated network',

doc/database/db_channel.md

@@ -19,6 +19,8 @@ Fields
 | full-text-search | Full text search pattern, see https://mariadb.com/kb/en/full-text-index-overview/#in-boolean-mode | varchar(1023)      | YES  |     | NULL    |                |
 | media-type       | Filtered media types                                                                              | smallint unsigned  | YES  |     | NULL    |                |
 | languages        | Desired languages                                                                                 | mediumtext         | YES  |     | NULL    |                |
+| publish          | publish channel content                                                                           | boolean            | YES  |     | NULL    |                |
+| valid            | Set, when the full-text-search is valid                                                           | boolean            | YES  |     | NULL    |                |
 
 Indexes
 ------------

doc/database/db_post-engagement.md

@@ -12,7 +12,7 @@ Fields
 | owner-id     | Item owner                                                            | int unsigned       | NO   |     | 0       |       |
 | contact-type | Person, organisation, news, community, relay                          | tinyint            | NO   |     | 0       |       |
 | media-type   | Type of media in a bit array (1 = image, 2 = video, 4 = audio         | tinyint            | NO   |     | 0       |       |
-| language     | Language information about this post                                  | varbinary(128)     | YES  |     | NULL    |       |
+| language     | Language information about this post                                  | varchar(128)       | YES  |     | NULL    |       |
 | searchtext   | Simplified text for the full text search                              | mediumtext         | YES  |     | NULL    |       |
 | created      |                                                                       | datetime           | YES  |     | NULL    |       |
 | restricted   | If true, this post is either unlisted or not from a federated network | boolean            | NO   |     | 0       |       |

src/Content/Conversation/Entity/Timeline.php

@@ -34,6 +34,8 @@ namespace Friendica\Content\Conversation\Entity;
  * @property-read int    $mediaType      Media types that are included in the channel
  * @property-read array  $languages      Channel languages
  * @property-read int    $circle         Circle or timeline this channel is based on
+ * @property-read bool   $publish        Publish the channel
+ * @property-read bool   $valid          Indicates that the search conditions are valid
  */
 class Timeline extends \Friendica\BaseEntity
 {
@@ -61,8 +63,12 @@ class Timeline extends \Friendica\BaseEntity
 	protected $mediaType;
 	/** @var array */
 	protected $languages;
+	/** @var bool */
+	protected $publish;
+	/** @var bool */
+	protected $valid;
 
-	public function __construct(string $code = null, string $label = null, string $description = null, string $accessKey = null, string $path = null, int $uid = null, string $includeTags = null, string $excludeTags = null, string $fullTextSearch = null, int $mediaType = null, int $circle = null, array $languages = null)
+	public function __construct(string $code = null, string $label = null, string $description = null, string $accessKey = null, string $path = null, int $uid = null, string $includeTags = null, string $excludeTags = null, string $fullTextSearch = null, int $mediaType = null, int $circle = null, array $languages = null, bool $publish = null, bool $valid = null)
 	{
 		$this->code           = $code;
 		$this->label          = $label;
@@ -76,5 +82,7 @@ class Timeline extends \Friendica\BaseEntity
 		$this->mediaType      = $mediaType;
 		$this->circle         = $circle;
 		$this->languages      = $languages;
+		$this->publish        = $publish;
+		$this->valid          = $valid;
 	}
 }

src/Content/Conversation/Factory/UserDefinedChannel.php

@@ -50,6 +50,8 @@ final class UserDefinedChannel extends Timeline implements ICanCreateFromTableRo
 			$row['media-type'] ?? null,
 			$row['circle'] ?? null,
 			$row['languages'] ?? null,
+			$row['publish'] ?? null,
+			$row['valid'] ?? null,
 		);
 	}
 }

src/Content/Conversation/Repository/UserDefinedChannel.php

@@ -25,24 +25,27 @@ use Friendica\BaseCollection;
 use Friendica\Content\Conversation\Collection\UserDefinedChannels;
 use Friendica\Content\Conversation\Entity;
 use Friendica\Content\Conversation\Factory;
-use Friendica\Core\PConfig\Capability\IManagePersonalConfigValues;
+use Friendica\Core\Config\Capability\IManageConfigValues;
 use Friendica\Database\Database;
+use Friendica\Database\DBA;
+use Friendica\Model\Contact;
 use Friendica\Model\Post\Engagement;
 use Friendica\Model\User;
+use Friendica\Util\DateTimeFormat;
 use Psr\Log\LoggerInterface;
 
 class UserDefinedChannel extends \Friendica\BaseRepository
 {
 	protected static $table_name = 'channel';
 
-	/** @var IManagePersonalConfigValues */
+	/** @var IManageConfigValues */
-	private $pConfig;
+	private $config;
 
-	public function __construct(Database $database, LoggerInterface $logger, Factory\UserDefinedChannel $factory, IManagePersonalConfigValues $pConfig)
+	public function __construct(Database $database, LoggerInterface $logger, Factory\UserDefinedChannel $factory, IManageConfigValues $config)
 	{
 		parent::__construct($database, $logger, $factory);
 
-		$this->pConfig = $pConfig;
+		$this->config = $config;
 	}
 
 	/**
@@ -63,6 +66,11 @@ class UserDefinedChannel extends \Friendica\BaseRepository
 		return $Entities;
 	}
 
+	public function select(array $condition, array $params = []): UserDefinedChannels
+	{
+		return $this->_select($condition, $params);
+	}
+
 	/**
 	 * Fetch a single user channel
 	 *
@@ -125,6 +133,8 @@ class UserDefinedChannel extends \Friendica\BaseRepository
 			'full-text-search' => $Channel->fullTextSearch,
 			'media-type'       => $Channel->mediaType,
 			'languages'        => serialize($Channel->languages),
+			'publish'          => $Channel->publish,
+			'valid'            => $this->isValid($Channel->fullTextSearch),
 		];
 
 		if ($Channel->code) {
@@ -140,9 +150,17 @@ class UserDefinedChannel extends \Friendica\BaseRepository
 		return $Channel;
 	}
 
+	private function isValid(string $searchtext): bool
+	{
+		if ($searchtext == '') {
+			return true;
+		}
+
+		return $this->db->select('check-full-text-search', [], ["`pid` = ? AND MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE)", getmypid(), $this->escapeKeywords($searchtext)]) !== false;
+	}
+
 	/**
-	 * Checks, if one of the user defined channels matches with the given search text
+	 * Checks, if one of the user defined channels matches with the given search text or languages
-	 * @todo To increase the performance, this functionality should be replaced with a single SQL call.
 	 *
 	 * @param string $searchtext
 	 * @param string $language
@@ -150,30 +168,167 @@ class UserDefinedChannel extends \Friendica\BaseRepository
 	 */
 	public function match(string $searchtext, string $language): bool
 	{
+		$users = $this->db->selectToArray('user', ['uid'], $this->getUserCondition());
+		if (empty($users)) {
+			return [];
+		}
+
+		$uids = array_column($users, 'uid');
+
+		$usercondition = ['uid' => $uids];
+		$condition = DBA::mergeConditions($usercondition, ["`languages` != ? AND `include-tags` = ? AND `full-text-search` = ? AND `circle` = ?", '', '', '', 0]);
+		foreach ($this->select($condition) as $channel) {
+			if (!empty($channel->languages) && in_array($language, $channel->languages)) {
+				return true;
+			}
+		}
+
+		$search = '';
+		$condition = DBA::mergeConditions($usercondition, ["`full-text-search` != ? AND `circle` = ? AND `valid`", '', 0]);
+		foreach ($this->select($condition) as $channel) {
+			$search .= '(' . $channel->fullTextSearch . ') ';
+		}
+
+		$this->insertCheckFullTextSearch($searchtext);
+		$result = $this->inFulltext($search);
+		$this->deleteCheckFullTextSearch();
+
+		return $result;
+	}
+
+	/**
+	 * Fetch the channel users that have got matching channels
+	 *
+	 * @param string $searchtext
+	 * @param string $language
+	 * @param array  $tags
+	 * @param int    $media_type
+	 * @param int    $owner_id
+	 * @param int    $reshare_id
+	 * @return array
+	 */
+	public function getMatchingChannelUsers(string $searchtext, string $language, array $tags, int $media_type, int $owner_id, int $reshare_id): array
+	{
+		$condition = $this->getUserCondition();
+		$condition = DBA::mergeConditions($condition, ["`account-type` IN (?, ?) AND `uid` != ?", User::ACCOUNT_TYPE_RELAY, User::ACCOUNT_TYPE_COMMUNITY, 0]);
+		$users = $this->db->selectToArray('user', ['uid'], $condition);
+		if (empty($users)) {
+			return [];
+		}
+
 		if (!in_array($language, User::getLanguages())) {
 			$this->logger->debug('Unwanted language found. No matched channel found.', ['language' => $language, 'searchtext' => $searchtext]);
+			return [];
+		}
+
+		$this->insertCheckFullTextSearch($searchtext);
+
+		$uids = [];
+
+		foreach ($this->select(['uid' => array_column($users, 'uid'), 'publish' => true, 'valid' => true]) as $channel) {
+			if (in_array($channel->uid, $uids)) {
+				continue;
+			}
+			if (!empty($channel->circle) && ($channel->circle > 0) && !in_array($channel->uid, $uids)) {
+				if (!$this->inCircle($channel->circle, $channel->uid, $owner_id) && !$this->inCircle($channel->circle, $channel->uid, $reshare_id)) {
+					continue;
+				}
+			}
+			if (!empty($channel->languages) && !in_array($channel->uid, $uids)) {
+				if (!in_array($language, $channel->languages)) {
+					continue;
+				}
+			} elseif (!in_array($language, User::getWantedLanguages($channel->uid))) {
+				continue;
+			}
+			if (!empty($channel->includeTags) && !in_array($channel->uid, $uids)) {
+				if (!$this->inTaglist($channel->includeTags, $tags)) {
+					continue;
+				}
+			}
+			if (!empty($channel->excludeTags) && !in_array($channel->uid, $uids)) {
+				if ($this->inTaglist($channel->excludeTags, $tags)) {
+					continue;
+				}
+			}
+			if (!empty($channel->mediaType) && !in_array($channel->uid, $uids)) {
+				if (!($channel->mediaType & $media_type)) {
+					continue;
+				}
+			}
+			if (!empty($channel->fullTextSearch) && !in_array($channel->uid, $uids)) {
+				if (!$this->inFulltext($channel->fullTextSearch)) {
+					continue;
+				}
+			}
+			$uids[] = $channel->uid;
+			$this->logger->debug('Matching channel found.', ['uid' => $channel->uid, 'label' => $channel->label, 'language' => $language, 'tags' => $tags, 'media_type' => $media_type, 'searchtext' => $searchtext]);
+		}
+
+		$this->deleteCheckFullTextSearch();
+		return $uids;
+	}
+
+	private function insertCheckFullTextSearch(string $searchtext)
+	{
+		$this->db->insert('check-full-text-search', ['pid' => getmypid(), 'searchtext' => $searchtext], Database::INSERT_UPDATE);
+	}
+
+	private function deleteCheckFullTextSearch()
+	{
+		$this->db->delete('check-full-text-search', ['pid' => getmypid()]);
+	}
+
+	private function inCircle(int $circleId, int $uid, int $cid): bool
+	{
+		if ($cid == 0) {
 			return false;
 		}
 
-		$store = false;
+		$account = Contact::selectFirstAccountUser(['id'], ['pid' => $cid, 'uid' => $uid]);
-		$this->db->insert('check-full-text-search', ['pid' => getmypid(), 'searchtext' => $searchtext], Database::INSERT_UPDATE);
+		if (empty($account['id'])) {
-		$channels = $this->db->select(self::$table_name, ['full-text-search', 'uid', 'label'], ["`full-text-search` != ? AND `circle` = ?", '', 0]);
+			return false;
-		while ($channel = $this->db->fetch($channels)) {
-			$channelsearchtext = $channel['full-text-search'];
-			foreach (Engagement::KEYWORDS as $keyword) {
-				$channelsearchtext = preg_replace('~(' . $keyword . ':.[\w@\.-]+)~', '"$1"', $channelsearchtext);
 		}
-			if ($this->db->exists('check-full-text-search', ["`pid` = ? AND MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE)", getmypid(), $channelsearchtext])) {
+		return $this->db->exists('group_member', ['gid' => $circleId, 'contact-id' => $account['id']]);
-				if (in_array($language, $this->pConfig->get($channel['uid'], 'channel', 'languages', [User::getLanguageCode($channel['uid'])]))) {
-					$store = true;
-					$this->logger->debug('Matching channel found.', ['uid' => $channel['uid'], 'label' => $channel['label'], 'language' => $language, 'channelsearchtext' => $channelsearchtext, 'searchtext' => $searchtext]);
-					break;
 	}
-			}
-		}
-		$this->db->close($channels);
 
-		$this->db->delete('check-full-text-search', ['pid' => getmypid()]);
+	private function inTaglist(string $tagList, array $tags): bool
-		return $store;
+	{
+		if (empty($tags)) {
+			return false;
+		}
+		array_walk($tags, function (&$value) {
+			$value = mb_strtolower($value);
+		});
+		foreach (explode(',', $tagList) as $tag) {
+			if (in_array($tag, $tags)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	private function inFulltext(string $fullTextSearch): bool
+	{
+		return $this->db->exists('check-full-text-search', ["`pid` = ? AND MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE)", getmypid(), $this->escapeKeywords($fullTextSearch)]);
+	}
+
+	private function escapeKeywords(string $fullTextSearch): string
+	{
+		foreach (Engagement::KEYWORDS as $keyword) {
+			$fullTextSearch = preg_replace('~(' . $keyword . ':.[\w@\.-]+)~', '"$1"', $fullTextSearch);
+		}
+		return $fullTextSearch;
+	}
+
+	private function getUserCondition()
+	{
+		$condition = ["`verified` AND NOT `blocked` AND NOT `account_removed` AND NOT `account_expired` AND `user`.`uid` > ?", 0];
+
+		$abandon_days = intval($this->config->get('system', 'account_abandon_days'));
+		if (!empty($abandon_days)) {
+			$condition = DBA::mergeConditions($condition, ["`last-activity` > ?", DateTimeFormat::utc('now - ' . $abandon_days . ' days')]);
+		}
+		return $condition;
 	}
 }

src/Content/Text/BBCode.php

@@ -258,6 +258,10 @@ class BBCode
 		// Add images because of possible alt texts
 		if (!empty($uri_id)) {
 			$text = Post\Media::addAttachmentsToBody($uri_id, $text, [Post\Media::IMAGE]);
+
+			foreach (Post\Media::getByURIId($uri_id, [Post\Media::HTML]) as $media) {
+				$text .= ' ' . $media['name'] . ' ' . $media['description'];
+			}
 		}
 
 		if (empty($text)) {
@@ -279,7 +283,7 @@ class BBCode
 		// Removes mentions, remove links from hashtags
 		$text = preg_replace('/[@!]\[url\=.*?\].*?\[\/url\]/ism', ' ', $text);
 		$text = preg_replace('/[#]\[url\=.*?\](.*?)\[\/url\]/ism', ' #$1 ', $text);
-		$text = preg_replace('/[@!#]?\[url.*?\[\/url\]/ism', ' ', $text);
+		$text = preg_replace('/[@!#]+?\[url.*?\[\/url\]/ism', ' ', $text);
 		$text = preg_replace("/\[url=[^\[\]]*\](.*)\[\/url\]/Usi", ' $1 ', $text);
 
 		// Convert it to plain text

src/Core/L10n.php

@@ -65,6 +65,9 @@ class L10n
 		'zh-cn' => '简体中文',
 	];
 
+	/** @var string Undetermined language */
+	const UNDETERMINED_LANGUAGE = 'un';
+
 	/**
 	 * A string indicating the current language used for translation:
 	 * - Two-letter ISO 639-1 code.
@@ -436,7 +439,9 @@ class L10n
 	{
 		$iso639 = new \Matriphe\ISO639\ISO639;
 
-		$languages = [];
+		// In ISO 639-2 undetermined languages have got the code "und".
+		// There is no official code for ISO 639-1, but "un" is not assigned to any language.   
+		$languages = [self::UNDETERMINED_LANGUAGE => $this->t('Undetermined')];
 
 		foreach ($this->getDetectableLanguages() as $code) {
 			$code     = $this->toISO6391($code);

src/Model/Contact.php

@@ -172,6 +172,11 @@ class Contact
 		return DBA::selectFirst('account-view', $fields, $condition, $params);
 	}
 
+	public static function selectFirstAccountUser(array $fields = [], array $condition = [], array $params = [])
+	{
+		return DBA::selectFirst('account-user-view', $fields, $condition, $params);
+	}
+
 	/**
 	 * Insert a row into the contact table
 	 * Important: You can't use DBA::lastInsertId() after this call since it will be set to 0.

src/Model/Item.php

@@ -28,6 +28,7 @@ use Friendica\Content\Post\Entity\PostMedia;
 use Friendica\Content\Text\BBCode;
 use Friendica\Content\Text\HTML;
 use Friendica\Core\Hook;
+use Friendica\Core\L10n;
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
 use Friendica\Core\Renderer;
@@ -1440,12 +1441,65 @@ class Item
 			if (in_array($posted_item['gravity'], [self::GRAVITY_ACTIVITY, self::GRAVITY_COMMENT])) {
 				Post\Counts::update($posted_item['thr-parent-id'], $posted_item['parent-uri-id'], $posted_item['vid'], $posted_item['verb'], $posted_item['body']);
 			}
-			Post\Engagement::storeFromItem($posted_item);
+
+			$engagement_uri_id = Post\Engagement::storeFromItem($posted_item);
+
+			if (($posted_item['gravity'] == self::GRAVITY_ACTIVITY) && ($posted_item['verb'] == Activity::ANNOUNCE) && ($posted_item['parent-uri-id'] == $posted_item['thr-parent-id'])) {
+				self::reshareChannelPost($posted_item['thr-parent-id'], $posted_item['author-id']);
+			} elseif ($engagement_uri_id) {
+				self::reshareChannelPost($engagement_uri_id);
+			}
 		}
 
 		return $post_user_id;
 	}
 
+	private static function reshareChannelPost(int $uri_id, int $reshare_id = 0)
+	{
+		if (!DI::config()->get('system', 'allow_relay_channels')) {
+			return;
+		}
+
+		$item = Post::selectFirst(['id', 'private', 'network', 'language', 'owner-id'], ['uri-id' => $uri_id, 'uid' => 0]);
+		if (empty($item['id'])) {
+			Logger::debug('Post not found', ['uri-id' => $uri_id]);
+			return;
+		}
+
+		if (($item['private'] != self::PUBLIC) || !in_array($item['network'], [Protocol::ACTIVITYPUB, Protocol::DFRN])) {
+			Logger::debug('Not a public post or no AP or DFRN post', ['uri-id' => $uri_id]);
+			return;
+		}
+
+		$engagement = DBA::selectFirst('post-engagement', ['searchtext', 'media-type'], ['uri-id' => $uri_id]);
+		if (empty($engagement['searchtext'])) {
+			Logger::debug('No engagement found', ['uri-id' => $uri_id]);
+			return;
+		}
+
+		$language = !empty($item['language']) ? array_key_first(json_decode($item['language'], true)) : '';
+		$tags     = array_column(Tag::getByURIId($uri_id, [Tag::HASHTAG]), 'name');
+
+		Logger::debug('Prepare check', ['uri-id' => $uri_id, 'language' => $language, 'tags' => $tags, 'searchtext' => $engagement['searchtext'], 'media_type' => $engagement['media-type'], 'owner' => $item['owner-id'], 'reshare' => $reshare_id]);
+
+		$count = 0;
+		foreach (DI::userDefinedChannel()->getMatchingChannelUsers($engagement['searchtext'], $language, $tags, $engagement['media-type'], $item['owner-id'], $reshare_id) as $uid) {
+			$condition = [
+				'verb' => Activity::ANNOUNCE, 'deleted' => false, 'gravity' => self::GRAVITY_ACTIVITY,
+				'author-id' => Contact::getPublicIdByUserId($uid), 'uid' => $uid, 'thr-parent-id' => $uri_id
+			];
+			if (!Post::exists($condition)) {
+				Logger::debug('Reshare post', ['uid' => $uid, 'uri-id' => $uri_id]);
+				self::performActivity($item['id'], 'announce', $uid);
+			} else {
+				Logger::debug('Reshare already exists', ['uid' => $uid, 'uri-id' => $uri_id]);
+			}
+			$count++;
+		}
+
+		Logger::debug('Check done', ['uri-id' => $uri_id, 'count' => $count]);
+	}
+
 	/**
 	 * Fetch the post reason for a given item array
 	 *
@@ -2036,16 +2090,20 @@ class Item
 			$transmitted[$language] = 0;
 		}
 
+		if (!in_array($item['gravity'], [self::GRAVITY_PARENT, self::GRAVITY_COMMENT])) {
+			return !empty($transmitted) ? json_encode($transmitted) : null;
+		}
+
 		$content = trim(($item['title'] ?? '') . ' ' . ($item['content-warning'] ?? '') . ' ' . ($item['body'] ?? ''));
 
-		if (!in_array($item['gravity'], [self::GRAVITY_PARENT, self::GRAVITY_COMMENT]) || empty($content)) {
+		if (empty($content) && !empty($item['quote-uri-id'])) {
-			return !empty($transmitted) ? json_encode($transmitted) : null;
+			$quoted = Post::selectFirstPost(['language'], ['uri-id' => $item['quote-uri-id']]);
+			if (!empty($quoted['language'])) {
+				return $quoted['language'];
+			}
 		}
 
-		$languages = self::getLanguageArray($content, 3, $item['uri-id'], $item['author-id']);
+		$languages = self::getLanguageArray($content, 3, $item['uri-id'], $item['author-id'], $transmitted);
-		if (empty($languages)) {
-			return !empty($transmitted) ? json_encode($transmitted) : null;
-		}
 
 		if (!empty($transmitted)) {
 			$languages = array_merge($transmitted, $languages);
@@ -2062,10 +2120,13 @@ class Item
 	 * @param integer $count
 	 * @param integer $uri_id
 	 * @param integer $author_id
+	 * @param array   $default
 	 * @return array
 	 */
-	public static function getLanguageArray(string $body, int $count, int $uri_id = 0, int $author_id = 0): array
+	public static function getLanguageArray(string $body, int $count, int $uri_id = 0, int $author_id = 0, array $default = []): array
 	{
+		$default = $default ?: [L10n::UNDETERMINED_LANGUAGE => 1];
+
 		$searchtext = BBCode::toSearchText($body, $uri_id);
 
 		if ((count(explode(' ', $searchtext)) < 10) && (mb_strlen($searchtext) < 30) && $author_id) {
@@ -2078,7 +2139,7 @@ class Item
 		}
 
 		if (empty($searchtext)) {
-			return [];
+			return $default;
 		}
 
 		$ld = new Language(DI::l10n()->getDetectableLanguages());
@@ -2102,6 +2163,9 @@ class Item
 		}
 
 		$result = self::compactLanguages($result);
+		if (empty($result)) {
+			return $default;
+		}
 
 		arsort($result);
 		return array_slice($result, 0, $count);
@@ -2211,8 +2275,13 @@ class Item
 		foreach (json_decode($item['language'], true) as $language => $reliability) {
 			$code = DI::l10n()->toISO6391($language);
 
+			if ($code == L10n::UNDETERMINED_LANGUAGE) {
+				$native = $language = DI::l10n()->t('Undetermined');
+			} else {
 				$native   = $iso639->nativeByCode1($code);
 				$language = $iso639->languageByCode1($code);
+			}
+
 			if ($native != $language) {
 				$used_languages .= DI::l10n()->t('%s (%s - %s): %s', $native, $language, $code, number_format($reliability, 5)) . '\n';
 			} else {

src/Model/Post/Engagement.php

@@ -45,13 +45,13 @@ class Engagement
 	 * Store engagement data from an item array
 	 *
 	 * @param array $item
-	 * @return void
+	 * @return int uri-id of the engagement post if newly inserted, 0 on update
 	 */
-	public static function storeFromItem(array $item)
+	public static function storeFromItem(array $item): int
 	{
 		if (in_array($item['verb'], [Activity::FOLLOW, Activity::VIEW, Activity::READ])) {
 			Logger::debug('Technical activities are not stored', ['uri-id' => $item['uri-id'], 'parent-uri-id' => $item['parent-uri-id'], 'verb' => $item['verb']]);
-			return;
+			return 0;
 		}
 
 		$parent = Post::selectFirst(['uri-id', 'created', 'author-id', 'owner-id', 'uid', 'private', 'contact-contact-type', 'language', 'network',
@@ -60,7 +60,7 @@ class Engagement
 
 		if ($parent['created'] < self::getCreationDateLimit(false)) {
 			Logger::debug('Post is too old', ['uri-id' => $item['uri-id'], 'parent-uri-id' => $item['parent-uri-id'], 'created' => $parent['created']]);
-			return;
+			return 0;
 		}
 
 		$store = ($item['gravity'] != Item::GRAVITY_PARENT);
@@ -69,10 +69,14 @@ class Engagement
 			$store = Contact::hasFollowers($parent['owner-id']);
 		}
 
+		if (!$store && ($parent['owner-id'] != $parent['author-id'])) {
+			$store = Contact::hasFollowers($parent['author-id']);
+		}
+
 		if (!$store) {
 			$tagList = Relay::getSubscribedTags();
 			foreach (array_column(Tag::getByURIId($item['parent-uri-id'], [Tag::HASHTAG]), 'name') as $tag) {
-				if (in_array($tag, $tagList)) {
+				if (in_array(mb_strtolower($tag), $tagList)) {
 					$store = true;
 					break;
 				}
@@ -87,9 +91,7 @@ class Engagement
 
 		$searchtext = self::getSearchTextForItem($parent);
 		if (!$store) {
-			$content   = trim(($parent['title'] ?? '') . ' ' . ($parent['content-warning'] ?? '') . ' ' . ($parent['body'] ?? ''));
+			$language = !empty($parent['language']) ? (array_key_first(json_decode($parent['language'], true)) ?? '') : '';
-			$languages = Item::getLanguageArray($content, 1, 0, $parent['author-id']);
-			$language  = !empty($languages) ? array_key_first($languages) : '';
 			$store    = DI::userDefinedChannel()->match($searchtext, $language);
 		}
 
@@ -111,10 +113,17 @@ class Engagement
 		];
 		if (!$store && ($engagement['comments'] == 0) && ($engagement['activities'] == 0)) {
 			Logger::debug('No media, follower, subscribed tags, comments or activities. Engagement not stored', ['fields' => $engagement]);
-			return;
+			return 0;
 		}
-		$ret = DBA::insert('post-engagement', $engagement, Database::INSERT_UPDATE);
+		$exists = DBA::exists('post-engagement', ['uri-id' => $engagement['uri-id']]);
-		Logger::debug('Engagement stored', ['fields' => $engagement, 'ret' => $ret]);
+		if ($exists) {
+			$ret = DBA::update('post-engagement', $engagement, ['uri-id' => $engagement['uri-id']]);
+			Logger::debug('Engagement updated', ['uri-id' => $engagement['uri-id'], 'ret' => $ret]);
+		} else {
+			$ret = DBA::insert('post-engagement', $engagement);
+			Logger::debug('Engagement inserted', ['uri-id' => $engagement['uri-id'], 'ret' => $ret]);
+		}
+		return ($ret && !$exists) ? $engagement['uri-id'] : 0;
 	}
 
 	public static function getSearchTextForActivity(string $content, int $author_id, array $tags, array $receivers): string

src/Model/User.php

@@ -638,6 +638,16 @@ class User
 		}
 		DBA::close($channels);
 
+		foreach (DI::userDefinedChannel()->select(["NOT `languages` IS NULL"]) as $channel) {
+			foreach ($channel->languages ?? [] as $language) {
+				$languages[$language] = $language;
+			}
+		}
+
+		if (!DI::config()->get('system', 'relay_deny_undetected_language')) {
+			$languages[L10n::UNDETERMINED_LANGUAGE] = L10n::UNDETERMINED_LANGUAGE;
+		}
+
 		ksort($languages);
 		$languages = array_keys($languages);
 		DI::cache()->set($cachekey, $languages);

src/Module/Admin/Site.php

@@ -92,6 +92,7 @@ class Site extends BaseAdmin
 		$private_addons         = !empty($_POST['private_addons']);
 		$disable_embedded       = !empty($_POST['disable_embedded']);
 		$allow_users_remote_self = !empty($_POST['allow_users_remote_self']);
+		$allow_relay_channels   = !empty($_POST['allow_relay_channels']);
 		$adjust_poll_frequency  = !empty($_POST['adjust_poll_frequency']);
 		$min_poll_interval      = (!empty($_POST['min_poll_interval']) ? intval(trim($_POST['min_poll_interval']))                : 0);
 		$explicit_content       = !empty($_POST['explicit_content']);
@@ -262,6 +263,7 @@ class Site extends BaseAdmin
 		$transactionConfig->set('system', 'enotify_no_content'     , $enotify_no_content);
 		$transactionConfig->set('system', 'disable_embedded'       , $disable_embedded);
 		$transactionConfig->set('system', 'allow_users_remote_self', $allow_users_remote_self);
+		$transactionConfig->set('system', 'allow_relay_channels'   , $allow_relay_channels);
 		$transactionConfig->set('system', 'adjust_poll_frequency'  , $adjust_poll_frequency);
 		$transactionConfig->set('system', 'min_poll_interval'      , $min_poll_interval);
 		$transactionConfig->set('system', 'explicit_content'       , $explicit_content);
@@ -514,6 +516,7 @@ class Site extends BaseAdmin
 			'$blocked_tags'           => ['blocked_tags', DI::l10n()->t('Blocked tags for trending tags'), DI::config()->get('system', 'blocked_tags'), DI::l10n()->t("Comma separated list of hashtags that shouldn't be displayed in the trending tags.")],
 			'$cache_contact_avatar'   => ['cache_contact_avatar', DI::l10n()->t('Cache contact avatars'), DI::config()->get('system', 'cache_contact_avatar'), DI::l10n()->t('Locally store the avatar pictures of the contacts. This uses a lot of storage space but it increases the performance.')],
 			'$allow_users_remote_self'=> ['allow_users_remote_self', DI::l10n()->t('Allow Users to set remote_self'), DI::config()->get('system', 'allow_users_remote_self'), DI::l10n()->t('With checking this, every user is allowed to mark every contact as a remote_self in the repair contact dialog. Setting this flag on a contact causes mirroring every posting of that contact in the users stream.')],
+			'$allow_relay_channels'   => ['allow_relay_channels', DI::l10n()->t('Allow Users to set up relay channels'), DI::config()->get('system', 'allow_relay_channels'), DI::l10n()->t('If enabled, it is possible to create relay users that are used to reshare content based on user defined channels.')],
 			'$adjust_poll_frequency'  => ['adjust_poll_frequency', DI::l10n()->t('Adjust the feed poll frequency'), DI::config()->get('system', 'adjust_poll_frequency'), DI::l10n()->t('Automatically detect and set the best feed poll frequency.')],
 			'$min_poll_interval'      => ['min_poll_interval', DI::l10n()->t('Minimum poll interval'), DI::config()->get('system', 'min_poll_interval'), DI::l10n()->t('Minimal distance in minutes between two polls for mail and feed contacts. Reasonable values are between 1 and 59.')],
 			'$enable_multi_reg'       => ['enable_multi_reg', DI::l10n()->t('Enable multiple registrations'), !DI::config()->get('system', 'block_extended_register'), DI::l10n()->t('Enable users to register additional accounts for use as pages.')],

src/Module/Conversation/Timeline.php

@@ -436,7 +436,7 @@ class Timeline extends BaseModule
 			$condition[]  = $language;
 		}
 		if (!empty($conditions)) {
-			$condition[0] .= " AND (`language` IS NULL OR " . implode(' OR ', $conditions) . ")";
+			$condition[0] .= " AND (" . implode(' OR ', $conditions) . ")";
 		}
 		return $condition;
 	}

src/Module/Settings/Account.php

@@ -316,6 +316,8 @@ class Account extends BaseSettings
 				$page_flags = User::PAGE_FLAGS_SOAPBOX;
 			} elseif ($account_type == User::ACCOUNT_TYPE_COMMUNITY && !in_array($page_flags, [User::PAGE_FLAGS_COMMUNITY, User::PAGE_FLAGS_PRVGROUP])) {
 				$page_flags = User::PAGE_FLAGS_COMMUNITY;
+			} elseif ($account_type == User::ACCOUNT_TYPE_RELAY && $page_flags != User::PAGE_FLAGS_SOAPBOX) {
+				$page_flags = User::PAGE_FLAGS_SOAPBOX;
 			}
 
 			$fields = [
@@ -423,6 +425,18 @@ class Account extends BaseSettings
 			$user['account-type'] = User::ACCOUNT_TYPE_COMMUNITY;
 		}
 
+		if (DI::config()->get('system', 'allow_relay_channels')) {
+			$account_relay = [
+				'account-type',
+				DI::l10n()->t('Channel Relay'),
+				User::ACCOUNT_TYPE_RELAY,
+				DI::l10n()->t('Account for a service that automatically shares content based on user defined channels.'),
+				$user['account-type'] == User::ACCOUNT_TYPE_RELAY
+			];
+		} else {
+			$account_relay = null;
+		}
+
 		$pageset_tpl = Renderer::getMarkupTemplate('settings/pagetypes.tpl');
 		$pagetype    = Renderer::replaceMacros($pageset_tpl, [
 			'$account_types'     => DI::l10n()->t("Account Types"),
@@ -433,6 +447,7 @@ class Account extends BaseSettings
 			'$type_organisation' => User::ACCOUNT_TYPE_ORGANISATION,
 			'$type_news'         => User::ACCOUNT_TYPE_NEWS,
 			'$type_community'    => User::ACCOUNT_TYPE_COMMUNITY,
+			'$type_relay'        => User::ACCOUNT_TYPE_RELAY,
 			'$account_person'    => [
 				'account-type',
 				DI::l10n()->t('Personal Page'),
@@ -461,6 +476,7 @@ class Account extends BaseSettings
 				DI::l10n()->t('Account for community discussions.'),
 				$user['account-type'] == User::ACCOUNT_TYPE_COMMUNITY
 			],
+			'$account_relay' => $account_relay,
 			'$page_normal' => [
 				'page-flags',
 				DI::l10n()->t('Normal Account Page'),

src/Module/Settings/Channels.php

@@ -24,6 +24,7 @@ namespace Friendica\Module\Settings;
 use Friendica\App;
 use Friendica\Content\Conversation\Factory;
 use Friendica\Content\Conversation\Repository\UserDefinedChannel;
+use Friendica\Core\Config\Capability\IManageConfigValues;
 use Friendica\Core\L10n;
 use Friendica\Core\Renderer;
 use Friendica\Core\Session\Capability\IHandleUserSessions;
@@ -41,13 +42,16 @@ class Channels extends BaseSettings
 	private $channel;
 	/** @var Factory\UserDefinedChannel */
 	private $userDefinedChannel;
+	/** @var IManageConfigValues */
+	private $config;
 
-	public function __construct(Factory\UserDefinedChannel $userDefinedChannel, UserDefinedChannel $channel, App\Page $page, IHandleUserSessions $session, L10n $l10n, App\BaseURL $baseUrl, App\Arguments $args, LoggerInterface $logger, Profiler $profiler, Response $response, array $server, array $parameters = [])
+	public function __construct(Factory\UserDefinedChannel $userDefinedChannel, UserDefinedChannel $channel, App\Page $page, IHandleUserSessions $session, L10n $l10n, App\BaseURL $baseUrl, App\Arguments $args, LoggerInterface $logger, Profiler $profiler, Response $response, IManageConfigValues $config, array $server, array $parameters = [])
 	{
 		parent::__construct($session, $page, $l10n, $baseUrl, $args, $logger, $profiler, $response, $server, $parameters);
 
 		$this->userDefinedChannel = $userDefinedChannel;
 		$this->channel            = $channel;
+		$this->config             = $config;
 	}
 
 	protected function post(array $request = [])
@@ -110,6 +114,7 @@ class Channels extends BaseSettings
 				'full-text-search' => $request['text_search'][$id],
 				'media-type'       => ($request['image'][$id] ? 1 : 0) | ($request['video'][$id] ? 2 : 0) | ($request['audio'][$id] ? 4 : 0),
 				'languages'        => $request['languages'][$id],
+				'publish'          => $request['publish'][$id] ?? false,
 			]);
 			$saved = $this->channel->save($channel);
 			$this->logger->debug('Save channel', ['id' => $id, 'saved' => $saved]);
@@ -125,12 +130,23 @@ class Channels extends BaseSettings
 			throw new HTTPException\ForbiddenException($this->t('Permission denied.'));
 		}
 
+		$user = User::getById($uid, ['account-type']);
+		$account_type = $user['account-type'];
+
+		if (in_array($account_type, [User::ACCOUNT_TYPE_COMMUNITY, User::ACCOUNT_TYPE_RELAY])) {
+			$intro   = $this->t('This page can be used to define the channels that will automatically be reshared by your account.');
+			$circles = [
+				0 => $this->l10n->t('Global Community')
+			];
+		} else {
+			$intro   = $this->t('This page can be used to define your own channels.');
 			$circles = [
 				0  => $this->l10n->t('Global Community'),
 				-3 => $this->l10n->t('Network'),
 				-1 => $this->l10n->t('Following'),
 				-2 => $this->l10n->t('Followers'),
 			];
+		}
 
 		foreach (Circle::getByUserId($uid) as $circle) {
 			$circles[$circle['id']] = $circle['name'];
@@ -149,6 +165,12 @@ class Channels extends BaseSettings
 				$open = false;
 			}
 
+			if ($this->config->get('system', 'allow_relay_channels') && in_array($account_type, [User::ACCOUNT_TYPE_COMMUNITY, User::ACCOUNT_TYPE_RELAY])) {
+				$publish = ["publish[$channel->code]", $this->t("Publish"), $channel->publish, $this->t("When selected, the channel results are reshared. This only works for public ActivityPub posts from the public timeline or the user defined circles.")];
+			} else {
+				$publish = null;
+			}
+
 			$channels[] = [
 				'id'           => $channel->code,
 				'open'         => $open,
@@ -163,6 +185,7 @@ class Channels extends BaseSettings
 				'video'        => ["video[$channel->code]", $this->t("Videos"), $channel->mediaType & 2],
 				'audio'        => ["audio[$channel->code]", $this->t("Audio"), $channel->mediaType & 4],
 				'languages'    => ["languages[$channel->code][]", $this->t('Languages'), $channel->languages ?? $channel_languages, $this->t('Select all languages that you want to see in this channel.'), $languages, 'multiple'],
+				'publish'      => $publish,
 				'delete'       => ["delete[$channel->code]", $this->t("Delete channel") . ' (' . $channel->label . ')', false, $this->t("Check to delete this entry from the channel list")]
 			];
 		}
@@ -183,7 +206,7 @@ class Channels extends BaseSettings
 			'languages'    => ["new_languages[]", $this->t('Languages'), $channel_languages, $this->t('Select all languages that you want to see in this channel.'), $languages, 'multiple'],
 			'$l10n'        => [
 				'title'          => $this->t('Channels'),
-				'intro'          => $this->t('This page can be used to define your own channels.'),
+				'intro'          => $intro,
 				'addtitle'       => $this->t('Add new entry to the channel list'),
 				'addsubmit'      => $this->t('Add'),
 				'savechanges'    => $this->t('Save'),

src/Protocol/Relay.php

@@ -23,7 +23,7 @@ namespace Friendica\Protocol;
 
 use Friendica\Content\Smilies;
 use Friendica\Content\Text\BBCode;
-use Friendica\Core\Cache\Enum\Duration;
+use Friendica\Core\L10n;
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
 use Friendica\Database\DBA;
@@ -157,20 +157,16 @@ class Relay
 	 */
 	public static function getSubscribedTags(): array
 	{
-		$systemTags  = [];
+		$tags  = [];
-		$server_tags = DI::config()->get('system', 'relay_server_tags');
+		foreach (explode(',', mb_strtolower(DI::config()->get('system', 'relay_server_tags'))) as $tag) {
-
+			$tags[] = trim($tag, '# ');
-		foreach (explode(',', mb_strtolower($server_tags)) as $tag) {
-			$systemTags[] = trim($tag, '# ');
 		}
 
 		if (DI::config()->get('system', 'relay_user_tags')) {
-			$userTags = Search::getUserTags();
+			$tags = array_merge($tags, Search::getUserTags());
-		} else {
-			$userTags = [];
 		}
 
-		return array_unique(array_merge($systemTags, $userTags));
+		return array_unique($tags);
 	}
 
 	/**
@@ -192,12 +188,15 @@ class Relay
 			}
 		}
 
-		if (empty($languages) && empty($detected) && (empty($body) || Smilies::isEmojiPost($body))) {
+		if (empty($detected) && empty($languages)) {
+			$detected = [L10n::UNDETERMINED_LANGUAGE];
+		}
+
+		if (empty($body) || Smilies::isEmojiPost($body)) {
 			Logger::debug('Empty body or only emojis', ['body' => $body]);
 			return true;
 		}
 
-		if (!empty($languages) || !empty($detected)) {
 		$user_languages = User::getLanguages();
 
 		foreach ($detected as $language) {
@@ -214,12 +213,6 @@ class Relay
 		}
 		Logger::debug('No wanted language found', ['languages' => $languages, 'detected' => $detected, 'userlang' => $user_languages, 'body' => $body]);
 		return false;
-		} elseif (DI::config()->get('system', 'relay_deny_undetected_language')) {
-			Logger::info('Undetected language found', ['body' => $body]);
-			return false;
-		}
-
-		return true;
 	}
 
 	/**

static/dbstructure.config.php

@@ -56,7 +56,7 @@ use Friendica\Database\DBA;
 
 // This file is required several times during the test in DbaDefinition which justifies this condition
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1545);
+	define('DB_UPDATE_VERSION', 1546);
 }
 
 return [
@@ -563,6 +563,8 @@ return [
 			"full-text-search" => ["type" => "varchar(1023)", "comment" => "Full text search pattern, see https://mariadb.com/kb/en/full-text-index-overview/#in-boolean-mode"],
 			"media-type" => ["type" => "smallint unsigned", "comment" => "Filtered media types"],
 			"languages" => ["type" => "mediumtext", "comment" => "Desired languages"],
+			"publish" => ["type" => "boolean", "comment" => "publish channel content"],
+			"valid" => ["type" => "boolean", "comment" => "Set, when the full-text-search is valid"],
 		],
 		"indexes" => [
 			"PRIMARY" => ["id"],
@@ -1364,7 +1366,7 @@ return [
 			"owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "foreign" => ["contact" => "id"], "comment" => "Item owner"],
 			"contact-type" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Person, organisation, news, community, relay"],
 			"media-type" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Type of media in a bit array (1 = image, 2 = video, 4 = audio"],
-			"language" => ["type" => "varbinary(128)", "comment" => "Language information about this post"],
+			"language" => ["type" => "varchar(128)", "comment" => "Language information about this post"],
 			"searchtext" => ["type" => "mediumtext", "comment" => "Simplified text for the full text search"],
 			"created" => ["type" => "datetime", "comment" => ""],
 			"restricted" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "If true, this post is either unlisted or not from a federated network"],

static/settings.config.php

@@ -56,6 +56,10 @@ return [
 		// Automatically detect and set the best feed poll frequency.
 		'adjust_poll_frequency' => false,
 
+		// allow_relay_channels (Boolean)
+		// Allow Users to set remote_self
+		'allow_relay_channels' => true,
+
 		// allowed_themes (Comma-separated list)
 		// Themes users can change to in their settings.
 		'allowed_themes' => 'frio,vier',

view/templates/admin/site.tpl

@@ -84,6 +84,7 @@
 		{{include file="field_checkbox.tpl" field=$private_addons}}
 		{{include file="field_checkbox.tpl" field=$disable_embedded}}
 		{{include file="field_checkbox.tpl" field=$allow_users_remote_self}}
+		{{include file="field_checkbox.tpl" field=$allow_relay_channels}}
 		{{include file="field_checkbox.tpl" field=$adjust_poll_frequency}}
 		{{include file="field_checkbox.tpl" field=$explicit_content}}
 		{{include file="field_checkbox.tpl" field=$proxify_content}}

view/templates/settings/channels.tpl

@@ -36,6 +36,9 @@
 			{{include file="field_checkbox.tpl" field=$e.video}}
 			{{include file="field_checkbox.tpl" field=$e.audio}}
 			{{include file="field_select.tpl" field=$e.languages}}
+			{{if $e.publish}}
+				{{include file="field_checkbox.tpl" field=$e.publish}}
+			{{/if}}
 			{{include file="field_checkbox.tpl" field=$e.delete}}
 			<hr>
 		{{/foreach}}

view/templates/settings/pagetypes.tpl

@@ -18,6 +18,10 @@
 	{{include file="field_radio.tpl" field=$page_prvgroup}}
 </div>
 
+{{if $account_relay}}
+	{{include file="field_radio.tpl" field=$account_relay}}
+{{/if}}
+
 <script language="javascript" type="text/javascript">
 	// This js part changes the state of page-flags radio buttons according
 	// to the selected account type.

view/theme/frio/templates/admin/site.tpl

@@ -164,6 +164,7 @@
 						{{include file="field_checkbox.tpl" field=$private_addons}}
 						{{include file="field_checkbox.tpl" field=$disable_embedded}}
 						{{include file="field_checkbox.tpl" field=$allow_users_remote_self}}
+						{{include file="field_checkbox.tpl" field=$allow_relay_channels}}
 						{{include file="field_checkbox.tpl" field=$adjust_poll_frequency}}
 						{{include file="field_checkbox.tpl" field=$explicit_content}}
 						{{include file="field_checkbox.tpl" field=$proxify_content}}

view/theme/frio/templates/settings/channels.tpl

@@ -53,6 +53,9 @@
 						{{include file="field_checkbox.tpl" field=$e.video}}
 						{{include file="field_checkbox.tpl" field=$e.audio}}
 						{{include file="field_select.tpl" field=$e.languages}}
+						{{if $e.publish}}
+							{{include file="field_checkbox.tpl" field=$e.publish}}
+						{{/if}}
 						{{include file="field_checkbox.tpl" field=$e.delete}}
 						<div class="submit">
 							<button type="submit" class="btn btn-primary" name="edit_channel" value="{{$l10n.savechanges}}">{{$l10n.savechanges}}</button>