From 3fe4991fcf14642c3748c0e7d2760bb664712263 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 30 Jan 2024 10:05:05 +0000 Subject: [PATCH] Filter user defined channels by size --- database.sql | 5 ++++- doc/database/db_channel.md | 2 ++ doc/database/db_post-engagement.md | 1 + src/Content/Conversation/Entity/Timeline.php | 10 ++++++++- .../Factory/UserDefinedChannel.php | 2 ++ .../Repository/UserDefinedChannel.php | 2 ++ src/Model/Post/Engagement.php | 22 +++++++++++++++++-- src/Module/Conversation/Timeline.php | 8 +++++++ src/Module/Settings/Channels.php | 8 +++++++ static/dbstructure.config.php | 5 ++++- view/templates/settings/channels.tpl | 4 ++++ .../frio/templates/settings/channels.tpl | 4 ++++ 12 files changed, 68 insertions(+), 5 deletions(-) diff --git a/database.sql b/database.sql index 3e74e28dcb..da167e2c41 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2024.03-dev (Yellow Archangel) --- DB_UPDATE_VERSION 1548 +-- DB_UPDATE_VERSION 1549 -- ------------------------------------------ @@ -502,6 +502,8 @@ CREATE TABLE IF NOT EXISTS `channel` ( `access-key` varchar(1) COMMENT 'Access key', `include-tags` varchar(1023) COMMENT 'Comma separated list of tags that will be included in the channel', `exclude-tags` varchar(1023) COMMENT 'Comma separated list of tags that aren\'t allowed in the channel', + `min-size` int unsigned COMMENT 'Minimum post size', + `max-size` int unsigned COMMENT 'Maximum post size', `full-text-search` varchar(1023) COMMENT 'Full text search pattern, see https://mariadb.com/kb/en/full-text-index-overview/#in-boolean-mode', `media-type` smallint unsigned COMMENT 'Filtered media types', `languages` mediumtext COMMENT 'Desired languages', @@ -1346,6 +1348,7 @@ CREATE TABLE IF NOT EXISTS `post-engagement` ( `media-type` tinyint NOT NULL DEFAULT 0 COMMENT 'Type of media in a bit array (1 = image, 2 = video, 4 = audio', `language` varchar(128) COMMENT 'Language information about this post', `searchtext` mediumtext COMMENT 'Simplified text for the full text search', + `size` int unsigned COMMENT 'Body size', `created` datetime COMMENT '', `restricted` boolean NOT NULL DEFAULT '0' COMMENT 'If true, this post is either unlisted or not from a federated network', `comments` mediumint unsigned COMMENT 'Number of comments', diff --git a/doc/database/db_channel.md b/doc/database/db_channel.md index 5b0636dc80..4a469b4ee3 100644 --- a/doc/database/db_channel.md +++ b/doc/database/db_channel.md @@ -16,6 +16,8 @@ Fields | access-key | Access key | varchar(1) | YES | | NULL | | | include-tags | Comma separated list of tags that will be included in the channel | varchar(1023) | YES | | NULL | | | exclude-tags | Comma separated list of tags that aren't allowed in the channel | varchar(1023) | YES | | NULL | | +| min-size | Minimum post size | int unsigned | YES | | NULL | | +| max-size | Maximum post size | int unsigned | YES | | NULL | | | full-text-search | Full text search pattern, see https://mariadb.com/kb/en/full-text-index-overview/#in-boolean-mode | varchar(1023) | YES | | NULL | | | media-type | Filtered media types | smallint unsigned | YES | | NULL | | | languages | Desired languages | mediumtext | YES | | NULL | | diff --git a/doc/database/db_post-engagement.md b/doc/database/db_post-engagement.md index e63f170b76..027ae56a79 100644 --- a/doc/database/db_post-engagement.md +++ b/doc/database/db_post-engagement.md @@ -14,6 +14,7 @@ Fields | media-type | Type of media in a bit array (1 = image, 2 = video, 4 = audio | tinyint | NO | | 0 | | | language | Language information about this post | varchar(128) | YES | | NULL | | | searchtext | Simplified text for the full text search | mediumtext | YES | | NULL | | +| size | Body size | int unsigned | YES | | NULL | | | created | | datetime | YES | | NULL | | | restricted | If true, this post is either unlisted or not from a federated network | boolean | NO | | 0 | | | comments | Number of comments | mediumint unsigned | YES | | NULL | | diff --git a/src/Content/Conversation/Entity/Timeline.php b/src/Content/Conversation/Entity/Timeline.php index a27d9fb98f..47d6e8db47 100644 --- a/src/Content/Conversation/Entity/Timeline.php +++ b/src/Content/Conversation/Entity/Timeline.php @@ -30,6 +30,8 @@ namespace Friendica\Content\Conversation\Entity; * @property-read int $uid User of the channel * @property-read string $includeTags The tags to include in the channel * @property-read string $excludeTags The tags to exclude in the channel + * @property-read int $minSize Minimum content size + * @property-read int $maxSize Maximum content size * @property-read string $fullTextSearch full text search pattern * @property-read int $mediaType Media types that are included in the channel * @property-read array $languages Channel languages @@ -57,6 +59,10 @@ class Timeline extends \Friendica\BaseEntity protected $includeTags; /** @var string */ protected $excludeTags; + /** @var int */ + protected $minSize; + /** @var int */ + protected $maxSize; /** @var string */ protected $fullTextSearch; /** @var int */ @@ -68,7 +74,7 @@ class Timeline extends \Friendica\BaseEntity /** @var bool */ protected $valid; - public function __construct(string $code = null, string $label = null, string $description = null, string $accessKey = null, string $path = null, int $uid = null, string $includeTags = null, string $excludeTags = null, string $fullTextSearch = null, int $mediaType = null, int $circle = null, array $languages = null, bool $publish = null, bool $valid = null) + public function __construct(string $code = null, string $label = null, string $description = null, string $accessKey = null, string $path = null, int $uid = null, string $includeTags = null, string $excludeTags = null, string $fullTextSearch = null, int $mediaType = null, int $circle = null, array $languages = null, bool $publish = null, bool $valid = null, int $minSize = null, int $maxSize = null) { $this->code = $code; $this->label = $label; @@ -78,6 +84,8 @@ class Timeline extends \Friendica\BaseEntity $this->uid = $uid; $this->includeTags = $includeTags; $this->excludeTags = $excludeTags; + $this->minSize = $minSize; + $this->maxSize = $maxSize; $this->fullTextSearch = $fullTextSearch; $this->mediaType = $mediaType; $this->circle = $circle; diff --git a/src/Content/Conversation/Factory/UserDefinedChannel.php b/src/Content/Conversation/Factory/UserDefinedChannel.php index 08a092205c..f3b5f65a14 100644 --- a/src/Content/Conversation/Factory/UserDefinedChannel.php +++ b/src/Content/Conversation/Factory/UserDefinedChannel.php @@ -52,6 +52,8 @@ final class UserDefinedChannel extends Timeline implements ICanCreateFromTableRo $row['languages'] ?? null, $row['publish'] ?? null, $row['valid'] ?? null, + $row['min-size'] ?? null, + $row['max-size'] ?? null, ); } } diff --git a/src/Content/Conversation/Repository/UserDefinedChannel.php b/src/Content/Conversation/Repository/UserDefinedChannel.php index c47991ac4e..6515bc7196 100644 --- a/src/Content/Conversation/Repository/UserDefinedChannel.php +++ b/src/Content/Conversation/Repository/UserDefinedChannel.php @@ -130,6 +130,8 @@ class UserDefinedChannel extends \Friendica\BaseRepository 'circle' => $Channel->circle, 'include-tags' => $Channel->includeTags, 'exclude-tags' => $Channel->excludeTags, + 'min-size' => $Channel->minSize, + 'max-size' => $Channel->maxSize, 'full-text-search' => $Channel->fullTextSearch, 'media-type' => $Channel->mediaType, 'languages' => serialize($Channel->languages), diff --git a/src/Model/Post/Engagement.php b/src/Model/Post/Engagement.php index 64f796ccc7..e787aec408 100644 --- a/src/Model/Post/Engagement.php +++ b/src/Model/Post/Engagement.php @@ -38,7 +38,8 @@ use Friendica\Util\DateTimeFormat; class Engagement { - const KEYWORDS = ['source', 'server', 'from', 'to', 'group', 'application', 'tag', 'network', 'platform', 'visibility', 'language']; + const KEYWORDS = ['source', 'server', 'from', 'to', 'group', 'application', 'tag', 'network', 'platform', 'visibility', 'language']; + const SHORTCUTS = ['lang' => 'language', 'net' => 'network', 'relay' => 'application']; /** * Store engagement data from an item array @@ -101,6 +102,7 @@ class Engagement 'media-type' => $mediatype, 'language' => $parent['language'], 'searchtext' => $searchtext, + 'size' => self::getContentSize($parent), 'created' => $parent['created'], 'restricted' => !in_array($item['network'], Protocol::FEDERATED) || ($parent['private'] != Item::PUBLIC), 'comments' => DBA::count('post', ['parent-uri-id' => $item['parent-uri-id'], 'gravity' => Item::GRAVITY_COMMENT]), @@ -125,6 +127,18 @@ class Engagement return ($ret && !$exists) ? $engagement['uri-id'] : 0; } + private static function getContentSize(array $item): int + { + $body = ' ' . $item['title'] . ' ' . $item['content-warning'] . ' ' . $item['body']; + $body = BBCode::removeAttachment($body); + $body = BBCode::removeSharedData($body); + $body = preg_replace('/[^@!#]\[url\=.*?\].*?\[\/url\]/ism', '', $body); + $body = BBCode::removeLinks($body); + $msg = BBCode::toPlaintext($body, false); + + return mb_strlen($msg); + } + public static function getSearchTextForActivity(string $content, int $author_id, array $tags, array $receivers): string { $author = Contact::getById($author_id); @@ -347,7 +361,11 @@ class Engagement public static function escapeKeywords(string $fullTextSearch): string { - foreach (Engagement::KEYWORDS as $keyword) { + foreach (SELF::SHORTCUTS as $search => $replace) { + $fullTextSearch = preg_replace('~' . $search . ':(.[\w\*@\.-]+)~', $replace . ':$1', $fullTextSearch); + } + + foreach (self::KEYWORDS as $keyword) { $fullTextSearch = preg_replace('~(' . $keyword . '):(.[\w\*@\.-]+)~', '"$1_$2"', $fullTextSearch); } return $fullTextSearch; diff --git a/src/Module/Conversation/Timeline.php b/src/Module/Conversation/Timeline.php index 1b375d099d..d0f8f09736 100644 --- a/src/Module/Conversation/Timeline.php +++ b/src/Module/Conversation/Timeline.php @@ -427,6 +427,14 @@ class Timeline extends BaseModule $condition = DBA::mergeConditions($condition, array_merge(["NOT `uri-id` IN (SELECT `uri-id` FROM `post-tag` INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid` WHERE `post-tag`.`type` = 1 AND `name` IN (" . $placeholders . "))"], $search)); } + if (!is_null($channel->minSize)) { + $condition = DBA::mergeConditions($condition, ["`size` >= ?", $channel->minSize]); + } + + if (!is_null($channel->maxSize)) { + $condition = DBA::mergeConditions($condition, ["`size` <= ?", $channel->maxSize]); + } + if (!empty($channel->mediaType)) { $condition = DBA::mergeConditions($condition, ["`media-type` & ?", $channel->mediaType]); } diff --git a/src/Module/Settings/Channels.php b/src/Module/Settings/Channels.php index 34ba0d1d80..a1b5949ede 100644 --- a/src/Module/Settings/Channels.php +++ b/src/Module/Settings/Channels.php @@ -83,6 +83,8 @@ class Channels extends BaseSettings 'circle' => (int)$request['new_circle'], 'include-tags' => Strings::cleanTags($request['new_include_tags']), 'exclude-tags' => Strings::cleanTags($request['new_exclude_tags']), + 'min-size' => $request['new_min_size'] != '' ? (int)$request['new_min_size'] : null, + 'max-size' => $request['new_max_size'] != '' ? (int)$request['new_max_size'] : null, 'full-text-search' => $request['new_text_search'], 'media-type' => ($request['new_image'] ? 1 : 0) | ($request['new_video'] ? 2 : 0) | ($request['new_audio'] ? 4 : 0), 'languages' => $request['new_languages'], @@ -112,6 +114,8 @@ class Channels extends BaseSettings 'circle' => (int)$request['circle'][$id], 'include-tags' => Strings::cleanTags($request['include_tags'][$id]), 'exclude-tags' => Strings::cleanTags($request['exclude_tags'][$id]), + 'min-size' => $request['min_size'][$id] != '' ? (int)$request['min_size'][$id] : null, + 'max-size' => $request['max_size'][$id] != '' ? (int)$request['max_size'][$id] : null, 'full-text-search' => $request['text_search'][$id], 'media-type' => ($request['image'][$id] ? 1 : 0) | ($request['video'][$id] ? 2 : 0) | ($request['audio'][$id] ? 4 : 0), 'languages' => $request['languages'][$id], @@ -181,6 +185,8 @@ class Channels extends BaseSettings 'circle' => ["circle[$channel->code]", $this->t('Circle/Channel'), $channel->circle, '', $circles], 'include_tags' => ["include_tags[$channel->code]", $this->t("Include Tags"), str_replace(',', ', ', $channel->includeTags)], 'exclude_tags' => ["exclude_tags[$channel->code]", $this->t("Exclude Tags"), str_replace(',', ', ', $channel->excludeTags)], + 'min_size' => ["min_size[$channel->code]", $this->t("Minimum Size"), $channel->minSize], + 'max_size' => ["max_size[$channel->code]", $this->t("Maximum Size"), $channel->maxSize], 'text_search' => ["text_search[$channel->code]", $this->t("Full Text Search"), $channel->fullTextSearch], 'image' => ["image[$channel->code]", $this->t("Images"), $channel->mediaType & 1], 'video' => ["video[$channel->code]", $this->t("Videos"), $channel->mediaType & 2], @@ -200,6 +206,8 @@ class Channels extends BaseSettings 'circle' => ['new_circle', $this->t('Circle/Channel'), 0, $this->t('Select a circle or channel, that your channel should be based on.'), $circles], 'include_tags' => ["new_include_tags", $this->t("Include Tags"), '', $this->t('Comma separated list of tags. A post will be used when it contains any of the listed tags.')], 'exclude_tags' => ["new_exclude_tags", $this->t("Exclude Tags"), '', $this->t('Comma separated list of tags. If a post contain any of these tags, then it will not be part of nthis channel.')], + 'min_size' => ["new_min_size", $this->t("Minimum Size"), '', $this->t('Minimum post size. Leave empty for no minimum size. The size is calculated without links, attached posts, mentions or hashtags.')], + 'max_size' => ["new_max_size", $this->t("Maximum Size"), '', $this->t('Maximum post size. Leave empty for no maximum size. The size is calculated without links, attached posts, mentions or hashtags.')], 'text_search' => ["new_text_search", $this->t("Full Text Search"), '', $this->t('Search terms for the body, supports the "boolean mode" operators from MariaDB. See the help for a complete list of operators and additional keywords: %s', 'help/Channels')], 'image' => ['new_image', $this->t("Images"), false, $this->t("Check to display images in the channel.")], 'video' => ["new_video", $this->t("Videos"), false, $this->t("Check to display videos in the channel.")], diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 5595a752b2..a75403fcd9 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -56,7 +56,7 @@ use Friendica\Database\DBA; // This file is required several times during the test in DbaDefinition which justifies this condition if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1548); + define('DB_UPDATE_VERSION', 1549); } return [ @@ -560,6 +560,8 @@ return [ "access-key" => ["type" => "varchar(1)", "comment" => "Access key"], "include-tags" => ["type" => "varchar(1023)", "comment" => "Comma separated list of tags that will be included in the channel"], "exclude-tags" => ["type" => "varchar(1023)", "comment" => "Comma separated list of tags that aren't allowed in the channel"], + "min-size" => ["type" => "int unsigned", "comment" => "Minimum post size"], + "max-size" => ["type" => "int unsigned", "comment" => "Maximum post size"], "full-text-search" => ["type" => "varchar(1023)", "comment" => "Full text search pattern, see https://mariadb.com/kb/en/full-text-index-overview/#in-boolean-mode"], "media-type" => ["type" => "smallint unsigned", "comment" => "Filtered media types"], "languages" => ["type" => "mediumtext", "comment" => "Desired languages"], @@ -1367,6 +1369,7 @@ return [ "media-type" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Type of media in a bit array (1 = image, 2 = video, 4 = audio"], "language" => ["type" => "varchar(128)", "comment" => "Language information about this post"], "searchtext" => ["type" => "mediumtext", "comment" => "Simplified text for the full text search"], + "size" => ["type" => "int unsigned", "comment" => "Body size"], "created" => ["type" => "datetime", "comment" => ""], "restricted" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "If true, this post is either unlisted or not from a federated network"], "comments" => ["type" => "mediumint unsigned", "comment" => "Number of comments"], diff --git a/view/templates/settings/channels.tpl b/view/templates/settings/channels.tpl index 44260a2336..903989cff6 100644 --- a/view/templates/settings/channels.tpl +++ b/view/templates/settings/channels.tpl @@ -11,6 +11,8 @@ {{include file="field_textarea.tpl" field=$include_tags}} {{include file="field_textarea.tpl" field=$exclude_tags}} {{include file="field_textarea.tpl" field=$text_search}} + {{include file="field_input.tpl" field=$min_size}} + {{include file="field_input.tpl" field=$max_size}} {{include file="field_checkbox.tpl" field=$image}} {{include file="field_checkbox.tpl" field=$video}} {{include file="field_checkbox.tpl" field=$audio}} @@ -31,6 +33,8 @@ {{include file="field_select.tpl" field=$e.circle}} {{include file="field_textarea.tpl" field=$e.include_tags}} {{include file="field_textarea.tpl" field=$e.exclude_tags}} + {{include file="field_input.tpl" field=$e.min_size}} + {{include file="field_input.tpl" field=$e.max_size}} {{include file="field_textarea.tpl" field=$e.text_search}} {{include file="field_checkbox.tpl" field=$e.image}} {{include file="field_checkbox.tpl" field=$e.video}} diff --git a/view/theme/frio/templates/settings/channels.tpl b/view/theme/frio/templates/settings/channels.tpl index 932783e1dc..1ec246978c 100644 --- a/view/theme/frio/templates/settings/channels.tpl +++ b/view/theme/frio/templates/settings/channels.tpl @@ -18,6 +18,8 @@ {{include file="field_select.tpl" field=$circle}} {{include file="field_textarea.tpl" field=$include_tags}} {{include file="field_textarea.tpl" field=$exclude_tags}} + {{include file="field_input.tpl" field=$min_size}} + {{include file="field_input.tpl" field=$max_size}} {{include file="field_textarea.tpl" field=$text_search}} {{include file="field_checkbox.tpl" field=$image}} {{include file="field_checkbox.tpl" field=$video}} @@ -48,6 +50,8 @@ {{include file="field_select.tpl" field=$e.circle}} {{include file="field_textarea.tpl" field=$e.include_tags}} {{include file="field_textarea.tpl" field=$e.exclude_tags}} + {{include file="field_input.tpl" field=$e.min_size}} + {{include file="field_input.tpl" field=$e.max_size}} {{include file="field_textarea.tpl" field=$e.text_search}} {{include file="field_checkbox.tpl" field=$e.image}} {{include file="field_checkbox.tpl" field=$e.video}}