diff --git a/mod/network.php b/mod/network.php index 9ec8c95b92..b6afda608b 100644 --- a/mod/network.php +++ b/mod/network.php @@ -37,8 +37,8 @@ use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Group; use Friendica\Model\Item; +use Friendica\Model\Post\Category; use Friendica\Model\Profile; -use Friendica\Model\Term; use Friendica\Module\Security\Login; use Friendica\Util\DateTimeFormat; use Friendica\Util\Proxy as ProxyUtils; @@ -379,25 +379,25 @@ function networkFlatView(App $a, $update = 0) networkPager($a, $pager, $update); - $item_params = ['order' => ['id' => true]]; if (strlen($file)) { - $term_condition = ["`term` = ? AND `otype` = ? AND `type` = ? AND `uid` = ?", - $file, Term::OBJECT_TYPE_POST, Term::FILE, local_user()]; + $item_params = ['order' => ['uri-id' => true]]; + $term_condition = ['name' => $file, 'type' => Category::FILE, 'uid' => local_user()]; $term_params = ['order' => ['tid' => true], 'limit' => [$pager->getStart(), $pager->getItemsPerPage()]]; - $result = DBA::select('term', ['oid'], $term_condition, $term_params); + $result = DBA::select('category-view', ['uri-id'], $term_condition, $term_params); $posts = []; while ($term = DBA::fetch($result)) { - $posts[] = $term['oid']; + $posts[] = $term['uri-id']; } DBA::close($result); if (count($posts) == 0) { return ''; } - $item_condition = ['uid' => local_user(), 'id' => $posts]; + $item_condition = ['uid' => local_user(), 'uri-id' => $posts]; } else { + $item_params = ['order' => ['id' => true]]; $item_condition = ['uid' => local_user()]; $item_params['limit'] = [$pager->getStart(), $pager->getItemsPerPage()]; diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 9ae7691d5f..9bce1f1f3c 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -28,7 +28,9 @@ use Friendica\Model\Contact; use Friendica\Model\Item; use Friendica\Model\ItemURI; use Friendica\Model\PermissionSet; +use Friendica\Model\Post\Category; use Friendica\Model\Tag; +use Friendica\Model\Term; use Friendica\Model\UserItem; use Friendica\Util\Strings; @@ -72,6 +74,9 @@ class PostUpdate if (!self::update1342()) { return false; } + if (!self::update1346()) { + return false; + } return true; } @@ -667,4 +672,66 @@ class PostUpdate return false; } + + /** + * Fill the "tag" table with tags and mentions from the "term" table + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + private static function update1346() + { + // Was the script completed? + if (DI::config()->get('system', 'post_update_version') >= 1346) { + return true; + } + + $id = DI::config()->get('system', 'post_update_version_1346_id', 0); + + Logger::info('Start', ['item' => $id]); + + $rows = 0; + + $terms = DBA::select('term', ['oid'], + ["`type` IN (?, ?) AND `oid` >= ?", Category::CATEGORY, Category::FILE, $id], + ['order' => ['oid'], 'limit' => 1000, 'group_by' => ['oid']]); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($term = DBA::fetch($terms)) { + $item = Item::selectFirst(['uri-id', 'uid'], ['id' => $term['oid']]); + if (!DBA::isResult($item)) { + continue; + } + + $file = Term::fileTextFromItemId($term['oid']); + if (!empty($file)) { + Category::storeTextByURIId($item['uri-id'], $item['uid'], $file); + } + + $id = $term['oid']; + ++$rows; + if ($rows % 100 == 0) { + DI::config()->set('system', 'post_update_version_1346_id', $id); + } + } + DBA::close($terms); + + DI::config()->set('system', 'post_update_version_1346_id', $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + // When there are less than 10 items processed this means that we reached the end + // The other entries will then be processed with the regular functionality + if ($rows < 10) { + DI::config()->set('system', 'post_update_version', 1346); + Logger::info('Done'); + return true; + } + + return false; + } } diff --git a/src/Model/FileTag.php b/src/Model/FileTag.php index 3539f2b96d..0b728e33d7 100644 --- a/src/Model/FileTag.php +++ b/src/Model/FileTag.php @@ -23,7 +23,7 @@ namespace Friendica\Model; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Model\Term; +use Friendica\Model\Post\Category; /** * This class handles FileTag related functions @@ -196,11 +196,11 @@ class FileTag if ($type == 'file') { $lbracket = '['; $rbracket = ']'; - $termtype = Term::FILE; + $termtype = Category::FILE; } else { $lbracket = '<'; $rbracket = '>'; - $termtype = Term::CATEGORY; + $termtype = Category::CATEGORY; } $filetags_updated = $saved; @@ -224,13 +224,7 @@ class FileTag } foreach ($deleted_tags as $key => $tag) { - $r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d", - DBA::escape($tag), - intval(Term::OBJECT_TYPE_POST), - intval($termtype), - intval($uid)); - - if (DBA::isResult($r)) { + if (DBA::exists('category-view', ['name' => $tag, 'type' => $termtype, 'uid' => $uid])) { unset($deleted_tags[$key]); } else { $filetags_updated = str_replace($lbracket . self::encode($tag) . $rbracket, '', $filetags_updated); @@ -303,10 +297,10 @@ class FileTag if ($cat == true) { $pattern = '<' . self::encode($file) . '>'; - $termtype = Term::CATEGORY; + $termtype = Category::CATEGORY; } else { $pattern = '[' . self::encode($file) . ']'; - $termtype = Term::FILE; + $termtype = Category::FILE; } $item = Item::selectFirst(['file'], ['id' => $item_id, 'uid' => $uid]); @@ -319,14 +313,7 @@ class FileTag Item::update($fields, ['id' => $item_id]); - $r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d", - DBA::escape($file), - intval(Term::OBJECT_TYPE_POST), - intval($termtype), - intval($uid) - ); - - if (!DBA::isResult($r)) { + if (!DBA::exists('category-view', ['name' => $file, 'type' => $termtype, 'uid' => $uid])) { $saved = DI::pConfig()->get($uid, 'system', 'filetags'); DI::pConfig()->set($uid, 'system', 'filetags', str_replace($pattern, '', $saved)); } diff --git a/src/Model/Item.php b/src/Model/Item.php index 1ef4810578..8c66e05f4f 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -32,6 +32,7 @@ use Friendica\Core\System; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Model\Post\Category; use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Diaspora; @@ -319,7 +320,7 @@ class Item if (!array_key_exists('verb', $row) || in_array($row['verb'], ['', Activity::POST, Activity::SHARE])) { // Build the file string out of the term entries if (array_key_exists('file', $row) && empty($row['file'])) { - $row['file'] = Term::fileTextFromItemId($row['internal-iid']); + $row['file'] = Category::getTextByURIId($row['internal-uri-id'], $row['internal-uid']); } } @@ -345,7 +346,8 @@ class Item // Remove internal fields unset($row['internal-activity']); unset($row['internal-network']); - unset($row['internal-iid']); + unset($row['internal-uri-id']); + unset($row['internal-uid']); unset($row['internal-psid']); unset($row['internal-iaid']); unset($row['internal-user-ignored']); @@ -671,7 +673,8 @@ class Item 'resource-id', 'event-id', 'attach', 'post-type', 'file', 'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark', 'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global', - 'id' => 'item_id', 'network', 'icid', 'iaid', 'id' => 'internal-iid', + 'id' => 'item_id', 'network', 'icid', 'iaid', + 'uri-id' => 'internal-uri-id', 'uid' => 'internal-uid', 'network' => 'internal-network', 'iaid' => 'internal-iaid', 'psid' => 'internal-psid']; if ($usermode) { @@ -834,7 +837,7 @@ class Item private static function constructSelectFields(array $fields, array $selected) { if (!empty($selected)) { - $selected = array_merge($selected, ['internal-iid', 'internal-psid', 'internal-iaid', 'internal-network']); + $selected = array_merge($selected, ['internal-uri-id', 'internal-uid', 'internal-psid', 'internal-iaid', 'internal-network']); } if (in_array('verb', $selected)) { @@ -917,7 +920,7 @@ class Item // We cannot simply expand the condition to check for origin entries // The condition needn't to be a simple array but could be a complex condition. // And we have to execute this query before the update to ensure to fetch the same data. - $items = DBA::select('item', ['id', 'origin', 'uri', 'uri-id', 'iaid', 'icid', 'file'], $condition); + $items = DBA::select('item', ['id', 'origin', 'uri', 'uri-id', 'iaid', 'icid', 'uid', 'file'], $condition); $content_fields = []; foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) { @@ -1013,7 +1016,7 @@ class Item } if (!is_null($files)) { - Term::insertFromFileFieldByItemId($item['id'], $files); + Category::storeTextByURIId($item['uri-id'], $item['uid'], $files); if (!empty($item['file'])) { DBA::update('item', ['file' => ''], ['id' => $item['id']]); } @@ -1099,7 +1102,7 @@ class Item { Logger::info('Mark item for deletion by id', ['id' => $item_id, 'callstack' => System::callstack()]); // locate item to be deleted - $fields = ['id', 'uri', 'uid', 'parent', 'parent-uri', 'origin', + $fields = ['id', 'uri', 'uri-id', 'uid', 'parent', 'parent-uri', 'origin', 'deleted', 'file', 'resource-id', 'event-id', 'attach', 'verb', 'object-type', 'object', 'target', 'contact-id', 'icid', 'iaid', 'psid']; @@ -1172,7 +1175,7 @@ class Item $item_fields = ['deleted' => true, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()]; DBA::update('item', $item_fields, ['id' => $item['id']]); - Term::insertFromFileFieldByItemId($item['id'], ''); + Category::storeTextByURIId($item['uri-id'], $item['uid'], ''); self::deleteThread($item['id'], $item['parent-uri']); if (!self::exists(["`uri` = ? AND `uid` != 0 AND NOT `deleted`", $item['uri']])) { @@ -1949,7 +1952,7 @@ class Item * This is not perfect - but a workable solution until we found the reason for the problem. */ if (!empty($files)) { - Term::insertFromFileFieldByItemId($current_post, $files); + Category::storeTextByURIId($item['uri-id'], $item['uid'], $files); } // In that function we check if this is a forum post. Additionally we delete the item under certain circumstances diff --git a/src/Model/Post/Category.php b/src/Model/Post/Category.php new file mode 100644 index 0000000000..680e32e3ad --- /dev/null +++ b/src/Model/Post/Category.php @@ -0,0 +1,119 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use Friendica\Database\DBA; +use Friendica\Model\Item; +use Friendica\Model\Tag; + +/** + * Class Category + * + * This Model class handles category table interactions. + * This tables stores user-applied categories related to posts. + */ +class Category +{ + const UNKNOWN = 0; + const CATEGORY = 3; + const FILE = 5; + + /** + * Generates the legacy item.file field string from an item ID. + * Includes only file and category terms. + * + * @param int $item_id + * @return string + * @throws \Exception + */ + public static function getTextByURIId(int $uri_id, int $uid) + { + $file_text = ''; + + $tags = DBA::selectToArray('post-category', ['type', 'name'], ['uri-id' => $uri_id, 'uid' => $uid]); + foreach ($tags as $tag) { + if ($tag['type'] == self::CATEGORY) { + $file_text .= '<' . $tag['term'] . '>'; + } else { + $file_text .= '[' . $tag['term'] . ']'; + } + } + + return $file_text; + } + + /** + * Inserts new terms for the provided item ID based on the legacy item.file field BBCode content. + * Deletes all previous file terms for the same item ID. + * + * @param integer $item_id item id + * @param $files + * @return void + * @throws \Exception + */ + public static function storeTextByURIId(int $uri_id, int $uid, string $files) + { + $message = Item::selectFirst(['deleted'], ['uri-id' => $uri_id, 'uid' => $uid]); + if (!DBA::isResult($message)) { + return; + } + + // Clean up all tags + DBA::delete('post-category', ['uri-id' => $uri_id, 'uid' => $uid]); + + if ($message['deleted']) { + return; + } + + if (preg_match_all("/\[(.*?)\]/ism", $files, $result)) { + foreach ($result[1] as $file) { + $tagid = Tag::getID($file); + if (empty($tagid)) { + continue; + } + + DBA::insert('post-category', [ + 'uri-id' => $uri_id, + 'uid' => $uid, + 'type' => self::FILE, + 'tid' => $tagid + ]); + } + } + + if (preg_match_all("/\<(.*?)\>/ism", $files, $result)) { + foreach ($result[1] as $file) { + $tagid = Tag::getID($file); + if (empty($tagid)) { + continue; + } + + DBA::insert('post-category', [ + 'uri-id' => $uri_id, + 'uid' => $uid, + 'type' => self::CATEGORY, + 'tid' => $tagid + ]); + } + } + } +} diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 0a06dcc250..9b4b1eb41b 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -124,22 +124,14 @@ class Tag } if (empty($cid)) { - $fields = ['name' => substr($name, 0, 96), 'url' => '']; - if (($type != self::HASHTAG) && !empty($url) && ($url != $name)) { - $fields['url'] = strtolower($url); - } - - $tag = DBA::selectFirst('tag', ['id'], $fields); - if (!DBA::isResult($tag)) { - DBA::insert('tag', $fields, true); - $tagid = DBA::lastInsertId(); + $url = strtolower($url); } else { - $tagid = $tag['id']; + $url = ''; } + $tagid = self::getID($name, $url); if (empty($tagid)) { - Logger::error('No tag id created', $fields); return; } } @@ -160,6 +152,32 @@ class Tag Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid, 'name' => $name, 'type' => $type, 'callstack' => System::callstack(8)]); } + /** + * Get a tag id for a given tag name and url + * + * @param string $name + * @param string $url + * @return void + */ + public static function getID(string $name, string $url = '') + { + $fields = ['name' => substr($name, 0, 96), 'url' => $url]; + + $tag = DBA::selectFirst('tag', ['id'], $fields); + if (DBA::isResult($tag)) { + return $tag['id']; + } + + DBA::insert('tag', $fields, true); + $tid = DBA::lastInsertId(); + if (!empty($tid)) { + return $tid; + } + + Logger::error('No tag id created', $fields); + return 0; + } + /** * Store tag/mention elements * diff --git a/src/Module/Profile/Status.php b/src/Module/Profile/Status.php index 685ab4e076..75ba60cb26 100644 --- a/src/Module/Profile/Status.php +++ b/src/Module/Profile/Status.php @@ -29,9 +29,9 @@ use Friendica\Core\Session; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Item; +use Friendica\Model\Post\Category; use Friendica\Model\Profile as ProfileModel; use Friendica\Model\User; -use Friendica\Model\Term; use Friendica\Module\BaseProfile; use Friendica\Module\Security\Login; use Friendica\Util\DateTimeFormat; @@ -142,8 +142,8 @@ class Status extends BaseProfile $sql_post_table = ""; if (!empty($category)) { - $sql_post_table = sprintf("INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ", - DBA::escape(Strings::protectSprintf($category)), intval(Term::OBJECT_TYPE_POST), intval(Term::CATEGORY), intval($a->profile['uid'])); + $sql_post_table = sprintf("INNER JOIN (SELECT `uri-id` FROM `category-view` WHERE `name` = '%s' AND `type` = %d AND `uid` = %d ORDER BY `uri-id` DESC) AS `category` ON `item`.`uri-id` = `category`.`uri-id` ", + DBA::escape(Strings::protectSprintf($category)), intval(Category::CATEGORY), intval($a->profile['uid'])); } if (!empty($hashtags)) { diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 5060a7d563..47272bf6c3 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -39,9 +39,9 @@ use Friendica\Model\ItemURI; use Friendica\Model\Mail; use Friendica\Model\Notify\Type; use Friendica\Model\PermissionSet; +use Friendica\Model\Post\Category; use Friendica\Model\Profile; use Friendica\Model\Tag; -use Friendica\Model\Term; use Friendica\Model\User; use Friendica\Network\Probe; use Friendica\Util\Crypto; @@ -241,13 +241,8 @@ class DFRN } if (isset($category)) { - $sql_post_table = sprintf( - "INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ", - DBA::escape(Strings::protectSprintf($category)), - intval(Term::OBJECT_TYPE_POST), - intval(Term::CATEGORY), - intval($owner_id) - ); + $sql_post_table = sprintf("INNER JOIN (SELECT `uri-id` FROM `category-view` WHERE `name` = '%s' AND `type` = %d AND `uid` = %d ORDER BY `uri-id` DESC) AS `category` ON `item`.`uri-id` = `category`.`uri-id` ", + DBA::escape(Strings::protectSprintf($category)), intval(Category::CATEGORY), intval($owner_id)); } if ($public_feed && ! $converse) { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 2c3a9fb5fd..788ff7d13b 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -51,7 +51,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1343); + define('DB_UPDATE_VERSION', 1345); } return [ @@ -1306,6 +1306,19 @@ return [ "url" => ["url"] ] ], + "post-category" => [ + "comment" => "post relation to categories", + "fields" => [ + "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "relation" => ["user" => "uid"], "comment" => "User id"], + "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""], + "tid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["tag" => "id"], "comment" => ""], + ], + "indexes" => [ + "PRIMARY" => ["uri-id", "uid", "type", "tid"], + "uri-id" => ["tid"] + ] + ], "post-tag" => [ "comment" => "post relation to tags", "fields" => [ diff --git a/static/dbview.config.php b/static/dbview.config.php index 0f0d691085..fa302b8669 100755 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -37,6 +37,21 @@ */ return [ + "category-view" => [ + "fields" => [ + "uri-id" => ["post-category", "uri-id"], + "uid" => ["post-category", "uid"], + "uri" => ["item-uri", "uri"], + "guid" => ["item-uri", "guid"], + "type" => ["post-category", "type"], + "tid" => ["post-category", "tid"], + "name" => ["tag", "name"], + "url" => ["tag", "url"], + ], + "query" => "FROM `post-category` + INNER JOIN `item-uri` ON `item-uri`.id = `post-category`.`uri-id` + LEFT JOIN `tag` ON `post-category`.`tid` = `tag`.`id`" + ], "tag-view" => [ "fields" => [ "uri-id" => ["post-tag", "uri-id"],