From fd5f151a7232d2b35dd823ec1763420eea97a831 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 13 Jan 2017 07:46:47 +0000 Subject: [PATCH 01/41] New function to detect heavily used indexes --- include/dba.php | 38 ++++++++++++++++++++++++++++++++++++++ include/dbstructure.php | 7 ++++--- include/photos.php | 5 +++-- mod/photos.php | 2 +- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/include/dba.php b/include/dba.php index 920027cbcf..17acf94328 100644 --- a/include/dba.php +++ b/include/dba.php @@ -138,6 +138,38 @@ class dba { return $return; } + public function log_index($query) { + $a = get_app(); + + if (($a->config["system"]["db_log_index"] == "") OR ($a->config["system"]["db_log_index_watch"] == "") OR + (intval($a->config["system"]["db_loglimit_index"]) == 0)) { + return; + } + + if (strtolower(substr($query, 0, 7)) == "explain") { + return; + } + + $r = $this->q("EXPLAIN ".$query); + if (!dbm::is_result($r)) { + return; + } + + $watchlist = explode(',', $a->config["system"]["db_log_index_watch"]); + + foreach ($r AS $row) { + if (in_array($row['key'], $watchlist) AND + ($row['rows'] >= intval($a->config["system"]["db_loglimit_index"]))) { + $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); + @file_put_contents($a->config["system"]["db_log_index"], datetime_convert()."\t". + $row['key']."\t".$row['rows']."\t". + basename($backtrace[1]["file"])."\t". + $backtrace[1]["line"]."\t".$backtrace[2]["function"]."\t". + substr($query, 0, 2000)."\n", FILE_APPEND); + } + } + } + public function q($sql, $onlyquery = false) { $a = get_app(); @@ -375,6 +407,9 @@ function q($sql) { //logger("dba: q: $stmt", LOGGER_ALL); if ($stmt === false) logger('dba: vsprintf error: ' . print_r(debug_backtrace(),true), LOGGER_DEBUG); + + $db->log_index($stmt); + return $db->q($stmt); } @@ -408,6 +443,9 @@ function qu($sql) { $stmt = @vsprintf($sql,$args); // Disabled warnings if ($stmt === false) logger('dba: vsprintf error: ' . print_r(debug_backtrace(),true), LOGGER_DEBUG); + + $db->log_index($stmt); + $db->q("SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;"); $retval = $db->q($stmt); $db->q("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;"); diff --git a/include/dbstructure.php b/include/dbstructure.php index 5c89f49fb9..535d7c5416 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -587,10 +587,10 @@ function db_definition($charset) { "indexes" => array( "PRIMARY" => array("id"), "uid_name" => array("uid", "name"), - "uid_self" => array("uid", "self"), + "self_uid" => array("self", "uid"), "alias_uid" => array("alias(32)", "uid"), - "uid_pending" => array("uid", "pending"), - "uid_blocked" => array("uid", "blocked"), + "pending_uid" => array("pending", "uid"), + "blocked_uid" => array("blocked", "uid"), "uid_rel_network_poll" => array("uid", "rel", "network", "poll(64)", "archive"), "uid_network_batch" => array("uid", "network", "batch(64)"), "addr_uid" => array("addr(32)", "uid"), @@ -1162,6 +1162,7 @@ function db_definition($charset) { "uid_contactid" => array("uid", "contact-id"), "uid_profile" => array("uid", "profile"), "uid_album_created" => array("uid", "album(32)", "created"), + "uid_album_scale_created" => array("uid", "album(32)", "scale", "created"), "uid_album_resource-id_created" => array("uid", "album(32)", "resource-id(64)", "created"), "resource-id" => array("resource-id(64)"), ) diff --git a/include/photos.php b/include/photos.php index 7cdd14bf6d..f3043e44ac 100644 --- a/include/photos.php +++ b/include/photos.php @@ -49,7 +49,7 @@ function photo_albums($uid, $update = false) { /// @todo This query needs to be renewed. It is really slow // At this time we just store the data in the cache $albums = qu("SELECT COUNT(DISTINCT `resource-id`) AS `total`, `album` - FROM `photo` USE INDEX (`uid_album_created`) + FROM `photo` WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' $sql_extra GROUP BY `album` ORDER BY `created` DESC", intval($uid), @@ -57,9 +57,10 @@ function photo_albums($uid, $update = false) { dbesc(t('Contact Photos')) ); } else { +// USE INDEX (`uid_album`) // This query doesn't do the count and is much faster $albums = qu("SELECT DISTINCT(`album`), '' AS `total` - FROM `photo` USE INDEX (`uid_album_created`) + FROM `photo` WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' $sql_extra GROUP BY `album` ORDER BY `created` DESC", intval($uid), diff --git a/mod/photos.php b/mod/photos.php index 60e5dee643..e23e4fcc92 100644 --- a/mod/photos.php +++ b/mod/photos.php @@ -1344,7 +1344,7 @@ function photos_content(App $a) { else $order = 'DESC'; - + /// @todo This query is totally bad, the whole functionality has to be changed $prvnxt = qu("SELECT `resource-id` FROM `photo` WHERE `album` = '%s' AND `uid` = %d AND `scale` = 0 $sql_extra ORDER BY `created` $order ", dbesc($ph[0]['album']), From a9833a395ff129a5fc4978b7ea08f0d2a84f9be6 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 13 Jan 2017 10:37:39 +0000 Subject: [PATCH 02/41] The relay query now uses a better index --- include/diaspora.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/diaspora.php b/include/diaspora.php index fc88c79bf2..fdbc0479f9 100644 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -45,13 +45,13 @@ class Diaspora { foreach($servers AS $server) { $server = trim($server); + $addr = "relay@".str_replace("http://", "", normalise_link($server)); $batch = $server."/receive/public"; - $relais = q("SELECT `batch`, `id`, `name`,`network` FROM `contact` WHERE `uid` = 0 AND `batch` = '%s' LIMIT 1", dbesc($batch)); + $relais = q("SELECT `batch`, `id`, `name`,`network` FROM `contact` WHERE `uid` = 0 AND `batch` = '%s' AND `addr` = '%s' AND `nurl` = '%s' LIMIT 1", + dbesc($batch), dbesc($addr), dbesc(normalise_link($server))); if (!$relais) { - $addr = "relay@".str_replace("http://", "", normalise_link($server)); - $r = q("INSERT INTO `contact` (`uid`, `created`, `name`, `nick`, `addr`, `url`, `nurl`, `batch`, `network`, `rel`, `blocked`, `pending`, `writable`, `name-date`, `uri-date`, `avatar-date`) VALUES (0, '%s', '%s', 'relay', '%s', '%s', '%s', '%s', '%s', %d, 0, 0, 1, '%s', '%s', '%s')", datetime_convert(), From 844fefc9c4eb67b1a161fb86c86edaffd5b18504 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 13 Jan 2017 13:04:37 +0000 Subject: [PATCH 03/41] Caching the events. --- mod/ping.php | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mod/ping.php b/mod/ping.php index cde03969f4..8b2090cb69 100644 --- a/mod/ping.php +++ b/mod/ping.php @@ -5,6 +5,7 @@ require_once('include/ForumManager.php'); require_once('include/group.php'); require_once('mod/proxy.php'); require_once('include/xml.php'); +require_once('include/cache.php'); /** * @brief Outputs the counts and the lists of various notifications @@ -195,15 +196,21 @@ function ping_init(App $a) } } - $ev = qu("SELECT count(`event`.`id`) AS total, type, start, adjust FROM `event` - WHERE `event`.`uid` = %d AND `start` < '%s' AND `finish` > '%s' and `ignore` = 0 - ORDER BY `start` ASC ", - intval(local_user()), - dbesc(datetime_convert('UTC', 'UTC', 'now + 7 days')), - dbesc(datetime_convert('UTC', 'UTC', 'now')) - ); + $cachekey = "ping:events:".local_user(); + $ev = Cache::get($cachekey); + if (is_null($ev)) { + $ev = qu("SELECT count(`event`.`id`) AS total, type, start, adjust FROM `event` + WHERE `event`.`uid` = %d AND `start` < '%s' AND `finish` > '%s' and `ignore` = 0 + ORDER BY `start` ASC ", + intval(local_user()), + dbesc(datetime_convert('UTC', 'UTC', 'now + 7 days')), + dbesc(datetime_convert('UTC', 'UTC', 'now')) + ); + } if (dbm::is_result($ev)) { + Cache::set($cachekey, $ev, CACHE_HOUR); + $all_events = intval($ev[0]['total']); if ($all_events) { From 3af099298c83b45f2f21cb5b81a8c7b6159e7459 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 13 Jan 2017 17:31:10 +0000 Subject: [PATCH 04/41] Some more caching --- include/identity.php | 25 ++++++++++++++++--------- mod/ping.php | 7 ++++--- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/include/identity.php b/include/identity.php index c18bc3a805..d3852b2c2c 100644 --- a/include/identity.php +++ b/include/identity.php @@ -6,6 +6,7 @@ require_once('include/ForumManager.php'); require_once('include/bbcode.php'); require_once("mod/proxy.php"); +require_once('include/cache.php'); /** * @@ -453,15 +454,21 @@ function get_birthdays() { $bd_format = t('g A l F d') ; // 8 AM Friday January 18 $bd_short = t('F d'); - $r = q("SELECT `event`.*, `event`.`id` AS `eid`, `contact`.* FROM `event` - INNER JOIN `contact` ON `contact`.`id` = `event`.`cid` - WHERE `event`.`uid` = %d AND `type` = 'birthday' AND `start` < '%s' AND `finish` > '%s' - ORDER BY `start` ASC ", - intval(local_user()), - dbesc(datetime_convert('UTC','UTC','now + 6 days')), - dbesc(datetime_convert('UTC','UTC','now')) - ); - + $cachekey = "get_birthdays:".local_user(); + $r = Cache::get($cachekey); + if (is_null($r)) { + $r = q("SELECT `event`.*, `event`.`id` AS `eid`, `contact`.* FROM `event` + INNER JOIN `contact` ON `contact`.`id` = `event`.`cid` + WHERE `event`.`uid` = %d AND `type` = 'birthday' AND `start` < '%s' AND `finish` > '%s' + ORDER BY `start` ASC ", + intval(local_user()), + dbesc(datetime_convert('UTC','UTC','now + 6 days')), + dbesc(datetime_convert('UTC','UTC','now')) + ); + if (dbm::is_result($r)) { + Cache::set($cachekey, $r, CACHE_HOUR); + } + } if (dbm::is_result($r)) { $total = 0; $now = strtotime('now'); diff --git a/mod/ping.php b/mod/ping.php index 8b2090cb69..b5330c7b33 100644 --- a/mod/ping.php +++ b/mod/ping.php @@ -196,7 +196,7 @@ function ping_init(App $a) } } - $cachekey = "ping:events:".local_user(); + $cachekey = "ping_init:".local_user(); $ev = Cache::get($cachekey); if (is_null($ev)) { $ev = qu("SELECT count(`event`.`id`) AS total, type, start, adjust FROM `event` @@ -206,11 +206,12 @@ function ping_init(App $a) dbesc(datetime_convert('UTC', 'UTC', 'now + 7 days')), dbesc(datetime_convert('UTC', 'UTC', 'now')) ); + if (dbm::is_result($ev)) { + Cache::set($cachekey, $ev, CACHE_HOUR); + } } if (dbm::is_result($ev)) { - Cache::set($cachekey, $ev, CACHE_HOUR); - $all_events = intval($ev[0]['total']); if ($all_events) { From 29ef8d29cee07d6677134db289f6975586c53913 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 13 Jan 2017 22:13:52 +0000 Subject: [PATCH 05/41] Better usage of text and mediumtext. --- database.sql | 67 +++++++++++++++++++++-------------------- include/dbstructure.php | 58 +++++++++++++++++------------------ 2 files changed, 63 insertions(+), 62 deletions(-) diff --git a/database.sql b/database.sql index becd14fc73..c07d2ae6d9 100644 --- a/database.sql +++ b/database.sql @@ -31,10 +31,10 @@ CREATE TABLE IF NOT EXISTS `attach` ( `data` longblob NOT NULL, `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `edited` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', - `allow_cid` mediumtext, - `allow_gid` mediumtext, - `deny_cid` mediumtext, - `deny_gid` mediumtext, + `allow_cid` text, + `allow_gid` text, + `deny_cid` text, + `deny_gid` text, PRIMARY KEY(`id`) ) DEFAULT CHARSET=utf8mb4; @@ -55,7 +55,7 @@ CREATE TABLE IF NOT EXISTS `auth_codes` ( -- CREATE TABLE IF NOT EXISTS `cache` ( `k` varbinary(255) NOT NULL, - `v` text, + `v` mediumtext, `expire_mode` int(11) NOT NULL DEFAULT 0, `updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`k`), @@ -96,7 +96,7 @@ CREATE TABLE IF NOT EXISTS `config` ( `id` int(10) unsigned NOT NULL auto_increment, `cat` varbinary(255) NOT NULL DEFAULT '', `k` varbinary(255) NOT NULL DEFAULT '', - `v` text, + `v` mediumtext, PRIMARY KEY(`id`), UNIQUE INDEX `cat_k` (`cat`,`k`) ) DEFAULT CHARSET=utf8mb4; @@ -172,13 +172,13 @@ CREATE TABLE IF NOT EXISTS `contact` ( `bd` date NOT NULL DEFAULT '0000-00-00', `notify_new_posts` tinyint(1) NOT NULL DEFAULT 0, `fetch_further_information` tinyint(1) NOT NULL DEFAULT 0, - `ffi_keyword_blacklist` mediumtext, + `ffi_keyword_blacklist` text, PRIMARY KEY(`id`), INDEX `uid_name` (`uid`,`name`), - INDEX `uid_self` (`uid`,`self`), + INDEX `self_uid` (`self`,`uid`), INDEX `alias_uid` (`alias`(32),`uid`), - INDEX `uid_pending` (`uid`,`pending`), - INDEX `uid_blocked` (`uid`,`blocked`), + INDEX `pending_uid` (`pending`,`uid`), + INDEX `blocked_uid` (`blocked`,`uid`), INDEX `uid_rel_network_poll` (`uid`,`rel`,`network`,`poll`(64),`archive`), INDEX `uid_network_batch` (`uid`,`network`,`batch`(64)), INDEX `addr_uid` (`addr`(32),`uid`), @@ -192,12 +192,12 @@ CREATE TABLE IF NOT EXISTS `contact` ( CREATE TABLE IF NOT EXISTS `conv` ( `id` int(10) unsigned NOT NULL auto_increment, `guid` varchar(64) NOT NULL DEFAULT '', - `recips` mediumtext, + `recips` text, `uid` int(11) NOT NULL DEFAULT 0, `creator` varchar(255) NOT NULL DEFAULT '', `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', - `subject` mediumtext, + `subject` text, PRIMARY KEY(`id`), INDEX `uid` (`uid`) ) DEFAULT CHARSET=utf8mb4; @@ -234,10 +234,10 @@ CREATE TABLE IF NOT EXISTS `event` ( `nofinish` tinyint(1) NOT NULL DEFAULT 0, `adjust` tinyint(1) NOT NULL DEFAULT 1, `ignore` tinyint(1) unsigned NOT NULL DEFAULT 0, - `allow_cid` mediumtext, - `allow_gid` mediumtext, - `deny_cid` mediumtext, - `deny_gid` mediumtext, + `allow_cid` text, + `allow_gid` text, + `deny_cid` text, + `deny_gid` text, PRIMARY KEY(`id`), INDEX `uid_start` (`uid`,`start`) ) DEFAULT CHARSET=utf8mb4; @@ -394,7 +394,7 @@ CREATE TABLE IF NOT EXISTS `group_member` ( `gid` int(10) unsigned NOT NULL DEFAULT 0, `contact-id` int(10) unsigned NOT NULL DEFAULT 0, PRIMARY KEY(`id`), - INDEX `cid_contactid` (`cid`,`contact-id`), + INDEX `gid_contactid` (`gid`,`contact-id`), INDEX `uid_contactid` (`uid`,`contact-id`), UNIQUE INDEX `uid_gid_contactid` (`uid`,`gid`,`contact-id`) ) DEFAULT CHARSET=utf8mb4; @@ -688,7 +688,7 @@ CREATE TABLE IF NOT EXISTS `notify-threads` ( -- CREATE TABLE IF NOT EXISTS `oembed` ( `url` varbinary(255) NOT NULL, - `content` text, + `content` mediumtext, `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`url`), INDEX `created` (`created`) @@ -701,7 +701,7 @@ CREATE TABLE IF NOT EXISTS `parsed_url` ( `url` varbinary(255) NOT NULL, `guessing` tinyint(1) NOT NULL DEFAULT 0, `oembed` tinyint(1) NOT NULL DEFAULT 0, - `content` text, + `content` mediumtext, `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`url`,`guessing`,`oembed`), INDEX `created` (`created`) @@ -750,6 +750,7 @@ CREATE TABLE IF NOT EXISTS `photo` ( INDEX `uid_contactid` (`uid`,`contact-id`), INDEX `uid_profile` (`uid`,`profile`), INDEX `uid_album_created` (`uid`,`album`(32),`created`), + INDEX `uid_album_scale_created` (`uid`,`album`(32),`scale`,`created`), INDEX `uid_album_resource-id_created` (`uid`,`album`(32),`resource-id`(64),`created`), INDEX `resource-id` (`resource-id`(64)) ) DEFAULT CHARSET=utf8mb4; @@ -760,16 +761,16 @@ CREATE TABLE IF NOT EXISTS `photo` ( CREATE TABLE IF NOT EXISTS `poll` ( `id` int(11) NOT NULL auto_increment, `uid` int(11) NOT NULL DEFAULT 0, - `q0` mediumtext, - `q1` mediumtext, - `q2` mediumtext, - `q3` mediumtext, - `q4` mediumtext, - `q5` mediumtext, - `q6` mediumtext, - `q7` mediumtext, - `q8` mediumtext, - `q9` mediumtext, + `q0` text, + `q1` text, + `q2` text, + `q3` text, + `q4` text, + `q5` text, + `q6` text, + `q7` text, + `q8` text, + `q9` text, PRIMARY KEY(`id`), INDEX `uid` (`uid`) ) DEFAULT CHARSET=utf8mb4; @@ -1082,10 +1083,10 @@ CREATE TABLE IF NOT EXISTS `user` ( `expire_notification_sent` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `service_class` varchar(32) NOT NULL DEFAULT '', `def_gid` int(11) NOT NULL DEFAULT 0, - `allow_cid` mediumtext, - `allow_gid` mediumtext, - `deny_cid` mediumtext, - `deny_gid` mediumtext, + `allow_cid` text, + `allow_gid` text, + `deny_cid` text, + `deny_gid` text, `openidserver` text, PRIMARY KEY(`uid`), INDEX `nickname` (`nickname`(32)) diff --git a/include/dbstructure.php b/include/dbstructure.php index 535d7c5416..126da9ce4b 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -441,10 +441,10 @@ function db_definition($charset) { "data" => array("type" => "longblob", "not null" => "1"), "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "edited" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), - "allow_cid" => array("type" => "mediumtext"), - "allow_gid" => array("type" => "mediumtext"), - "deny_cid" => array("type" => "mediumtext"), - "deny_gid" => array("type" => "mediumtext"), + "allow_cid" => array("type" => "text"), + "allow_gid" => array("type" => "text"), + "deny_cid" => array("type" => "text"), + "deny_gid" => array("type" => "text"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -465,7 +465,7 @@ function db_definition($charset) { $database["cache"] = array( "fields" => array( "k" => array("type" => "varbinary(255)", "not null" => "1", "primary" => "1"), - "v" => array("type" => "text"), + "v" => array("type" => "mediumtext"), "expire_mode" => array("type" => "int(11)", "not null" => "1", "default" => "0"), "updated" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), ), @@ -506,7 +506,7 @@ function db_definition($charset) { "id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), "cat" => array("type" => "varbinary(255)", "not null" => "1", "default" => ""), "k" => array("type" => "varbinary(255)", "not null" => "1", "default" => ""), - "v" => array("type" => "text"), + "v" => array("type" => "mediumtext"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -582,7 +582,7 @@ function db_definition($charset) { "bd" => array("type" => "date", "not null" => "1", "default" => "0000-00-00"), "notify_new_posts" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), "fetch_further_information" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), - "ffi_keyword_blacklist" => array("type" => "mediumtext"), + "ffi_keyword_blacklist" => array("type" => "text"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -602,12 +602,12 @@ function db_definition($charset) { "fields" => array( "id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), "guid" => array("type" => "varchar(64)", "not null" => "1", "default" => ""), - "recips" => array("type" => "mediumtext"), + "recips" => array("type" => "text"), "uid" => array("type" => "int(11)", "not null" => "1", "default" => "0"), "creator" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "updated" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), - "subject" => array("type" => "mediumtext"), + "subject" => array("type" => "text"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -644,10 +644,10 @@ function db_definition($charset) { "nofinish" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), "adjust" => array("type" => "tinyint(1)", "not null" => "1", "default" => "1"), "ignore" => array("type" => "tinyint(1) unsigned", "not null" => "1", "default" => "0"), - "allow_cid" => array("type" => "mediumtext"), - "allow_gid" => array("type" => "mediumtext"), - "deny_cid" => array("type" => "mediumtext"), - "deny_gid" => array("type" => "mediumtext"), + "allow_cid" => array("type" => "text"), + "allow_gid" => array("type" => "text"), + "deny_cid" => array("type" => "text"), + "deny_gid" => array("type" => "text"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -1098,7 +1098,7 @@ function db_definition($charset) { $database["oembed"] = array( "fields" => array( "url" => array("type" => "varbinary(255)", "not null" => "1", "primary" => "1"), - "content" => array("type" => "text"), + "content" => array("type" => "mediumtext"), "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), ), "indexes" => array( @@ -1111,7 +1111,7 @@ function db_definition($charset) { "url" => array("type" => "varbinary(255)", "not null" => "1", "primary" => "1"), "guessing" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0", "primary" => "1"), "oembed" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0", "primary" => "1"), - "content" => array("type" => "text"), + "content" => array("type" => "mediumtext"), "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), ), "indexes" => array( @@ -1171,16 +1171,16 @@ function db_definition($charset) { "fields" => array( "id" => array("type" => "int(11)", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), "uid" => array("type" => "int(11)", "not null" => "1", "default" => "0"), - "q0" => array("type" => "mediumtext"), - "q1" => array("type" => "mediumtext"), - "q2" => array("type" => "mediumtext"), - "q3" => array("type" => "mediumtext"), - "q4" => array("type" => "mediumtext"), - "q5" => array("type" => "mediumtext"), - "q6" => array("type" => "mediumtext"), - "q7" => array("type" => "mediumtext"), - "q8" => array("type" => "mediumtext"), - "q9" => array("type" => "mediumtext"), + "q0" => array("type" => "text"), + "q1" => array("type" => "text"), + "q2" => array("type" => "text"), + "q3" => array("type" => "text"), + "q4" => array("type" => "text"), + "q5" => array("type" => "text"), + "q6" => array("type" => "text"), + "q7" => array("type" => "text"), + "q8" => array("type" => "text"), + "q9" => array("type" => "text"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -1493,10 +1493,10 @@ function db_definition($charset) { "expire_notification_sent" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "service_class" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), "def_gid" => array("type" => "int(11)", "not null" => "1", "default" => "0"), - "allow_cid" => array("type" => "mediumtext"), - "allow_gid" => array("type" => "mediumtext"), - "deny_cid" => array("type" => "mediumtext"), - "deny_gid" => array("type" => "mediumtext"), + "allow_cid" => array("type" => "text"), + "allow_gid" => array("type" => "text"), + "deny_cid" => array("type" => "text"), + "deny_gid" => array("type" => "text"), "openidserver" => array("type" => "text"), ), "indexes" => array( From 85397f3bb9955981cbe5fb356aaf93cd10b54e62 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 14 Jan 2017 00:49:41 +0000 Subject: [PATCH 06/41] Some removed indexes --- include/dbstructure.php | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index 126da9ce4b..90b6f2d068 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -471,7 +471,6 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("k"), - "updated" => array("updated"), "expire_mode_updated" => array("expire_mode", "updated"), ) ); @@ -781,7 +780,6 @@ function db_definition($charset) { "PRIMARY" => array("id"), "cid_uid_gcid_zcid" => array("UNIQUE", "cid","uid","gcid","zcid"), "gcid" => array("gcid"), - "zcid" => array("zcid"), ) ); $database["group"] = array( @@ -807,7 +805,6 @@ function db_definition($charset) { "indexes" => array( "PRIMARY" => array("id"), "gid_contactid" => array("gid", "contact-id"), - "uid_contactid" => array("uid", "contact-id"), "uid_gid_contactid" => array("UNIQUE", "uid", "gid", "contact-id"), ) ); @@ -944,19 +941,14 @@ function db_definition($charset) { "uid_created" => array("uid","created"), "uid_unseen_contactid" => array("uid","unseen","contact-id"), "uid_network_received" => array("uid","network","received"), - "uid_received" => array("uid","received"), "uid_network_commented" => array("uid","network","commented"), - "uid_title" => array("uid","title"), "uid_thrparent" => array("uid","thr-parent"), "uid_parenturi" => array("uid","parent-uri"), - "uid_contactid_id" => array("uid","contact-id","id"), "uid_contactid_created" => array("uid","contact-id","created"), "authorid_created" => array("author-id","created"), "uid_uri" => array("uid", "uri"), - "uid_wall_created" => array("uid","wall","created"), "resource-id" => array("resource-id"), - "uid_type" => array("uid","type"), - "contactid_allowcid_allowpid_denycid_denygid" => array("contact-id","allow_cid(10)","allow_gid(10)","deny_cid(10)","deny_gid(10)"), + "contactid_allowcid_allowpid_denycid_denygid" => array("contact-id","allow_cid(10)","allow_gid(10)","deny_cid(10)","deny_gid(10)"), // "uid_type_changed" => array("uid","type","changed"), "contactid_verb" => array("contact-id","verb"), "deleted_changed" => array("deleted","changed"), @@ -1015,7 +1007,6 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "uid" => array("uid"), "uid_seen" => array("uid", "seen"), "convid" => array("convid"), "uri" => array("uri(64)"), @@ -1050,7 +1041,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "uid_mid" => array("uid","mid"), + "uid_mid" => array("UNIQUE", "uid","mid"), ) ); $database["notify"] = array( @@ -1076,9 +1067,8 @@ function db_definition($charset) { "indexes" => array( "PRIMARY" => array("id"), "uid_hash" => array("uid", "hash"), - "uid_seen_date" => array("uid", "seen", "date"), - "uid_type_link" => array("uid", "type", "link"), - "uid_link" => array("uid", "link"), + "seen_uid_date" => array("seen", "uid", "date"), + "type_uid_link" => array("type", "uid", "link"), "uid_date" => array("uid", "date"), ) ); @@ -1092,7 +1082,6 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "master-parent-item" => array("master-parent-item"), ) ); $database["oembed"] = array( @@ -1161,7 +1150,6 @@ function db_definition($charset) { "PRIMARY" => array("id"), "uid_contactid" => array("uid", "contact-id"), "uid_profile" => array("uid", "profile"), - "uid_album_created" => array("uid", "album(32)", "created"), "uid_album_scale_created" => array("uid", "album(32)", "scale", "created"), "uid_album_resource-id_created" => array("uid", "album(32)", "resource-id(64)", "created"), "resource-id" => array("resource-id(64)"), @@ -1392,8 +1380,6 @@ function db_definition($charset) { "indexes" => array( "PRIMARY" => array("tid"), "oid_otype_type_term" => array("oid","otype","type","term"), - "uid_term_tid" => array("uid","term(32)","tid"), - "type_term" => array("type","term(32)"), "uid_otype_type_term_global_created" => array("uid","otype","type","term(32)","global","created"), "uid_otype_type_url" => array("uid","otype","type","url(64)"), "guid" => array("guid(64)"), @@ -1430,8 +1416,6 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("iid"), - "created" => array("created"), - "commented" => array("commented"), "uid_network_commented" => array("uid","network","commented"), "uid_network_created" => array("uid","network","created"), "uid_contactid_commented" => array("uid","contact-id","commented"), @@ -1525,7 +1509,6 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "created" => array("created"), ) ); From e16afc0450c812b76a79c4fb71fcf8e5260bca0b Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 14 Jan 2017 21:36:34 +0000 Subject: [PATCH 07/41] Some more changed indexes and improved queries --- include/cron.php | 12 +++++++----- include/dba.php | 27 +++++++++++++++++++++++---- include/dbstructure.php | 11 +++++++++-- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/include/cron.php b/include/cron.php index 059bcea437..f7def61215 100644 --- a/include/cron.php +++ b/include/cron.php @@ -239,11 +239,13 @@ function cron_poll_contacts($argc, $argv) { : '' ); - $contacts = q("SELECT `contact`.`id` FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid` - WHERE `rel` IN (%d, %d) AND `poll` != '' AND `network` IN ('%s', '%s', '%s', '%s', '%s', '%s') - $sql_extra - AND NOT `self` AND NOT `contact`.`blocked` AND NOT `contact`.`readonly` AND NOT `contact`.`archive` - AND NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql ORDER BY RAND()", + $contacts = q("SELECT `contact`.`id` FROM `user` + STRAIGHT_JOIN `contact` + ON `contact`.`uid` = `user`.`uid` AND `contact`.`rel` IN (%d, %d) AND `contact`.`poll` != '' + AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s', '%s') $sql_extra + AND NOT `contact`.`self` AND NOT `contact`.`blocked` AND NOT `contact`.`readonly` + AND NOT `contact`.`archive` + WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql ORDER BY RAND()", intval(CONTACT_IS_SHARING), intval(CONTACT_IS_FRIEND), dbesc(NETWORK_DFRN), diff --git a/include/dba.php b/include/dba.php index 17acf94328..4b76f55ab0 100644 --- a/include/dba.php +++ b/include/dba.php @@ -141,25 +141,44 @@ class dba { public function log_index($query) { $a = get_app(); - if (($a->config["system"]["db_log_index"] == "") OR ($a->config["system"]["db_log_index_watch"] == "") OR - (intval($a->config["system"]["db_loglimit_index"]) == 0)) { + if ($a->config["system"]["db_log_index"] == "") { return; } + // Don't explain an explain statement if (strtolower(substr($query, 0, 7)) == "explain") { return; } + // Only do the explain on "select", "update" and "delete" + if (!in_array(strtolower(substr($query, 0, 6)), array("select", "update", "delete"))) { + return; + } + $r = $this->q("EXPLAIN ".$query); if (!dbm::is_result($r)) { return; } $watchlist = explode(',', $a->config["system"]["db_log_index_watch"]); + $blacklist = explode(',', $a->config["system"]["db_log_index_blacklist"]); foreach ($r AS $row) { - if (in_array($row['key'], $watchlist) AND - ($row['rows'] >= intval($a->config["system"]["db_loglimit_index"]))) { + if ((intval($a->config["system"]["db_loglimit_index"]) > 0)) { + $log = (in_array($row['key'], $watchlist) AND + ($row['rows'] >= intval($a->config["system"]["db_loglimit_index"]))); + } else + $log = false; + + if ((intval($a->config["system"]["db_loglimit_index_high"]) > 0) AND ($row['rows'] >= intval($a->config["system"]["db_loglimit_index_high"]))) { + $log = true; + } + + if (in_array($row['key'], $blacklist) OR ($row['key'] == "")) { + $log = false; + } + + if ($log) { $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); @file_put_contents($a->config["system"]["db_log_index"], datetime_convert()."\t". $row['key']."\t".$row['rows']."\t". diff --git a/include/dbstructure.php b/include/dbstructure.php index 90b6f2d068..551f254a53 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -595,6 +595,8 @@ function db_definition($charset) { "addr_uid" => array("addr(32)", "uid"), "nurl_uid" => array("nurl(32)", "uid"), "nick_uid" => array("nick(32)", "uid"), + "dfrn-id" => array("dfrn-id"), + "issued-id" => array("issued-id"), ) ); $database["conv"] = array( @@ -676,6 +678,7 @@ function db_definition($charset) { "indexes" => array( "PRIMARY" => array("id"), "addr" => array("addr(32)"), + "url" => array("url"), ) ); $database["ffinder"] = array( @@ -764,6 +767,7 @@ function db_definition($charset) { "name" => array("name(32)"), "nick" => array("nick(32)"), "addr" => array("addr(32)"), + "hide_network_updated" => array("hide", "network", "updated"), "updated" => array("updated"), ) ); @@ -804,6 +808,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), + "contactid" => array("contact-id"), "gid_contactid" => array("gid", "contact-id"), "uid_gid_contactid" => array("UNIQUE", "uid", "gid", "contact-id"), ) @@ -953,6 +958,7 @@ function db_definition($charset) { "contactid_verb" => array("contact-id","verb"), "deleted_changed" => array("deleted","changed"), "uid_wall_changed" => array("uid","wall","changed"), + "wall_uid_changed" => array("wall","uid","changed"), "uid_eventid" => array("uid","event-id"), "uid_authorlink" => array("uid","author-link"), "uid_ownerlink" => array("uid","owner-link"), @@ -1066,10 +1072,10 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "uid_hash" => array("uid", "hash"), + "hash_uid" => array("hash", "uid"), "seen_uid_date" => array("seen", "uid", "date"), - "type_uid_link" => array("type", "uid", "link"), "uid_date" => array("uid", "date"), + "uid_type_link" => array("uid", "type", "link"), ) ); $database["notify-threads"] = array( @@ -1245,6 +1251,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), + "uid_is-default" => array("uid", "is-default"), ) ); $database["profile_check"] = array( From 021a4fad714c25c9e031ff4792ba0e2b7377568b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 15 Jan 2017 12:36:06 +0000 Subject: [PATCH 08/41] Some more database stuff --- database.sql | 45 ++++++++++++++++------------------------- include/dba.php | 2 +- include/dbstructure.php | 1 - include/dfrn.php | 2 +- include/ostatus.php | 4 +++- include/post_update.php | 2 +- include/socgraph.php | 2 +- 7 files changed, 24 insertions(+), 34 deletions(-) diff --git a/database.sql b/database.sql index c07d2ae6d9..9d4d289f2f 100644 --- a/database.sql +++ b/database.sql @@ -59,7 +59,6 @@ CREATE TABLE IF NOT EXISTS `cache` ( `expire_mode` int(11) NOT NULL DEFAULT 0, `updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`k`), - INDEX `updated` (`updated`), INDEX `expire_mode_updated` (`expire_mode`,`updated`) ) DEFAULT CHARSET=utf8mb4; @@ -183,7 +182,9 @@ CREATE TABLE IF NOT EXISTS `contact` ( INDEX `uid_network_batch` (`uid`,`network`,`batch`(64)), INDEX `addr_uid` (`addr`(32),`uid`), INDEX `nurl_uid` (`nurl`(32),`uid`), - INDEX `nick_uid` (`nick`(32),`uid`) + INDEX `nick_uid` (`nick`(32),`uid`), + INDEX `dfrn-id` (`dfrn-id`), + INDEX `issued-id` (`issued-id`) ) DEFAULT CHARSET=utf8mb4; -- @@ -264,7 +265,8 @@ CREATE TABLE IF NOT EXISTS `fcontact` ( `pubkey` text, `updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`id`), - INDEX `addr` (`addr`(32)) + INDEX `addr` (`addr`(32)), + INDEX `url` (`url`) ) DEFAULT CHARSET=utf8mb4; -- @@ -353,6 +355,7 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( INDEX `name` (`name`(32)), INDEX `nick` (`nick`(32)), INDEX `addr` (`addr`(32)), + INDEX `hide_network_updated` (`hide`,`network`,`updated`), INDEX `updated` (`updated`) ) DEFAULT CHARSET=utf8mb4; @@ -368,8 +371,7 @@ CREATE TABLE IF NOT EXISTS `glink` ( `updated` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`id`), UNIQUE INDEX `cid_uid_gcid_zcid` (`cid`,`uid`,`gcid`,`zcid`), - INDEX `gcid` (`gcid`), - INDEX `zcid` (`zcid`) + INDEX `gcid` (`gcid`) ) DEFAULT CHARSET=utf8mb4; -- @@ -394,8 +396,8 @@ CREATE TABLE IF NOT EXISTS `group_member` ( `gid` int(10) unsigned NOT NULL DEFAULT 0, `contact-id` int(10) unsigned NOT NULL DEFAULT 0, PRIMARY KEY(`id`), + INDEX `contactid` (`contact-id`), INDEX `gid_contactid` (`gid`,`contact-id`), - INDEX `uid_contactid` (`uid`,`contact-id`), UNIQUE INDEX `uid_gid_contactid` (`uid`,`gid`,`contact-id`) ) DEFAULT CHARSET=utf8mb4; @@ -532,18 +534,13 @@ CREATE TABLE IF NOT EXISTS `item` ( INDEX `uid_created` (`uid`,`created`), INDEX `uid_unseen_contactid` (`uid`,`unseen`,`contact-id`), INDEX `uid_network_received` (`uid`,`network`,`received`), - INDEX `uid_received` (`uid`,`received`), INDEX `uid_network_commented` (`uid`,`network`,`commented`), - INDEX `uid_title` (`uid`,`title`), INDEX `uid_thrparent` (`uid`,`thr-parent`), INDEX `uid_parenturi` (`uid`,`parent-uri`), - INDEX `uid_contactid_id` (`uid`,`contact-id`,`id`), INDEX `uid_contactid_created` (`uid`,`contact-id`,`created`), INDEX `authorid_created` (`author-id`,`created`), INDEX `uid_uri` (`uid`,`uri`), - INDEX `uid_wall_created` (`uid`,`wall`,`created`), INDEX `resource-id` (`resource-id`), - INDEX `uid_type` (`uid`,`type`), INDEX `contactid_allowcid_allowpid_denycid_denygid` (`contact-id`,`allow_cid`(10),`allow_gid`(10),`deny_cid`(10),`deny_gid`(10)), INDEX `uid_type_changed` (`uid`,`type`,`changed`), INDEX `contactid_verb` (`contact-id`,`verb`), @@ -603,7 +600,6 @@ CREATE TABLE IF NOT EXISTS `mail` ( `parent-uri` varchar(255) NOT NULL DEFAULT '', `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY(`id`), - INDEX `uid` (`uid`), INDEX `uid_seen` (`uid`,`seen`), INDEX `convid` (`convid`), INDEX `uri` (`uri`(64)), @@ -638,7 +634,7 @@ CREATE TABLE IF NOT EXISTS `manage` ( `uid` int(11) NOT NULL DEFAULT 0, `mid` int(11) NOT NULL DEFAULT 0, PRIMARY KEY(`id`), - INDEX `uid_mid` (`uid`,`mid`) + UNIQUE INDEX `uid_mid` (`uid`,`mid`) ) DEFAULT CHARSET=utf8mb4; -- @@ -663,11 +659,10 @@ CREATE TABLE IF NOT EXISTS `notify` ( `name_cache` tinytext, `msg_cache` mediumtext, PRIMARY KEY(`id`), - INDEX `uid_hash` (`uid`,`hash`), - INDEX `uid_seen_date` (`uid`,`seen`,`date`), - INDEX `uid_type_link` (`uid`,`type`,`link`), - INDEX `uid_link` (`uid`,`link`), - INDEX `uid_date` (`uid`,`date`) + INDEX `hash_uid` (`hash`,`uid`), + INDEX `seen_uid_date` (`seen`,`uid`,`date`), + INDEX `uid_date` (`uid`,`date`), + INDEX `uid_type_link` (`uid`,`type`,`link`) ) DEFAULT CHARSET=utf8mb4; -- @@ -679,8 +674,7 @@ CREATE TABLE IF NOT EXISTS `notify-threads` ( `master-parent-item` int(10) unsigned NOT NULL DEFAULT 0, `parent-item` int(10) unsigned NOT NULL DEFAULT 0, `receiver-uid` int(11) NOT NULL DEFAULT 0, - PRIMARY KEY(`id`), - INDEX `master-parent-item` (`master-parent-item`) + PRIMARY KEY(`id`) ) DEFAULT CHARSET=utf8mb4; -- @@ -749,7 +743,6 @@ CREATE TABLE IF NOT EXISTS `photo` ( PRIMARY KEY(`id`), INDEX `uid_contactid` (`uid`,`contact-id`), INDEX `uid_profile` (`uid`,`profile`), - INDEX `uid_album_created` (`uid`,`album`(32),`created`), INDEX `uid_album_scale_created` (`uid`,`album`(32),`scale`,`created`), INDEX `uid_album_resource-id_created` (`uid`,`album`(32),`resource-id`(64),`created`), INDEX `resource-id` (`resource-id`(64)) @@ -844,7 +837,8 @@ CREATE TABLE IF NOT EXISTS `profile` ( `thumb` varchar(255) NOT NULL DEFAULT '', `publish` tinyint(1) NOT NULL DEFAULT 0, `net-publish` tinyint(1) NOT NULL DEFAULT 0, - PRIMARY KEY(`id`) + PRIMARY KEY(`id`), + INDEX `uid_is-default` (`uid`,`is-default`) ) DEFAULT CHARSET=utf8mb4; -- @@ -980,8 +974,6 @@ CREATE TABLE IF NOT EXISTS `term` ( `uid` int(10) unsigned NOT NULL DEFAULT 0, PRIMARY KEY(`tid`), INDEX `oid_otype_type_term` (`oid`,`otype`,`type`,`term`), - INDEX `uid_term_tid` (`uid`,`term`(32),`tid`), - INDEX `type_term` (`type`,`term`(32)), INDEX `uid_otype_type_term_global_created` (`uid`,`otype`,`type`,`term`(32),`global`,`created`), INDEX `uid_otype_type_url` (`uid`,`otype`,`type`,`url`(64)), INDEX `guid` (`guid`(64)) @@ -1018,8 +1010,6 @@ CREATE TABLE IF NOT EXISTS `thread` ( `mention` tinyint(1) NOT NULL DEFAULT 0, `network` varchar(32) NOT NULL DEFAULT '', PRIMARY KEY(`iid`), - INDEX `created` (`created`), - INDEX `commented` (`commented`), INDEX `uid_network_commented` (`uid`,`network`,`commented`), INDEX `uid_network_created` (`uid`,`network`,`created`), INDEX `uid_contactid_commented` (`uid`,`contact-id`,`commented`), @@ -1112,7 +1102,6 @@ CREATE TABLE IF NOT EXISTS `workerqueue` ( `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `pid` int(11) NOT NULL DEFAULT 0, `executed` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', - PRIMARY KEY(`id`), - INDEX `created` (`created`) + PRIMARY KEY(`id`) ) DEFAULT CHARSET=utf8mb4; diff --git a/include/dba.php b/include/dba.php index 4b76f55ab0..7d2d3bbd0f 100644 --- a/include/dba.php +++ b/include/dba.php @@ -181,7 +181,7 @@ class dba { if ($log) { $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); @file_put_contents($a->config["system"]["db_log_index"], datetime_convert()."\t". - $row['key']."\t".$row['rows']."\t". + $row['key']."\t".$row['rows']."\t".$row['Extra']."\t". basename($backtrace[1]["file"])."\t". $backtrace[1]["line"]."\t".$backtrace[2]["function"]."\t". substr($query, 0, 2000)."\n", FILE_APPEND); diff --git a/include/dbstructure.php b/include/dbstructure.php index 551f254a53..6154ddeb08 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -958,7 +958,6 @@ function db_definition($charset) { "contactid_verb" => array("contact-id","verb"), "deleted_changed" => array("deleted","changed"), "uid_wall_changed" => array("uid","wall","changed"), - "wall_uid_changed" => array("wall","uid","changed"), "uid_eventid" => array("uid","event-id"), "uid_authorlink" => array("uid","author-link"), "uid_ownerlink" => array("uid","owner-link"), diff --git a/include/dfrn.php b/include/dfrn.php index 5e4f037796..ffdc2cbaf9 100644 --- a/include/dfrn.php +++ b/include/dfrn.php @@ -194,7 +194,7 @@ class dfrn { `contact`.`name-date`, `contact`.`uri-date`, `contact`.`avatar-date`, `contact`.`thumb`, `contact`.`dfrn-id`, `contact`.`self`, `sign`.`signed_text`, `sign`.`signature`, `sign`.`signer` - FROM `item` USE INDEX (`uid_wall_changed`, `uid_type_changed`) $sql_post_table + FROM `item` USE INDEX (`uid_wall_changed`, `wall_uid_changed`) $sql_post_table STRAIGHT_JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND (NOT `contact`.`blocked` OR `contact`.`pending`) LEFT JOIN `sign` ON `sign`.`iid` = `item`.`id` diff --git a/include/ostatus.php b/include/ostatus.php index ba64f493db..2c4b677a53 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -523,7 +523,9 @@ class ostatus { $r = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s'", intval($importer["uid"]), dbesc($item["parent-uri"])); - if (!$r AND ($related != "")) { + // Only fetch missing stuff if it is a comment or reshare. + if (in_array($item["verb"], array(ACTIVITY_POST, ACTIVITY_SHARE)) AND + !dbm::is_result($r) AND ($related != "")) { $reply_path = str_replace("/notice/", "/api/statuses/show/", $related).".atom"; if ($reply_path != $related) { diff --git a/include/post_update.php b/include/post_update.php index b2d682d722..ae5aad26da 100644 --- a/include/post_update.php +++ b/include/post_update.php @@ -239,7 +239,7 @@ function post_update_1206() { logger("Start", LOGGER_DEBUG); $r = q("SELECT `contact`.`id`, `contact`.`last-item`, - (SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date` + (SELECT MAX(`changed`) FROM `item` USE INDEX (`uid_wall_changed`, `wall_uid_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date` FROM `user` INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`"); diff --git a/include/socgraph.php b/include/socgraph.php index 6249f189e9..32c151c043 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -1529,7 +1529,7 @@ function get_gcontact_id($contact) { if (in_array($contact["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS))) $contact["url"] = clean_contact_url($contact["url"]); - $r = q("SELECT `id`, `last_contact`, `last_failure`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 2", + $r = q("SELECT `id`, `last_contact`, `last_failure`, `network` FROM `gcontact` WHERE `nurl` = '%s' LIMIT 2", dbesc(normalise_link($contact["url"]))); if ($r) { From 7e846ba7acf2cf452f4250d42105ba24d429f67e Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 15 Jan 2017 14:50:26 +0000 Subject: [PATCH 09/41] Hide the calculation for "previous" and "next" behind a setting --- mod/photos.php | 57 +++++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/mod/photos.php b/mod/photos.php index e23e4fcc92..af4c60b268 100644 --- a/mod/photos.php +++ b/mod/photos.php @@ -10,6 +10,8 @@ require_once('include/tags.php'); require_once('include/threads.php'); require_once('include/Probe.php'); +use \Friendica\Core\Config; + function photos_init(App $a) { if ($a->argc > 1) @@ -1339,35 +1341,38 @@ function photos_content(App $a) { $prevlink = ''; $nextlink = ''; - if ($_GET['order'] === 'posted') - $order = 'ASC'; - else - $order = 'DESC'; - /// @todo This query is totally bad, the whole functionality has to be changed - $prvnxt = qu("SELECT `resource-id` FROM `photo` WHERE `album` = '%s' AND `uid` = %d AND `scale` = 0 - $sql_extra ORDER BY `created` $order ", - dbesc($ph[0]['album']), - intval($owner_uid) - ); + // The query leads to a really intense used index. + // By now we hide it if someone wants to. + if (!Config::get('system', 'no_count', false)) { + if ($_GET['order'] === 'posted') + $order = 'ASC'; + else + $order = 'DESC'; - if (count($prvnxt)) { - for($z = 0; $z < count($prvnxt); $z++) { - if ($prvnxt[$z]['resource-id'] == $ph[0]['resource-id']) { - $prv = $z - 1; - $nxt = $z + 1; - if ($prv < 0) - $prv = count($prvnxt) - 1; - if ($nxt >= count($prvnxt)) - $nxt = 0; - break; + $prvnxt = qu("SELECT `resource-id` FROM `photo` WHERE `album` = '%s' AND `uid` = %d AND `scale` = 0 + $sql_extra ORDER BY `created` $order ", + dbesc($ph[0]['album']), + intval($owner_uid) + ); + + if (count($prvnxt)) { + for($z = 0; $z < count($prvnxt); $z++) { + if ($prvnxt[$z]['resource-id'] == $ph[0]['resource-id']) { + $prv = $z - 1; + $nxt = $z + 1; + if ($prv < 0) + $prv = count($prvnxt) - 1; + if ($nxt >= count($prvnxt)) + $nxt = 0; + break; + } } - } - $edit_suffix = ((($cmd === 'edit') && ($can_post)) ? '/edit' : ''); - $prevlink = 'photos/' . $a->data['user']['nickname'] . '/image/' . $prvnxt[$prv]['resource-id'] . $edit_suffix . (($_GET['order'] === 'posted') ? '?f=&order=posted' : ''); - $nextlink = 'photos/' . $a->data['user']['nickname'] . '/image/' . $prvnxt[$nxt]['resource-id'] . $edit_suffix . (($_GET['order'] === 'posted') ? '?f=&order=posted' : ''); - } - + $edit_suffix = ((($cmd === 'edit') && ($can_post)) ? '/edit' : ''); + $prevlink = 'photos/' . $a->data['user']['nickname'] . '/image/' . $prvnxt[$prv]['resource-id'] . $edit_suffix . (($_GET['order'] === 'posted') ? '?f=&order=posted' : ''); + $nextlink = 'photos/' . $a->data['user']['nickname'] . '/image/' . $prvnxt[$nxt]['resource-id'] . $edit_suffix . (($_GET['order'] === 'posted') ? '?f=&order=posted' : ''); + } + } if (count($ph) == 1) $hires = $lores = $ph[0]; From 99e9791d1ff2737a1b15e28800cc664968674cb8 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 15 Jan 2017 15:06:32 +0000 Subject: [PATCH 10/41] We fetched a little bit too much ... --- mod/fetch.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/fetch.php b/mod/fetch.php index 3576384f01..b87fc0e8e0 100644 --- a/mod/fetch.php +++ b/mod/fetch.php @@ -42,7 +42,7 @@ function fetch_init(App $a) { // Fetch some data from the author (We could combine both queries - but I think this is more readable) $r = q("SELECT `user`.`prvkey`, `contact`.`addr`, `user`.`nickname`, `contact`.`nick` FROM `user` - INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` + INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self` WHERE `user`.`uid` = %d", intval($item[0]["uid"])); if (!$r) { header($_SERVER["SERVER_PROTOCOL"].' 404 '.t('Not Found')); From 47d04416b16de856f4e4227d640a207e10ad4c34 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 15 Jan 2017 15:18:31 +0000 Subject: [PATCH 11/41] As well here ... --- mod/p.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/p.php b/mod/p.php index 8755464519..3cd7a9eb7d 100644 --- a/mod/p.php +++ b/mod/p.php @@ -46,7 +46,7 @@ function p_init($a){ // Fetch some data from the author (We could combine both queries - but I think this is more readable) $r = q("SELECT `user`.`prvkey`, `contact`.`addr`, `user`.`nickname`, `contact`.`nick` FROM `user` - INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` + INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self` WHERE `user`.`uid` = %d", intval($item[0]["uid"])); if (!dbm::is_result($r)) { header($_SERVER["SERVER_PROTOCOL"].' 404 '.t('Not Found')); From ced7cb6828da3f3acce0522161dd7ca5da3aa05b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 15 Jan 2017 23:30:43 +0000 Subject: [PATCH 12/41] Smarter way to create unique indexes --- include/dbstructure.php | 102 ++++++++++++++++++++++++++++++++++------ include/dfrn.php | 2 +- include/post_update.php | 2 +- 3 files changed, 89 insertions(+), 17 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index 6154ddeb08..9a84cfbc8d 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -176,15 +176,6 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { $definition = db_definition($charset); } - // Ensure index conversion to unique removes duplicates - $sql_config = "SET session old_alter_table=1;"; - if ($verbose) { - echo $sql_config."\n"; - } - if ($action) { - $db->q($sql_config); - } - // MySQL >= 5.7.4 doesn't support the IGNORE keyword in ALTER TABLE statements if ((version_compare($db->server_info(), '5.7.4') >= 0) AND !(strpos($db->server_info(), 'MariaDB') !== false)) { @@ -204,6 +195,26 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { } $is_new_table = True; } else { + $is_unique = false; + $temp_name = $name; + + foreach ($structure["indexes"] AS $indexname => $fieldnames) { + if (isset($database[$name]["indexes"][$indexname])) { + $current_index_definition = implode(",",$database[$name]["indexes"][$indexname]); + } else { + $current_index_definition = "__NOT_SET__"; + } + $new_index_definition = implode(",",$fieldnames); + if ($current_index_definition != $new_index_definition) { + if ($fieldnames[0] == "UNIQUE") { + $is_unique = true; + if ($ignore == "") { + $temp_name = "temp-".$name; + } + } + } + } + /* * Drop the index if it isn't present in the definition * or the definition differ from current status @@ -219,7 +230,7 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { if ($current_index_definition != $new_index_definition && substr($indexname, 0, 6) != 'local_') { $sql2=db_drop_index($indexname); if ($sql3 == "") { - $sql3 = "ALTER".$ignore." TABLE `".$name."` ".$sql2; + $sql3 = "ALTER".$ignore." TABLE `".$temp_name."` ".$sql2; } else { $sql3 .= ", ".$sql2; } @@ -230,7 +241,7 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { if (!isset($database[$name]["fields"][$fieldname])) { $sql2=db_add_table_field($fieldname, $parameters); if ($sql3 == "") { - $sql3 = "ALTER TABLE `".$name."` ".$sql2; + $sql3 = "ALTER TABLE `".$temp_name."` ".$sql2; } else { $sql3 .= ", ".$sql2; } @@ -241,7 +252,7 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { if ($current_field_definition != $new_field_definition) { $sql2=db_modify_table_field($fieldname, $parameters); if ($sql3 == "") { - $sql3 = "ALTER TABLE `".$name."` ".$sql2; + $sql3 = "ALTER TABLE `".$temp_name."` ".$sql2; } else { $sql3 .= ", ".$sql2; } @@ -268,7 +279,7 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { $sql2=db_create_index($indexname, $fieldnames); if ($sql2 != "") { if ($sql3 == "") - $sql3 = "ALTER" . $ignore . " TABLE `".$name."` ".$sql2; + $sql3 = "ALTER" . $ignore . " TABLE `".$temp_name."` ".$sql2; else $sql3 .= ", ".$sql2; } @@ -278,13 +289,74 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { if ($sql3 != "") { $sql3 .= ";"; - if ($verbose) + if ($verbose) { + // Ensure index conversion to unique removes duplicates + if ($is_unique) { + if ($ignore != "") { + echo "SET session old_alter_table=1;\n"; + } else { + echo "DROP TABLE IF EXISTS `".$temp_name."`;\n"; + echo "CREATE TABLE `".$temp_name."` LIKE `".$name."`;\n"; + } + } + echo $sql3."\n"; + if ($is_unique) { + if ($ignore != "") { + echo "SET session old_alter_table=0;\n"; + } else { + echo "INSERT IGNORE INTO `".$temp_name."` SELECT * FROM `".$name."`;\n"; + echo "DROP TABLE `".$name."`;\n"; + echo "RENAME TABLE `".$temp_name."` TO `".$name."`;\n"; + } + } + } + if ($action) { + // Ensure index conversion to unique removes duplicates + if ($is_unique) { + if ($ignore != "") { + $db->q("SET session old_alter_table=1;"); + } else { + $r = $db->q("DROP TABLE IF EXISTS `".$temp_name."`;"); + if (!dbm::is_result($r)) { + $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + return $errors; + } + $r = $db->q("CREATE TABLE `".$temp_name."` LIKE `".$name."`;"); + if (!dbm::is_result($r)) { + $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + return $errors; + } + } + } + $r = @$db->q($sql3); - if (dbm::is_result($r)) + if (!dbm::is_result($r)) $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + + if ($is_unique) { + if ($ignore != "") { + $db->q("SET session old_alter_table=0;"); + } else { + $r = $db->q("INSERT IGNORE INTO `".$temp_name."` SELECT * FROM `".$name."`;"); + if (!dbm::is_result($r)) { + $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + return $errors; + } + $r = $db->q("DROP TABLE `".$name."`;"); + if (!dbm::is_result($r)) { + $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + return $errors; + } + $r = $db->q("RENAME TABLE `".$temp_name."` TO `".$name."`;"); + if (!dbm::is_result($r)) { + $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + return $errors; + } + } + } } } } diff --git a/include/dfrn.php b/include/dfrn.php index ffdc2cbaf9..ccb43fa98e 100644 --- a/include/dfrn.php +++ b/include/dfrn.php @@ -194,7 +194,7 @@ class dfrn { `contact`.`name-date`, `contact`.`uri-date`, `contact`.`avatar-date`, `contact`.`thumb`, `contact`.`dfrn-id`, `contact`.`self`, `sign`.`signed_text`, `sign`.`signature`, `sign`.`signer` - FROM `item` USE INDEX (`uid_wall_changed`, `wall_uid_changed`) $sql_post_table + FROM `item` USE INDEX (`uid_wall_changed`) $sql_post_table STRAIGHT_JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND (NOT `contact`.`blocked` OR `contact`.`pending`) LEFT JOIN `sign` ON `sign`.`iid` = `item`.`id` diff --git a/include/post_update.php b/include/post_update.php index ae5aad26da..f9649961d9 100644 --- a/include/post_update.php +++ b/include/post_update.php @@ -239,7 +239,7 @@ function post_update_1206() { logger("Start", LOGGER_DEBUG); $r = q("SELECT `contact`.`id`, `contact`.`last-item`, - (SELECT MAX(`changed`) FROM `item` USE INDEX (`uid_wall_changed`, `wall_uid_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date` + (SELECT MAX(`changed`) FROM `item` USE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date` FROM `user` INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`"); From 2cdf87c56a42281b657e155bde82ef7d57426e01 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 16 Jan 2017 21:35:35 +0000 Subject: [PATCH 13/41] Deactivated the alternative update script by now --- boot.php | 2 +- include/dbstructure.php | 96 +++++++++++++++++++++-------------------- update.php | 2 +- 3 files changed, 52 insertions(+), 48 deletions(-) diff --git a/boot.php b/boot.php index cd6db384a2..e95cbc9d22 100644 --- a/boot.php +++ b/boot.php @@ -38,7 +38,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Asparagus'); define ( 'FRIENDICA_VERSION', '3.5.1-dev' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1212 ); +define ( 'DB_UPDATE_VERSION', 1213 ); /** * @brief Constant with a HTML line break. diff --git a/include/dbstructure.php b/include/dbstructure.php index 9a84cfbc8d..c1dc1ba877 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -208,9 +208,10 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { if ($current_index_definition != $new_index_definition) { if ($fieldnames[0] == "UNIQUE") { $is_unique = true; - if ($ignore == "") { - $temp_name = "temp-".$name; - } + // Deactivated. See below for the reason + //if ($ignore == "") { + // $temp_name = "temp-".$name; + //} } } } @@ -292,44 +293,44 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { if ($verbose) { // Ensure index conversion to unique removes duplicates if ($is_unique) { - if ($ignore != "") { + // By now the alternative is commented out. + // This is a preparation for the time when we found a good SQL routine. + //if ($ignore != "") { echo "SET session old_alter_table=1;\n"; - } else { - echo "DROP TABLE IF EXISTS `".$temp_name."`;\n"; - echo "CREATE TABLE `".$temp_name."` LIKE `".$name."`;\n"; - } + //} else { + // echo "CREATE TABLE `".$temp_name."` LIKE `".$name."`;\n"; + //} } echo $sql3."\n"; if ($is_unique) { - if ($ignore != "") { + // By now the alternative is commented out. + // This is a preparation for the time when we found a good SQL routine. + //if ($ignore != "") { echo "SET session old_alter_table=0;\n"; - } else { - echo "INSERT IGNORE INTO `".$temp_name."` SELECT * FROM `".$name."`;\n"; - echo "DROP TABLE `".$name."`;\n"; - echo "RENAME TABLE `".$temp_name."` TO `".$name."`;\n"; - } + //} else { + // echo "INSERT IGNORE INTO `".$temp_name."` SELECT * FROM `".$name."`;\n"; + // echo "DROP TABLE `".$name."`;\n"; + // echo "RENAME TABLE `".$temp_name."` TO `".$name."`;\n"; + //} } } if ($action) { // Ensure index conversion to unique removes duplicates if ($is_unique) { - if ($ignore != "") { + // By now the alternative is commented out. + // This is a preparation for the time when we found a good SQL routine. + //if ($ignore != "") { $db->q("SET session old_alter_table=1;"); - } else { - $r = $db->q("DROP TABLE IF EXISTS `".$temp_name."`;"); - if (!dbm::is_result($r)) { - $errors .= t('Errors encountered performing database changes.').$sql3.EOL; - return $errors; - } - $r = $db->q("CREATE TABLE `".$temp_name."` LIKE `".$name."`;"); - if (!dbm::is_result($r)) { - $errors .= t('Errors encountered performing database changes.').$sql3.EOL; - return $errors; - } - } + //} else { + // $r = $db->q("CREATE TABLE `".$temp_name."` LIKE `".$name."`;"); + // if (!dbm::is_result($r)) { + // $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + // return $errors; + // } + //} } $r = @$db->q($sql3); @@ -337,25 +338,28 @@ function update_structure($verbose, $action, $tables=null, $definition=null) { $errors .= t('Errors encountered performing database changes.').$sql3.EOL; if ($is_unique) { - if ($ignore != "") { + // By now the alternative is commented out. + // This is a preparation for the time when we found a good SQL routine. + //if ($ignore != "") { $db->q("SET session old_alter_table=0;"); - } else { - $r = $db->q("INSERT IGNORE INTO `".$temp_name."` SELECT * FROM `".$name."`;"); - if (!dbm::is_result($r)) { - $errors .= t('Errors encountered performing database changes.').$sql3.EOL; - return $errors; - } - $r = $db->q("DROP TABLE `".$name."`;"); - if (!dbm::is_result($r)) { - $errors .= t('Errors encountered performing database changes.').$sql3.EOL; - return $errors; - } - $r = $db->q("RENAME TABLE `".$temp_name."` TO `".$name."`;"); - if (!dbm::is_result($r)) { - $errors .= t('Errors encountered performing database changes.').$sql3.EOL; - return $errors; - } - } + //} else { + // We have to check if "INSERT IGNORE" will work on newer MySQL versions + // $r = $db->q("INSERT IGNORE INTO `".$temp_name."` SELECT * FROM `".$name."`;"); + // if (!dbm::is_result($r)) { + // $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + // return $errors; + // } + // $r = $db->q("DROP TABLE `".$name."`;"); + // if (!dbm::is_result($r)) { + // $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + // return $errors; + // } + // $r = $db->q("RENAME TABLE `".$temp_name."` TO `".$name."`;"); + // if (!dbm::is_result($r)) { + // $errors .= t('Errors encountered performing database changes.').$sql3.EOL; + // return $errors; + // } + //} } } } @@ -1260,7 +1264,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "poll_id" => array("poll_id"), + "poll_id" => array("UNIQUE", "poll_id"), "choice" => array("choice"), ) ); diff --git a/update.php b/update.php index 058536d821..25d6cb9cbf 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ Date: Mon, 16 Jan 2017 21:45:27 +0000 Subject: [PATCH 14/41] Changed database version --- database.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.sql b/database.sql index 9d4d289f2f..1b7e2cb4cc 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 3.5.1-dev (Asparagus) --- DB_UPDATE_VERSION 1212 +-- DB_UPDATE_VERSION 1213 -- ------------------------------------------ From 6e1986a0f28a04a0d3b8398cdf8db482b7a0deb5 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 16 Jan 2017 21:48:35 +0000 Subject: [PATCH 15/41] Removed commented text --- include/photos.php | 1 - 1 file changed, 1 deletion(-) diff --git a/include/photos.php b/include/photos.php index f3043e44ac..9d8d3309c2 100644 --- a/include/photos.php +++ b/include/photos.php @@ -57,7 +57,6 @@ function photo_albums($uid, $update = false) { dbesc(t('Contact Photos')) ); } else { -// USE INDEX (`uid_album`) // This query doesn't do the count and is much faster $albums = qu("SELECT DISTINCT(`album`), '' AS `total` FROM `photo` From ef3112743030c3e1606637a847b33d23afdff247 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 16 Jan 2017 22:00:58 +0000 Subject: [PATCH 16/41] Added documentation --- doc/htconfig.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/htconfig.md b/doc/htconfig.md index 54808aaae5..a7dd59d1f5 100644 --- a/doc/htconfig.md +++ b/doc/htconfig.md @@ -26,6 +26,13 @@ Example: To set the directory value please add this line to your .htconfig.php: * **birthday_input_format** - Default value is "ymd". * **block_local_dir** (Boolean) - Blocks the access to the directory of the local users. * **curl_range_bytes** - Maximum number of bytes that should be fetched. Default is 0, which mean "no limit". +* **db_log** - Name of a logfile to log slow database queries +* **db_loglimit** - If a database call lasts longer than this value it is logged +* **db_log_index** - Name of a logfile to log queries with bad indexes +* **db_log_index_watch** - Watchlist of indexes to watch +* **db_loglimit_index** - Number of index rows needed to be logged for indexes on the watchlist +* **db_loglimit_index_high** - Number of index rows to be logged anyway (for any index) +* **db_log_index_blacklist** - Blacklist of indexes that shouldn't be watched * **dbclean** (Boolean) - Enable the automatic database cleanup process * **default_service_class** - * **delivery_batch_count** - Number of deliveries per process. Default value is 1. (Disabled when using the worker) From 6972faa3a8d756590233c67929ae8bee58622f04 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 16 Jan 2017 22:11:35 +0000 Subject: [PATCH 17/41] Redo some work --- database.sql | 24 ++++++++++++------------ include/dbstructure.php | 26 +++++++++++++------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/database.sql b/database.sql index 1b7e2cb4cc..a12f9d2046 100644 --- a/database.sql +++ b/database.sql @@ -31,10 +31,10 @@ CREATE TABLE IF NOT EXISTS `attach` ( `data` longblob NOT NULL, `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `edited` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', - `allow_cid` text, - `allow_gid` text, - `deny_cid` text, - `deny_gid` text, + `allow_cid` mediumtext, + `allow_gid` medium_text, + `deny_cid` medium_text, + `deny_gid` medium_text, PRIMARY KEY(`id`) ) DEFAULT CHARSET=utf8mb4; @@ -235,10 +235,10 @@ CREATE TABLE IF NOT EXISTS `event` ( `nofinish` tinyint(1) NOT NULL DEFAULT 0, `adjust` tinyint(1) NOT NULL DEFAULT 1, `ignore` tinyint(1) unsigned NOT NULL DEFAULT 0, - `allow_cid` text, - `allow_gid` text, - `deny_cid` text, - `deny_gid` text, + `allow_cid` medium_text, + `allow_gid` medium_text, + `deny_cid` medium_text, + `deny_gid` medium_text, PRIMARY KEY(`id`), INDEX `uid_start` (`uid`,`start`) ) DEFAULT CHARSET=utf8mb4; @@ -1073,10 +1073,10 @@ CREATE TABLE IF NOT EXISTS `user` ( `expire_notification_sent` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `service_class` varchar(32) NOT NULL DEFAULT '', `def_gid` int(11) NOT NULL DEFAULT 0, - `allow_cid` text, - `allow_gid` text, - `deny_cid` text, - `deny_gid` text, + `allow_cid` medium_text, + `allow_gid` medium_text, + `deny_cid` medium_text, + `deny_gid` medium_text, `openidserver` text, PRIMARY KEY(`uid`), INDEX `nickname` (`nickname`(32)) diff --git a/include/dbstructure.php b/include/dbstructure.php index c1dc1ba877..56d2a17033 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -517,10 +517,10 @@ function db_definition($charset) { "data" => array("type" => "longblob", "not null" => "1"), "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "edited" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), - "allow_cid" => array("type" => "text"), - "allow_gid" => array("type" => "text"), - "deny_cid" => array("type" => "text"), - "deny_gid" => array("type" => "text"), + "allow_cid" => array("type" => "mediumtext"), + "allow_gid" => array("type" => "medium_text"), + "deny_cid" => array("type" => "medium_text"), + "deny_gid" => array("type" => "medium_text"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -721,10 +721,10 @@ function db_definition($charset) { "nofinish" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), "adjust" => array("type" => "tinyint(1)", "not null" => "1", "default" => "1"), "ignore" => array("type" => "tinyint(1) unsigned", "not null" => "1", "default" => "0"), - "allow_cid" => array("type" => "text"), - "allow_gid" => array("type" => "text"), - "deny_cid" => array("type" => "text"), - "deny_gid" => array("type" => "text"), + "allow_cid" => array("type" => "medium_text"), + "allow_gid" => array("type" => "medium_text"), + "deny_cid" => array("type" => "medium_text"), + "deny_gid" => array("type" => "medium_text"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -1264,7 +1264,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "poll_id" => array("UNIQUE", "poll_id"), + "poll_id" => array("poll_id"), "choice" => array("choice"), ) ); @@ -1559,10 +1559,10 @@ function db_definition($charset) { "expire_notification_sent" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "service_class" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), "def_gid" => array("type" => "int(11)", "not null" => "1", "default" => "0"), - "allow_cid" => array("type" => "text"), - "allow_gid" => array("type" => "text"), - "deny_cid" => array("type" => "text"), - "deny_gid" => array("type" => "text"), + "allow_cid" => array("type" => "medium_text"), + "allow_gid" => array("type" => "medium_text"), + "deny_cid" => array("type" => "medium_text"), + "deny_gid" => array("type" => "medium_text"), "openidserver" => array("type" => "text"), ), "indexes" => array( From 79866f620abcd065e16d81550e54a705fcf340ce Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 16 Jan 2017 22:15:04 +0000 Subject: [PATCH 18/41] OOpppss ... --- include/dbstructure.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index 56d2a17033..283d39d22c 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -518,9 +518,9 @@ function db_definition($charset) { "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "edited" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "allow_cid" => array("type" => "mediumtext"), - "allow_gid" => array("type" => "medium_text"), - "deny_cid" => array("type" => "medium_text"), - "deny_gid" => array("type" => "medium_text"), + "allow_gid" => array("type" => "mediumtext"), + "deny_cid" => array("type" => "mediumtext"), + "deny_gid" => array("type" => "mediumtext"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -721,10 +721,10 @@ function db_definition($charset) { "nofinish" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), "adjust" => array("type" => "tinyint(1)", "not null" => "1", "default" => "1"), "ignore" => array("type" => "tinyint(1) unsigned", "not null" => "1", "default" => "0"), - "allow_cid" => array("type" => "medium_text"), - "allow_gid" => array("type" => "medium_text"), - "deny_cid" => array("type" => "medium_text"), - "deny_gid" => array("type" => "medium_text"), + "allow_cid" => array("type" => "mediumtext"), + "allow_gid" => array("type" => "mediumtext"), + "deny_cid" => array("type" => "mediumtext"), + "deny_gid" => array("type" => "mediumtext"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -1559,10 +1559,10 @@ function db_definition($charset) { "expire_notification_sent" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "service_class" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), "def_gid" => array("type" => "int(11)", "not null" => "1", "default" => "0"), - "allow_cid" => array("type" => "medium_text"), - "allow_gid" => array("type" => "medium_text"), - "deny_cid" => array("type" => "medium_text"), - "deny_gid" => array("type" => "medium_text"), + "allow_cid" => array("type" => "mediumtext"), + "allow_gid" => array("type" => "mediumtext"), + "deny_cid" => array("type" => "mediumtext"), + "deny_gid" => array("type" => "mediumtext"), "openidserver" => array("type" => "text"), ), "indexes" => array( From 53393233c34ce9fa0a38ec62e884283cd5ce40b1 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 Jan 2017 19:21:46 +0000 Subject: [PATCH 19/41] Replace the direct access of config variables --- boot.php | 18 ++++++++++-------- include/api.php | 7 +++++-- include/datetime.php | 6 ++++-- include/pgettext.php | 8 +++++--- mod/dfrn_confirm.php | 4 +--- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/boot.php b/boot.php index e95cbc9d22..a9b4e4d293 100644 --- a/boot.php +++ b/boot.php @@ -19,6 +19,8 @@ require_once('include/autoloader.php'); +use \Friendica\Core\Config; + require_once('include/config.php'); require_once('include/network.php'); require_once('include/plugin.php'); @@ -1475,9 +1477,7 @@ function system_unavailable() { function clean_urls() { $a = get_app(); - // if($a->config['system']['clean_urls']) return true; - // return false; } function z_path() { @@ -2041,16 +2041,18 @@ function current_theme(){ // $is_mobile = $mobile_detect->isMobile() || $mobile_detect->isTablet(); $is_mobile = $a->is_mobile || $a->is_tablet; - $standard_system_theme = ((isset($a->config['system']['theme'])) ? $a->config['system']['theme'] : ''); + $standard_system_theme = Config::get('system', 'theme', ''); $standard_theme_name = ((isset($_SESSION) && x($_SESSION,'theme')) ? $_SESSION['theme'] : $standard_system_theme); - if($is_mobile) { - if(isset($_SESSION['show-mobile']) && !$_SESSION['show-mobile']) { + if ($is_mobile) { + if (isset($_SESSION['show-mobile']) && !$_SESSION['show-mobile']) { $system_theme = $standard_system_theme; $theme_name = $standard_theme_name; - } - else { - $system_theme = ((isset($a->config['system']['mobile-theme'])) ? $a->config['system']['mobile-theme'] : $standard_system_theme); + } else { + $system_theme = Config::get('system', 'mobile-theme', ''); + if ($system_theme == '') { + $system_theme = $standard_system_theme; + } $theme_name = ((isset($_SESSION) && x($_SESSION,'mobile-theme')) ? $_SESSION['mobile-theme'] : $system_theme); if($theme_name === '---') { diff --git a/include/api.php b/include/api.php index 64a896b554..ce76103129 100644 --- a/include/api.php +++ b/include/api.php @@ -5,6 +5,9 @@ * * @todo Automatically detect if incoming data is HTML or BBCode */ + +use \Friendica\Core\Config; + require_once('include/HTTPExceptions.php'); require_once('include/bbcode.php'); @@ -2696,11 +2699,11 @@ $logo = App::get_baseurl() . '/images/friendica-64.png'; $email = $a->config['admin_email']; $closed = (($a->config['register_policy'] == REGISTER_CLOSED) ? 'true' : 'false'); - $private = (($a->config['system']['block_public']) ? 'true' : 'false'); + $private = ((Config::get('system', 'block_public')) ? 'true' : 'false'); $textlimit = (string) (($a->config['max_import_size']) ? $a->config['max_import_size'] : 200000); if($a->config['api_import_size']) $texlimit = string($a->config['api_import_size']); - $ssl = (($a->config['system']['have_ssl']) ? 'true' : 'false'); + $ssl = ((Config::get('system', 'have_ssl')) ? 'true' : 'false'); $sslserver = (($ssl === 'true') ? str_replace('http:','https:',App::get_baseurl()) : ''); $config = array( diff --git a/include/datetime.php b/include/datetime.php index a17c405dc3..779c7a5aad 100644 --- a/include/datetime.php +++ b/include/datetime.php @@ -4,6 +4,7 @@ * @brief Some functions for date and time related tasks. */ +use \Friendica\Core\Config; /** * @brief Two-level sort for timezones. @@ -271,8 +272,9 @@ function datetimesel($format, $min, $max, $default, $label, $id = 'datetimepicke $lang = substr(get_browser_language(), 0, 2); // Check if the detected language is supported by the picker - if (!in_array($lang, array("ar", "ro", "id", "bg", "fa", "ru", "uk", "en", "el", "de", "nl", "tr", "fr", "es", "th", "pl", "pt", "ch", "se", "kr", "it", "da", "no", "ja", "vi", "sl", "cs", "hu"))) - $lang = ((isset($a->config['system']['language'])) ? $a->config['system']['language'] : 'en'); + if (!in_array($lang, array("ar", "ro", "id", "bg", "fa", "ru", "uk", "en", "el", "de", "nl", "tr", "fr", "es", "th", "pl", "pt", "ch", "se", "kr", "it", "da", "no", "ja", "vi", "sl", "cs", "hu"))) { + $lang = Config::get('system', 'language', 'en'); + } $o = ''; $dateformat = ''; diff --git a/include/pgettext.php b/include/pgettext.php index fb87798ff7..335869eda2 100644 --- a/include/pgettext.php +++ b/include/pgettext.php @@ -10,6 +10,8 @@ * */ +use \Friendica\Core\Config; + require_once("include/dba.php"); if(! function_exists('get_browser_language')) { @@ -47,12 +49,12 @@ function get_browser_language() { break; } } - if(isset($preferred)) + if (isset($preferred)) { return $preferred; + } // in case none matches, get the system wide configured language, or fall back to English - $a = get_app(); - return ((isset($a->config['system']['language'])) ? $a->config['system']['language'] : 'en'); + return Config::get('system', 'language', 'en'); }} diff --git a/mod/dfrn_confirm.php b/mod/dfrn_confirm.php index 57ddc58f2c..e2ce806275 100644 --- a/mod/dfrn_confirm.php +++ b/mod/dfrn_confirm.php @@ -224,9 +224,7 @@ function dfrn_confirm_post(App $a, $handsfree = null) { * */ - $a->config['system']['curl_timeout'] = 120; - - $res = post_url($dfrn_confirm,$params); + $res = post_url($dfrn_confirm, $params, null, $redirects, 120); logger(' Confirm: received data: ' . $res, LOGGER_DATA); From 045e94ccf3291c817bf1683b63682b1e0af74980 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 Jan 2017 19:33:06 +0000 Subject: [PATCH 20/41] Some more config stuff --- boot.php | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/boot.php b/boot.php index a9b4e4d293..d598ef8666 100644 --- a/boot.php +++ b/boot.php @@ -825,24 +825,22 @@ class App { $scheme = $this->scheme; - if ((x($this->config, 'system')) && (x($this->config['system'], 'ssl_policy'))) { - if (intval($this->config['system']['ssl_policy']) === SSL_POLICY_FULL) { + if (Config::get('system', 'ssl_policy') === SSL_POLICY_FULL) { + $scheme = 'https'; + } + + // Basically, we have $ssl = true on any links which can only be seen by a logged in user + // (and also the login link). Anything seen by an outsider will have it turned off. + + if (Config::get('system', 'ssl_policy') == SSL_POLICY_SELFSIGN) { + if ($ssl) { $scheme = 'https'; - } - - // Basically, we have $ssl = true on any links which can only be seen by a logged in user - // (and also the login link). Anything seen by an outsider will have it turned off. - - if ($this->config['system']['ssl_policy'] == SSL_POLICY_SELFSIGN) { - if ($ssl) { - $scheme = 'https'; - } else { - $scheme = 'http'; - } + } else { + $scheme = 'http'; } } - if (get_config('config', 'hostname') != '') { + if (Config::get('config', 'hostname') != '') { $this->hostname = get_config('config', 'hostname'); } From 0548099f6cb89ccb1c798d092b298b3a03fd8d88 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 18 Jan 2017 21:45:32 +0000 Subject: [PATCH 21/41] The config class now makes less database reads. --- boot.php | 2 +- database.sql | 29 +++++++-------- include/Core/Config.php | 64 +++++++++++++++++++++------------- include/cli_startup.php | 17 ++++----- include/create_shadowentry.php | 6 ++-- include/cron.php | 5 +-- include/cronhooks.php | 6 ++-- include/cronjobs.php | 5 +-- include/dbclean.php | 3 +- include/dbstructure.php | 5 +-- include/dbupdate.php | 5 +-- include/delivery.php | 6 ++-- include/directory.php | 8 ++--- include/discover_poco.php | 6 ++-- include/expire.php | 15 ++++---- include/gprobe.php | 5 +-- include/notifier.php | 7 ++-- include/onepoll.php | 5 +-- include/pubsubpublish.php | 3 +- include/queue.php | 6 ++-- include/remove_contact.php | 6 ++-- include/shadowupdate.php | 6 ++-- include/spool_post.php | 6 ++-- include/tagupdate.php | 6 ++-- include/threadupdate.php | 6 ++-- include/update_gcontact.php | 5 +-- index.php | 5 +-- mod/friendica.php | 4 ++- util/maintenance.php | 8 +++-- 29 files changed, 153 insertions(+), 107 deletions(-) diff --git a/boot.php b/boot.php index d598ef8666..501ae38c42 100644 --- a/boot.php +++ b/boot.php @@ -1569,7 +1569,7 @@ function update_db(App $a) { $stored = intval($build); $current = intval(DB_UPDATE_VERSION); if($stored < $current) { - load_config('database'); + Config::load('database'); // We're reporting a different version than what is currently installed. // Run any existing update scripts to bring the database up to current. diff --git a/database.sql b/database.sql index a12f9d2046..383253fcb1 100644 --- a/database.sql +++ b/database.sql @@ -9,13 +9,14 @@ -- CREATE TABLE IF NOT EXISTS `addon` ( `id` int(11) NOT NULL auto_increment, - `name` varchar(255) NOT NULL DEFAULT '', + `name` varchar(190) NOT NULL DEFAULT '', `version` varchar(255) NOT NULL DEFAULT '', `installed` tinyint(1) NOT NULL DEFAULT 0, `hidden` tinyint(1) NOT NULL DEFAULT 0, `timestamp` bigint(20) NOT NULL DEFAULT 0, `plugin_admin` tinyint(1) NOT NULL DEFAULT 0, - PRIMARY KEY(`id`) + PRIMARY KEY(`id`), + UNIQUE INDEX `name` (`name`) ) DEFAULT CHARSET=utf8mb4; -- @@ -32,9 +33,9 @@ CREATE TABLE IF NOT EXISTS `attach` ( `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `edited` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `allow_cid` mediumtext, - `allow_gid` medium_text, - `deny_cid` medium_text, - `deny_gid` medium_text, + `allow_gid` mediumtext, + `deny_cid` mediumtext, + `deny_gid` mediumtext, PRIMARY KEY(`id`) ) DEFAULT CHARSET=utf8mb4; @@ -235,10 +236,10 @@ CREATE TABLE IF NOT EXISTS `event` ( `nofinish` tinyint(1) NOT NULL DEFAULT 0, `adjust` tinyint(1) NOT NULL DEFAULT 1, `ignore` tinyint(1) unsigned NOT NULL DEFAULT 0, - `allow_cid` medium_text, - `allow_gid` medium_text, - `deny_cid` medium_text, - `deny_gid` medium_text, + `allow_cid` mediumtext, + `allow_gid` mediumtext, + `deny_cid` mediumtext, + `deny_gid` mediumtext, PRIMARY KEY(`id`), INDEX `uid_start` (`uid`,`start`) ) DEFAULT CHARSET=utf8mb4; @@ -434,7 +435,7 @@ CREATE TABLE IF NOT EXISTS `hook` ( `function` varchar(255) NOT NULL DEFAULT '', `priority` int(11) unsigned NOT NULL DEFAULT 0, PRIMARY KEY(`id`), - INDEX `hook_file_function` (`hook`(30),`file`(60),`function`(30)) + UNIQUE INDEX `hook_file_function` (`hook`(50),`file`(80),`function`(60)) ) DEFAULT CHARSET=utf8mb4; -- @@ -1073,10 +1074,10 @@ CREATE TABLE IF NOT EXISTS `user` ( `expire_notification_sent` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `service_class` varchar(32) NOT NULL DEFAULT '', `def_gid` int(11) NOT NULL DEFAULT 0, - `allow_cid` medium_text, - `allow_gid` medium_text, - `deny_cid` medium_text, - `deny_gid` medium_text, + `allow_cid` mediumtext, + `allow_gid` mediumtext, + `deny_cid` mediumtext, + `deny_gid` mediumtext, `openidserver` text, PRIMARY KEY(`uid`), INDEX `nickname` (`nickname`(32)) diff --git a/include/Core/Config.php b/include/Core/Config.php index 7b7045a9ee..574ff5b8a5 100644 --- a/include/Core/Config.php +++ b/include/Core/Config.php @@ -22,6 +22,8 @@ use dbm; */ class Config { + private static $cache; + /** * @brief Loads all configuration values of family into a cached storage. * @@ -32,10 +34,17 @@ class Config { * The category of the configuration value * @return void */ - public static function load($family) { + public static function load($family = "config") { + + // We don't preload "system" anymore. + // This reduces the number of database reads a lot. + if ($family == 'system') { + return; + } + $a = get_app(); - $r = q("SELECT `v`, `k` FROM `config` WHERE `cat` = '%s' ORDER BY `cat`, `k`, `id`", dbesc($family)); + $r = q("SELECT `v`, `k` FROM `config` WHERE `cat` = '%s'", dbesc($family)); if (dbm::is_result($r)) { foreach ($r as $rr) { $k = $rr['k']; @@ -43,11 +52,9 @@ class Config { $a->config[$k] = $rr['v']; } else { $a->config[$family][$k] = $rr['v']; + self::$cache[$family][$k] = $rr['v']; } } - } else if ($family != 'config') { - // Negative caching - $a->config[$family] = "!!"; } } @@ -78,34 +85,38 @@ class Config { $a = get_app(); if (!$refresh) { - // Looking if the whole family isn't set - if (isset($a->config[$family])) { - if ($a->config[$family] === '!!') { - return $default_value; - } - } - if (isset($a->config[$family][$key])) { - if ($a->config[$family][$key] === '!!') { + // Do we have the cached value? Then return it + if (isset(self::$cache[$family][$key])) { + if (self::$cache[$family][$key] == '!!') { return $default_value; + } else { + return self::$cache[$family][$key]; } - return $a->config[$family][$key]; } } - $ret = q("SELECT `v` FROM `config` WHERE `cat` = '%s' AND `k` = '%s' ORDER BY `id` DESC LIMIT 1", + $ret = q("SELECT `v` FROM `config` WHERE `cat` = '%s' AND `k` = '%s'", dbesc($family), dbesc($key) ); - if (count($ret)) { + if (dbm::is_result($ret)) { // manage array value $val = (preg_match("|^a:[0-9]+:{.*}$|s", $ret[0]['v'])?unserialize( $ret[0]['v']):$ret[0]['v']); - $a->config[$family][$key] = $val; + // Assign the value from the database to the cache + self::$cache[$family][$key] = $val; return $val; - } else { - $a->config[$family][$key] = '!!'; + } elseif (isset($a->config[$family][$key])) { + + // Assign the value (mostly) from the .htconfig.php to the cache + self::$cache[$family][$key] = $a->config[$family][$key]; + + return $a->config[$family][$key]; } + + self::$cache[$family][$key] = '!!'; + return $default_value; } @@ -134,7 +145,14 @@ class Config { return true; } - $a->config[$family][$key] = $value; + if ($family === 'config') { + $a->config[$key] = $value; + } elseif ($family != 'system') { + $a->config[$family][$key] = $value; + } + + // Assign the just added value to the cache + self::$cache[$family][$key] = $value; // manage array value $dbvalue = (is_array($value) ? serialize($value) : $value); @@ -174,9 +192,8 @@ class Config { */ public static function delete($family, $key) { - $a = get_app(); - if (x($a->config[$family],$key)) { - unset($a->config[$family][$key]); + if (isset(self::$cache[$family][$key])) { + unset(self::$cache[$family][$key]); } $ret = q("DELETE FROM `config` WHERE `cat` = '%s' AND `k` = '%s'", dbesc($family), @@ -185,5 +202,4 @@ class Config { return $ret; } - } diff --git a/include/cli_startup.php b/include/cli_startup.php index 4cb86adef8..4b66b47a00 100644 --- a/include/cli_startup.php +++ b/include/cli_startup.php @@ -1,5 +1,7 @@ set_baseurl(get_config('system','url')); diff --git a/include/create_shadowentry.php b/include/create_shadowentry.php index f06a0dd1bf..005295c978 100644 --- a/include/create_shadowentry.php +++ b/include/create_shadowentry.php @@ -5,6 +5,9 @@ * * This script is started from mod/item.php to save some time when doing a post. */ + +use \Friendica\Core\Config; + require_once("boot.php"); require_once("include/threads.php"); @@ -21,8 +24,7 @@ function create_shadowentry_run($argv, $argc) { unset($db_host, $db_user, $db_pass, $db_data); } - load_config('config'); - load_config('system'); + Config::load(); if ($argc != 2) { return; diff --git a/include/cron.php b/include/cron.php index f7def61215..2fc8de51c5 100644 --- a/include/cron.php +++ b/include/cron.php @@ -10,6 +10,8 @@ if (!file_exists("boot.php") AND (sizeof($_SERVER["argv"]) != 0)) { chdir($directory); } +use \Friendica\Core\Config; + require_once("boot.php"); require_once("include/photos.php"); require_once("include/user.php"); @@ -38,8 +40,7 @@ function cron_run(&$argv, &$argc){ require_once('mod/nodeinfo.php'); require_once('include/post_update.php'); - load_config('config'); - load_config('system'); + Config::load(); // Don't check this stuff if the function is called by the poller if (App::callstack() != "poller_run") { diff --git a/include/cronhooks.php b/include/cronhooks.php index 7524a0c3a8..72b86be427 100644 --- a/include/cronhooks.php +++ b/include/cronhooks.php @@ -1,7 +1,8 @@ set_baseurl(get_config('system','url')); diff --git a/include/dbclean.php b/include/dbclean.php index c80e5a3be3..8408ad1882 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -23,8 +23,7 @@ function dbclean_run(&$argv, &$argc) { unset($db_host, $db_user, $db_pass, $db_data); } - Config::load('config'); - Config::load('system'); + Config::load(); if (!Config::get('system', 'dbclean', false)) { return; diff --git a/include/dbstructure.php b/include/dbstructure.php index 283d39d22c..0fe157f329 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -495,7 +495,7 @@ function db_definition($charset) { $database["addon"] = array( "fields" => array( "id" => array("type" => "int(11)", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), - "name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "name" => array("type" => "varchar(190)", "not null" => "1", "default" => ""), "version" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "installed" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), "hidden" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), @@ -504,6 +504,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), + "name" => array("UNIQUE", "name"), ) ); $database["attach"] = array( @@ -922,7 +923,7 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "hook_file_function" => array("hook(30)","file(60)","function(30)"), + "hook_file_function" => array("UNIQUE", "hook(50)","file(80)","function(60)"), ) ); $database["intro"] = array( diff --git a/include/dbupdate.php b/include/dbupdate.php index 28f1de340b..3583be3106 100644 --- a/include/dbupdate.php +++ b/include/dbupdate.php @@ -1,5 +1,7 @@ set_baseurl(get_config('system','url')); diff --git a/include/gprobe.php b/include/gprobe.php index 7169aada3f..4407fa6d6c 100644 --- a/include/gprobe.php +++ b/include/gprobe.php @@ -1,5 +1,7 @@ set_baseurl(get_config('system','url')); diff --git a/include/notifier.php b/include/notifier.php index 7bea239c6f..24830a11ab 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -1,4 +1,7 @@ set_baseurl(get_config('system','url')); diff --git a/include/pubsubpublish.php b/include/pubsubpublish.php index 6bd90bfc21..428103a971 100644 --- a/include/pubsubpublish.php +++ b/include/pubsubpublish.php @@ -72,8 +72,7 @@ function pubsubpublish_run(&$argv, &$argc){ require_once('include/items.php'); - load_config('config'); - load_config('system'); + Config::load(); // Don't check this stuff if the function is called by the poller if (App::callstack() != "poller_run") { diff --git a/include/queue.php b/include/queue.php index f36e7723cd..bcd32985db 100644 --- a/include/queue.php +++ b/include/queue.php @@ -1,4 +1,7 @@ set_baseurl(get_config('system','url')); diff --git a/index.php b/index.php index f05151757b..7408f495cd 100644 --- a/index.php +++ b/index.php @@ -13,6 +13,8 @@ * */ +use \Friendica\Core\Config; + require_once('boot.php'); require_once('object/BaseObject.php'); @@ -54,8 +56,7 @@ if(!$install) { * Load configs from db. Overwrite configs from .htconfig.php */ - load_config('config'); - load_config('system'); + Config::load(); if ($a->max_processes_reached() OR $a->maxload_reached()) { header($_SERVER["SERVER_PROTOCOL"].' 503 Service Temporarily Unavailable'); diff --git a/mod/friendica.php b/mod/friendica.php index 3f242f7c56..f613dfd39c 100644 --- a/mod/friendica.php +++ b/mod/friendica.php @@ -1,5 +1,7 @@ argv[1]=="json"){ $register_policy = Array('REGISTER_CLOSED', 'REGISTER_APPROVE', 'REGISTER_OPEN'); @@ -29,7 +31,7 @@ function friendica_init(App $a) { $visible_plugins[] = $rr['name']; } - load_config('feature_lock'); + Config::load('feature_lock'); $locked_features = array(); if(is_array($a->config['feature_lock']) && count($a->config['feature_lock'])) { foreach($a->config['feature_lock'] as $k => $v) { diff --git a/util/maintenance.php b/util/maintenance.php index d1ff94524e..28f3a503ad 100644 --- a/util/maintenance.php +++ b/util/maintenance.php @@ -1,5 +1,7 @@ 1) From 4e91379f4adc719d2e78906341c8ea12331cb43e Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 19 Jan 2017 07:09:56 +0000 Subject: [PATCH 22/41] Added reminder --- include/acl_selectors.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/acl_selectors.php b/include/acl_selectors.php index fccdb80665..f4b644d68f 100644 --- a/include/acl_selectors.php +++ b/include/acl_selectors.php @@ -495,6 +495,8 @@ function acl_lookup(App $a, $out_type = 'json') { if ($type=='' || $type=='g'){ + /// @todo We should cache this query. + // This can be done when we can delete cache entries via wildcard $r = q("SELECT `group`.`id`, `group`.`name`, GROUP_CONCAT(DISTINCT `group_member`.`contact-id` SEPARATOR ',') AS uids FROM `group` INNER JOIN `group_member` ON `group_member`.`gid`=`group`.`id` AND `group_member`.`uid` = `group`.`uid` From 75097ebf37df318dc95ad997664924c0c2be6462 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 19 Jan 2017 10:00:44 +0000 Subject: [PATCH 23/41] Added limit --- mod/network.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mod/network.php b/mod/network.php index 2f33c62f72..12223494a7 100644 --- a/mod/network.php +++ b/mod/network.php @@ -762,24 +762,25 @@ function network_content(App $a, $update = 0) { // on they just get buried deeper. It has happened to me a couple of times also. - if((! $group) && (! $cid) && (! $star)) { + if (!$group && !$cid && !$star) { - $unseen = q("SELECT `id` FROM `item` WHERE `unseen` AND `uid` = %d", + $unseen = q("SELECT `id` FROM `item` WHERE `unseen` AND `uid` = %d LIMIT 1", intval(local_user())); - if ($unseen) + if (dbm::is_result($unseen)) { $r = q("UPDATE `item` SET `unseen` = 0 WHERE `unseen` = 1 AND `uid` = %d", intval(local_user()) ); - } - else { - if($update_unseen) { + } + } else { + if ($update_unseen) { - $unseen = q("SELECT `id` FROM `item` ".$update_unseen); + $unseen = q("SELECT `id` FROM `item` ".$update_unseen. " LIMIT 1"); - if ($unseen) + if (dbm::is_result($unseen)) { $r = q("UPDATE `item` SET `unseen` = 0 $update_unseen"); + } } } From 1360a0a003ca6a2aa0ebaec92156d296e92bf14d Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 19 Jan 2017 17:06:23 +0000 Subject: [PATCH 24/41] The worker now tells the process runtime length. --- include/poller.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/poller.php b/include/poller.php index b631d2acd1..b0c594db15 100644 --- a/include/poller.php +++ b/include/poller.php @@ -153,6 +153,9 @@ function poller_execute($queue) { $funcname = str_replace(".php", "", basename($argv[0]))."_run"; if (function_exists($funcname)) { + + $stamp = (float)microtime(true); + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]); // For better logging create a new process id for every worker call @@ -169,7 +172,9 @@ function poller_execute($queue) { sleep($cooldown); } - logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done"); + $duration = (float)round(microtime(true)-$stamp, 3); + + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds."); q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"])); } else { From 6bbc0e4c78b638e52291f93179e0e50bb2985287 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 19 Jan 2017 22:47:20 +0000 Subject: [PATCH 25/41] Bugfix: The caching of values didn't really work for boolean values --- include/Core/Config.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/Core/Config.php b/include/Core/Config.php index 574ff5b8a5..1d0e66ddc0 100644 --- a/include/Core/Config.php +++ b/include/Core/Config.php @@ -38,7 +38,7 @@ class Config { // We don't preload "system" anymore. // This reduces the number of database reads a lot. - if ($family == 'system') { + if ($family === 'system') { return; } @@ -88,7 +88,7 @@ class Config { // Do we have the cached value? Then return it if (isset(self::$cache[$family][$key])) { - if (self::$cache[$family][$key] == '!!') { + if (self::$cache[$family][$key] === '!!') { return $default_value; } else { return self::$cache[$family][$key]; @@ -141,7 +141,7 @@ class Config { $stored = self::get($family, $key); - if ($stored == $value) { + if ($stored === $value) { return true; } From 7757505e40dc93daced623ff803aecbb33486c93 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 19 Jan 2017 23:07:12 +0000 Subject: [PATCH 26/41] Bugfix: Quarter Hour cache was cleared every time. --- include/cache.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/cache.php b/include/cache.php index e8af8f9de1..98799bcf40 100644 --- a/include/cache.php +++ b/include/cache.php @@ -186,7 +186,7 @@ class Cache { set_config("system", "cache_cleared_half_hour", time()); } - if (($max_level <= CACHE_QUARTER_HOUR) AND (get_config("system", "cache_cleared_hour")) < time() - self::duration(CACHE_QUARTER_HOUR)) { + if (($max_level <= CACHE_QUARTER_HOUR) AND (get_config("system", "cache_cleared_quarter_hour")) < time() - self::duration(CACHE_QUARTER_HOUR)) { q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d", dbesc(datetime_convert('UTC','UTC',"now - 15 minutes")), intval(CACHE_QUARTER_HOUR)); From 850419dd147887055b7c3d135382a55c1d232cb7 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Jan 2017 13:30:06 +0000 Subject: [PATCH 27/41] Hopefully better index length. --- include/dbstructure.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index 0fe157f329..346920ab07 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -840,10 +840,10 @@ function db_definition($charset) { ), "indexes" => array( "PRIMARY" => array("id"), - "nurl" => array("nurl(32)"), - "name" => array("name(32)"), + "nurl" => array("nurl(64)"), + "name" => array("name(64)"), "nick" => array("nick(32)"), - "addr" => array("addr(32)"), + "addr" => array("addr(64)"), "hide_network_updated" => array("hide", "network", "updated"), "updated" => array("updated"), ) From 402c74f8ea4fd4a37854960335d3dedcde3f4658 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Jan 2017 15:00:14 +0000 Subject: [PATCH 28/41] Changed database.sql --- database.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/database.sql b/database.sql index 383253fcb1..64d8d9f7d0 100644 --- a/database.sql +++ b/database.sql @@ -352,10 +352,10 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( `generation` tinyint(3) NOT NULL DEFAULT 0, `server_url` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY(`id`), - INDEX `nurl` (`nurl`(32)), - INDEX `name` (`name`(32)), + INDEX `nurl` (`nurl`(64)), + INDEX `name` (`name`(64)), INDEX `nick` (`nick`(32)), - INDEX `addr` (`addr`(32)), + INDEX `addr` (`addr`(64)), INDEX `hide_network_updated` (`hide`,`network`,`updated`), INDEX `updated` (`updated`) ) DEFAULT CHARSET=utf8mb4; From 9d77e91f5f2030e03d9d68684a2c26a0600a7af8 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Jan 2017 21:58:53 +0000 Subject: [PATCH 29/41] Some more logging --- include/plaintext.php | 3 ++- include/poller.php | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/include/plaintext.php b/include/plaintext.php index 632abf30cb..6ab4ec77d6 100644 --- a/include/plaintext.php +++ b/include/plaintext.php @@ -272,12 +272,13 @@ function shortenmsg($msg, $limit, $twitter = false) { $lines = explode("\n", $msg); $msg = ""; $recycle = html_entity_decode("♲ ", ENT_QUOTES, 'UTF-8'); + $ellipsis = html_entity_decode("…", ENT_QUOTES, 'UTF-8'); foreach ($lines AS $row=>$line) { if (iconv_strlen(trim($msg."\n".$line), "UTF-8") <= $limit) $msg = trim($msg."\n".$line); // Is the new message empty by now or is it a reshared message? elseif (($msg == "") OR (($row == 1) AND (substr($msg, 0, 4) == $recycle))) - $msg = iconv_substr(iconv_substr(trim($msg."\n".$line), 0, $limit, "UTF-8"), 0, -3, "UTF-8")."..."; + $msg = iconv_substr(iconv_substr(trim($msg."\n".$line), 0, $limit, "UTF-8"), 0, -3, "UTF-8").$ellipsis; else break; } diff --git a/include/poller.php b/include/poller.php index b0c594db15..3a83770ceb 100644 --- a/include/poller.php +++ b/include/poller.php @@ -156,6 +156,18 @@ function poller_execute($queue) { $stamp = (float)microtime(true); + if (get_config("system", "profiler")) { + $a->performance["start"] = microtime(true); + $a->performance["database"] = 0; + $a->performance["database_write"] = 0; + $a->performance["network"] = 0; + $a->performance["file"] = 0; + $a->performance["rendering"] = 0; + $a->performance["parser"] = 0; + $a->performance["marktime"] = 0; + $a->performance["markstart"] = microtime(true); + } + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]); // For better logging create a new process id for every worker call @@ -176,6 +188,19 @@ function poller_execute($queue) { logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds."); + if (get_config("system", "profiler")) { + $duration = microtime(true)-$a->performance["start"]; + + logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s", + round($a->performance["database"] - $a->performance["database_write"], 2), + round($a->performance["database_write"], 2), + round($a->performance["network"], 2), + round($a->performance["file"], 2), + round($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), + round($duration, 2)), + LOGGER_DEBUG); + } + q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"])); } else { logger("Function ".$funcname." does not exist"); From 8939a2550273d16005e757e0dde134bd9538dc7b Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Jan 2017 22:05:50 +0000 Subject: [PATCH 30/41] Better number format --- include/poller.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/poller.php b/include/poller.php index 3a83770ceb..dc7f70a51b 100644 --- a/include/poller.php +++ b/include/poller.php @@ -184,7 +184,7 @@ function poller_execute($queue) { sleep($cooldown); } - $duration = (float)round(microtime(true)-$stamp, 3); + $duration = number_format(microtime(true) - $stamp, 3); logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds."); @@ -192,12 +192,12 @@ function poller_execute($queue) { $duration = microtime(true)-$a->performance["start"]; logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s", - round($a->performance["database"] - $a->performance["database_write"], 2), - round($a->performance["database_write"], 2), - round($a->performance["network"], 2), - round($a->performance["file"], 2), - round($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), - round($duration, 2)), + number_format($a->performance["database"] - $a->performance["database_write"], 2), + number_format($a->performance["database_write"], 2), + number_format($a->performance["network"], 2), + number_format($a->performance["file"], 2), + number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), + number_format($duration, 2)), LOGGER_DEBUG); } From 68115581d092356496100809311bf6c8a0868417 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Jan 2017 22:22:05 +0000 Subject: [PATCH 31/41] Avoiding some error messages --- include/session.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/session.php b/include/session.php index 31024060f3..055bfcb4ef 100644 --- a/include/session.php +++ b/include/session.php @@ -64,8 +64,9 @@ function ref_session_write($id, $data) { $default_expire = time() + 300; $memcache = cache::memcache(); - if (is_object($memcache)) { - $memcache->set(get_app()->get_hostname().":session:".$id, $data, MEMCACHE_COMPRESSED, $expire); + $a = get_app(); + if (is_object($memcache) AND is_object($a)) { + $memcache->set($a->get_hostname().":session:".$id, $data, MEMCACHE_COMPRESSED, $expire); return true; } From 72884d75d54fc759481d184542ff0fa761d20327 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Jan 2017 05:05:41 +0000 Subject: [PATCH 32/41] Added callstack --- include/poller.php | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/include/poller.php b/include/poller.php index dc7f70a51b..1f0594691d 100644 --- a/include/poller.php +++ b/include/poller.php @@ -166,6 +166,7 @@ function poller_execute($queue) { $a->performance["parser"] = 0; $a->performance["marktime"] = 0; $a->performance["markstart"] = microtime(true); + $a->callstack = array(); } logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]); @@ -199,6 +200,29 @@ function poller_execute($queue) { number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), number_format($duration, 2)), LOGGER_DEBUG); + + if (get_config("rendertime", "callstack")) { + $o = "ID ".$queue["id"].": ".$funcname.": Database Read:\n"; + foreach ($a->callstack["database"] AS $func => $time) { + $time = round($time, 3); + if ($time > 0) + $o .= $func.": ".$time."\n"; + } + $o .= "\nDatabase Write:\n"; + foreach ($a->callstack["database_write"] AS $func => $time) { + $time = round($time, 3); + if ($time > 0) + $o .= $func.": ".$time."\n"; + } + + $o .= "\nNetwork:\n"; + foreach ($a->callstack["network"] AS $func => $time) { + $time = round($time, 3); + if ($time > 0) + $o .= $func.": ".$time."\n"; + } + logger($o, LOGGER_DEBUG); + } } q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"])); From 45d6f6c0a3ab8a0048dfb4fd3b28db065ad3a485 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Jan 2017 06:06:29 +0000 Subject: [PATCH 33/41] Some changed logging --- include/poller.php | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/include/poller.php b/include/poller.php index 1f0594691d..0c5a4ff628 100644 --- a/include/poller.php +++ b/include/poller.php @@ -30,8 +30,9 @@ function poller_run($argv, $argc){ }; // Quit when in maintenance - if (get_config('system', 'maintenance', true)) + if (Config::get('system', 'maintenance', true)) { return; + } $a->start_process(); @@ -93,7 +94,7 @@ function poller_execute($queue) { $cooldown = Config::get("system", "worker_cooldown", 0); // Quit when in maintenance - if (get_config('system', 'maintenance', true)) { + if (Config::get('system', 'maintenance', true)) { return false; } @@ -154,9 +155,11 @@ function poller_execute($queue) { if (function_exists($funcname)) { + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]); + $stamp = (float)microtime(true); - if (get_config("system", "profiler")) { + if (Config::get("system", "profiler")) { $a->performance["start"] = microtime(true); $a->performance["database"] = 0; $a->performance["database_write"] = 0; @@ -169,8 +172,6 @@ function poller_execute($queue) { $a->callstack = array(); } - logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]); - // For better logging create a new process id for every worker call // But preserve the old one for the worker $old_process_id = $a->process_id; @@ -180,16 +181,11 @@ function poller_execute($queue) { $a->process_id = $old_process_id; - if ($cooldown > 0) { - logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds"); - sleep($cooldown); - } - $duration = number_format(microtime(true) - $stamp, 3); logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds."); - if (get_config("system", "profiler")) { + if (Config::get("system", "profiler")) { $duration = microtime(true)-$a->performance["start"]; logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s", @@ -201,7 +197,7 @@ function poller_execute($queue) { number_format($duration, 2)), LOGGER_DEBUG); - if (get_config("rendertime", "callstack")) { + if (Config::get("rendertime", "callstack")) { $o = "ID ".$queue["id"].": ".$funcname.": Database Read:\n"; foreach ($a->callstack["database"] AS $func => $time) { $time = round($time, 3); @@ -225,6 +221,11 @@ function poller_execute($queue) { } } + if ($cooldown > 0) { + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds"); + sleep($cooldown); + } + q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"])); } else { logger("Function ".$funcname." does not exist"); @@ -241,7 +242,7 @@ function poller_execute($queue) { function poller_max_connections_reached() { // Fetch the max value from the config. This is needed when the system cannot detect the correct value by itself. - $max = get_config("system", "max_connections"); + $max = Config::get("system", "max_connections"); // Fetch the percentage level where the poller will get active $maxlevel = Config::get("system", "max_connections_level", 75); @@ -425,7 +426,7 @@ function poller_too_much_workers() { logger("Load: ".$load."/".$maxsysload." - processes: ".$active."/".$entries." (".$processlist.") - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG); // Are there fewer workers running as possible? Then fork a new one. - if (!get_config("system", "worker_dont_fork") AND ($queues > ($active + 1)) AND ($entries > 1)) { + if (!Config::get("system", "worker_dont_fork") AND ($queues > ($active + 1)) AND ($entries > 1)) { logger("Active workers: ".$active."/".$queues." Fork a new worker.", LOGGER_DEBUG); $args = array("php", "include/poller.php", "no_cron"); $a = get_app(); @@ -554,7 +555,7 @@ function call_worker_if_idle() { if (function_exists("proc_open")) { // When was the last time that we called the worker? // Less than one minute? Then we quit - if ((time() - get_config("system", "worker_started")) < 60) { + if ((time() - Config::get("system", "worker_started")) < 60) { return; } From c74b7565a9fe681dd77cb2575659e3c4217847b6 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Jan 2017 06:16:15 +0000 Subject: [PATCH 34/41] Rearranged the logging --- include/poller.php | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/include/poller.php b/include/poller.php index 0c5a4ff628..391c2e9d23 100644 --- a/include/poller.php +++ b/include/poller.php @@ -188,17 +188,8 @@ function poller_execute($queue) { if (Config::get("system", "profiler")) { $duration = microtime(true)-$a->performance["start"]; - logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s", - number_format($a->performance["database"] - $a->performance["database_write"], 2), - number_format($a->performance["database_write"], 2), - number_format($a->performance["network"], 2), - number_format($a->performance["file"], 2), - number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), - number_format($duration, 2)), - LOGGER_DEBUG); - if (Config::get("rendertime", "callstack")) { - $o = "ID ".$queue["id"].": ".$funcname.": Database Read:\n"; + $o = "\nDatabase Read:\n"; foreach ($a->callstack["database"] AS $func => $time) { $time = round($time, 3); if ($time > 0) @@ -217,8 +208,18 @@ function poller_execute($queue) { if ($time > 0) $o .= $func.": ".$time."\n"; } - logger($o, LOGGER_DEBUG); + } else { + $o = ''; } + + logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s".$o, + number_format($a->performance["database"] - $a->performance["database_write"], 2), + number_format($a->performance["database_write"], 2), + number_format($a->performance["network"], 2), + number_format($a->performance["file"], 2), + number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), + number_format($duration, 2)), + LOGGER_DEBUG); } if ($cooldown > 0) { From dc439c6e50b02337af08f6855121ed44b7205e63 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Jan 2017 19:50:56 +0000 Subject: [PATCH 35/41] Bugfix for masses of php warnings --- include/poller.php | 170 +++++++++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 75 deletions(-) diff --git a/include/poller.php b/include/poller.php index 391c2e9d23..c0b92bb811 100644 --- a/include/poller.php +++ b/include/poller.php @@ -91,8 +91,6 @@ function poller_execute($queue) { $mypid = getmypid(); - $cooldown = Config::get("system", "worker_cooldown", 0); - // Quit when in maintenance if (Config::get('system', 'maintenance', true)) { return false; @@ -138,8 +136,6 @@ function poller_execute($queue) { $argv = json_decode($queue["parameter"]); - $argc = count($argv); - // Check for existance and validity of the include file $include = $argv[0]; @@ -155,77 +151,7 @@ function poller_execute($queue) { if (function_exists($funcname)) { - logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]); - - $stamp = (float)microtime(true); - - if (Config::get("system", "profiler")) { - $a->performance["start"] = microtime(true); - $a->performance["database"] = 0; - $a->performance["database_write"] = 0; - $a->performance["network"] = 0; - $a->performance["file"] = 0; - $a->performance["rendering"] = 0; - $a->performance["parser"] = 0; - $a->performance["marktime"] = 0; - $a->performance["markstart"] = microtime(true); - $a->callstack = array(); - } - - // For better logging create a new process id for every worker call - // But preserve the old one for the worker - $old_process_id = $a->process_id; - $a->process_id = uniqid("wrk", true); - - $funcname($argv, $argc); - - $a->process_id = $old_process_id; - - $duration = number_format(microtime(true) - $stamp, 3); - - logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds."); - - if (Config::get("system", "profiler")) { - $duration = microtime(true)-$a->performance["start"]; - - if (Config::get("rendertime", "callstack")) { - $o = "\nDatabase Read:\n"; - foreach ($a->callstack["database"] AS $func => $time) { - $time = round($time, 3); - if ($time > 0) - $o .= $func.": ".$time."\n"; - } - $o .= "\nDatabase Write:\n"; - foreach ($a->callstack["database_write"] AS $func => $time) { - $time = round($time, 3); - if ($time > 0) - $o .= $func.": ".$time."\n"; - } - - $o .= "\nNetwork:\n"; - foreach ($a->callstack["network"] AS $func => $time) { - $time = round($time, 3); - if ($time > 0) - $o .= $func.": ".$time."\n"; - } - } else { - $o = ''; - } - - logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s".$o, - number_format($a->performance["database"] - $a->performance["database_write"], 2), - number_format($a->performance["database_write"], 2), - number_format($a->performance["network"], 2), - number_format($a->performance["file"], 2), - number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), - number_format($duration, 2)), - LOGGER_DEBUG); - } - - if ($cooldown > 0) { - logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds"); - sleep($cooldown); - } + poller_exec_function($queue, $funcname, $argv); q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"])); } else { @@ -235,6 +161,100 @@ function poller_execute($queue) { return true; } +/** + * @brief Execute a function from the queue + * + * @param array $queue Workerqueue entry + * @param string $funcname name of the function + */ +function poller_exec_function($queue, $funcname, $argv) { + + $a = get_app(); + + $mypid = getmypid(); + + $argc = count($argv); + + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]); + + $stamp = (float)microtime(true); + + if (Config::get("system", "profiler")) { + $a->performance["start"] = microtime(true); + $a->performance["database"] = 0; + $a->performance["database_write"] = 0; + $a->performance["network"] = 0; + $a->performance["file"] = 0; + $a->performance["rendering"] = 0; + $a->performance["parser"] = 0; + $a->performance["marktime"] = 0; + $a->performance["markstart"] = microtime(true); + $a->callstack = array(); + } + + // For better logging create a new process id for every worker call + // But preserve the old one for the worker + $old_process_id = $a->process_id; + $a->process_id = uniqid("wrk", true); + + $funcname($argv, $argc); + + $a->process_id = $old_process_id; + + $duration = number_format(microtime(true) - $stamp, 3); + + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds."); + + if (Config::get("system", "profiler")) { + $duration = microtime(true)-$a->performance["start"]; + + if (Config::get("rendertime", "callstack")) { + if (isset($a->callstack["database"])) { + $o = "\nDatabase Read:\n"; + foreach ($a->callstack["database"] AS $func => $time) { + $time = round($time, 3); + if ($time > 0) + $o .= $func.": ".$time."\n"; + } + } + if (isset($a->callstack["database_write"])) { + $o .= "\nDatabase Write:\n"; + foreach ($a->callstack["database_write"] AS $func => $time) { + $time = round($time, 3); + if ($time > 0) + $o .= $func.": ".$time."\n"; + } + } + if (isset($a->callstack["network"])) { + $o .= "\nNetwork:\n"; + foreach ($a->callstack["network"] AS $func => $time) { + $time = round($time, 3); + if ($time > 0) + $o .= $func.": ".$time."\n"; + } + } + } else { + $o = ''; + } + + logger("ID ".$queue["id"].": ".$funcname.": ".sprintf("DB: %s/%s, Net: %s, I/O: %s, Other: %s, Total: %s".$o, + number_format($a->performance["database"] - $a->performance["database_write"], 2), + number_format($a->performance["database_write"], 2), + number_format($a->performance["network"], 2), + number_format($a->performance["file"], 2), + number_format($duration - ($a->performance["database"] + $a->performance["network"] + $a->performance["file"]), 2), + number_format($duration, 2)), + LOGGER_DEBUG); + } + + $cooldown = Config::get("system", "worker_cooldown", 0); + + if ($cooldown > 0) { + logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds"); + sleep($cooldown); + } +} + /** * @brief Checks if the number of database connections has reached a critical limit. * From 69f1deb16674f9d40ed8245cc5774aa5f234b80f Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Jan 2017 20:15:49 +0000 Subject: [PATCH 36/41] Some added logging --- include/dba.php | 5 +++++ include/poller.php | 3 +++ 2 files changed, 8 insertions(+) diff --git a/include/dba.php b/include/dba.php index 7d2d3bbd0f..8e2d18db60 100644 --- a/include/dba.php +++ b/include/dba.php @@ -138,6 +138,11 @@ class dba { return $return; } + /** + * @brief Analyze a database query and log this if some conditions are met. + * + * @param string $query The database query that will be analyzed + */ public function log_index($query) { $a = get_app(); diff --git a/include/poller.php b/include/poller.php index c0b92bb811..10bc70aae3 100644 --- a/include/poller.php +++ b/include/poller.php @@ -179,6 +179,8 @@ function poller_exec_function($queue, $funcname, $argv) { $stamp = (float)microtime(true); + // We use the callstack here to analyze the performance of executed worker entries. + // For this reason the variables have to be initialized. if (Config::get("system", "profiler")) { $a->performance["start"] = microtime(true); $a->performance["database"] = 0; @@ -205,6 +207,7 @@ function poller_exec_function($queue, $funcname, $argv) { logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds."); + // Write down the performance values into the log if (Config::get("system", "profiler")) { $duration = microtime(true)-$a->performance["start"]; From be1db7bdb0563671979d6afb36d322bfc25822b9 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 22 Jan 2017 05:20:14 +0000 Subject: [PATCH 37/41] Use cache instead of config for storing last proc_run time --- boot.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/boot.php b/boot.php index 501ae38c42..ef787c6353 100644 --- a/boot.php +++ b/boot.php @@ -1391,11 +1391,15 @@ class App { // If the last worker fork was less than 10 seconds before then don't fork another one. // This should prevent the forking of masses of workers. if (get_config("system", "worker")) { - if ((time() - get_config("system", "proc_run_started")) < 10) - return; - + $cachekey = "app:proc_run:started"; + $result = Cache::get($cachekey); + if (!is_null($result)) { + if ((time() - $result) < 10) { + return; + } + } // Set the timestamp of the last proc_run - set_config("system", "proc_run_started", time()); + Cache::set($cachekey, time(), CACHE_MINUTE); } $args[0] = ((x($this->config,'php_path')) && (strlen($this->config['php_path'])) ? $this->config['php_path'] : 'php'); From ea18d1829ff3439cc1cea0aeb88c22b59a0d5c79 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 24 Jan 2017 06:45:46 +0000 Subject: [PATCH 38/41] Reformatted stuff, improved query --- include/contact_widgets.php | 2 +- mod/network.php | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/include/contact_widgets.php b/include/contact_widgets.php index fee569e94d..1e35db2c78 100644 --- a/include/contact_widgets.php +++ b/include/contact_widgets.php @@ -90,7 +90,7 @@ function networks_widget($baseurl,$selected = '') { $extra_sql = unavailable_networks(); - $r = q("SELECT DISTINCT(`network`) FROM `contact` WHERE `uid` = %d AND NOT `self` $extra_sql ORDER BY `network`", + $r = q("SELECT DISTINCT(`network`) FROM `contact` WHERE `uid` = %d AND `network` != '' $extra_sql ORDER BY `network`", intval(local_user()) ); diff --git a/mod/network.php b/mod/network.php index 12223494a7..ad0347ba0c 100644 --- a/mod/network.php +++ b/mod/network.php @@ -773,14 +773,12 @@ function network_content(App $a, $update = 0) { intval(local_user()) ); } - } else { - if ($update_unseen) { + } elseif ($update_unseen) { - $unseen = q("SELECT `id` FROM `item` ".$update_unseen. " LIMIT 1"); + $unseen = q("SELECT `id` FROM `item` ".$update_unseen. " LIMIT 1"); - if (dbm::is_result($unseen)) { - $r = q("UPDATE `item` SET `unseen` = 0 $update_unseen"); - } + if (dbm::is_result($unseen)) { + $r = q("UPDATE `item` SET `unseen` = 0 $update_unseen"); } } @@ -791,10 +789,10 @@ function network_content(App $a, $update = 0) { $o .= conversation($a,$items,$mode,$update); - if(!$update) { - if(get_pconfig(local_user(),'system','infinite_scroll')) { + if (!$update) { + if (get_pconfig(local_user(),'system','infinite_scroll')) { $o .= scroll_loader(); - } elseif(!get_config('system', 'old_pager')) { + } elseif (!get_config('system', 'old_pager')) { $o .= alt_pager($a,count($items)); } else { $o .= paginate($a); From becfeaf0b7e96485cc8f7a958288ba1573b5b8a2 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 24 Jan 2017 18:55:07 +0000 Subject: [PATCH 39/41] Bugfix: Caching of non string cache values now works. --- include/Core/Config.php | 19 +++++++++++++------ mod/admin.php | 13 ++++++++++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/include/Core/Config.php b/include/Core/Config.php index 1d0e66ddc0..2e92b119a4 100644 --- a/include/Core/Config.php +++ b/include/Core/Config.php @@ -139,24 +139,31 @@ class Config { public static function set($family, $key, $value) { $a = get_app(); + // We always store boolean values as integer. + // And when fetching we don't convert them back to boolean. + // So we have to do the conversion here so that the compare below works. + $dbvalue = (is_bool($value) ? (string)intval($value) : $value); + + // Convert the numeric values to string to make the compare work + $dbvalue = (is_numeric($value) ? (string)$value : $dbvalue); + $stored = self::get($family, $key); - if ($stored === $value) { + if ($stored === $dbvalue) { return true; } if ($family === 'config') { - $a->config[$key] = $value; + $a->config[$key] = $dbvalue; } elseif ($family != 'system') { - $a->config[$family][$key] = $value; + $a->config[$family][$key] = $dbvalue; } // Assign the just added value to the cache - self::$cache[$family][$key] = $value; + self::$cache[$family][$key] = $dbvalue; // manage array value - $dbvalue = (is_array($value) ? serialize($value) : $value); - $dbvalue = (is_bool($dbvalue) ? intval($dbvalue) : $dbvalue); + $dbvalue = (is_array($value) ? serialize($value) : $dbvalue); if (is_null($stored)) { $ret = q("INSERT INTO `config` (`cat`, `k`, `v`) VALUES ('%s', '%s', '%s') ON DUPLICATE KEY UPDATE `v` = '%s'", diff --git a/mod/admin.php b/mod/admin.php index b1bc96fd4f..13a00ff6aa 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -6,6 +6,7 @@ * @brief Friendica admin */ +use \Friendica\Core\Config; require_once("include/enotify.php"); require_once("include/text.php"); @@ -948,6 +949,16 @@ function admin_page_site(App $a) { $diaspora_able = ($a->get_path() == ""); + $optimize_max_tablesize = Config::get('system','optimize_max_tablesize', 100); + + if ($optimize_max_tablesize < -1) { + $optimize_max_tablesize = -1; + } + + if ($optimize_max_tablesize == 0) { + $optimize_max_tablesize = 100; + } + $t = get_markup_template("admin_site.tpl"); return replace_macros($t, array( '$title' => t('Administration'), @@ -1019,7 +1030,7 @@ function admin_page_site(App $a) { '$poll_interval' => array('poll_interval', t("Poll interval"), (x(get_config('system','poll_interval'))?get_config('system','poll_interval'):2), t("Delay background polling processes by this many seconds to reduce system load. If 0, use delivery interval.")), '$maxloadavg' => array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")), '$maxloadavg_frontend' => array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")), - '$optimize_max_tablesize'=> array('optimize_max_tablesize', t("Maximum table size for optimization"), ((intval(get_config('system','optimize_max_tablesize')) > 0)?get_config('system','optimize_max_tablesize'):100), t("Maximum table size (in MB) for the automatic optimization - default 100 MB. Enter -1 to disable it.")), + '$optimize_max_tablesize'=> array('optimize_max_tablesize', t("Maximum table size for optimization"), $optimize_max_tablesize, t("Maximum table size (in MB) for the automatic optimization - default 100 MB. Enter -1 to disable it.")), '$optimize_fragmentation'=> array('optimize_fragmentation', t("Minimum level of fragmentation"), ((intval(get_config('system','optimize_fragmentation')) > 0)?get_config('system','optimize_fragmentation'):30), t("Minimum fragmenation level to start the automatic optimization - default value is 30%.")), '$poco_completion' => array('poco_completion', t("Periodical check of global contacts"), get_config('system','poco_completion'), t("If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.")), From 422be7e212cc52a6776d3b4d01f49e5ff116368a Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 25 Jan 2017 05:33:23 +0000 Subject: [PATCH 40/41] Improved handling of non string values in the config --- include/Core/Config.php | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/include/Core/Config.php b/include/Core/Config.php index 2e92b119a4..c495bbd4c7 100644 --- a/include/Core/Config.php +++ b/include/Core/Config.php @@ -102,7 +102,7 @@ class Config { ); if (dbm::is_result($ret)) { // manage array value - $val = (preg_match("|^a:[0-9]+:{.*}$|s", $ret[0]['v'])?unserialize( $ret[0]['v']):$ret[0]['v']); + $val = (preg_match("|^a:[0-9]+:{.*}$|s", $ret[0]['v']) ? unserialize($ret[0]['v']) : $ret[0]['v']); // Assign the value from the database to the cache self::$cache[$family][$key] = $val; @@ -139,13 +139,10 @@ class Config { public static function set($family, $key, $value) { $a = get_app(); - // We always store boolean values as integer. - // And when fetching we don't convert them back to boolean. + // We store our setting values in a string variable. // So we have to do the conversion here so that the compare below works. - $dbvalue = (is_bool($value) ? (string)intval($value) : $value); - - // Convert the numeric values to string to make the compare work - $dbvalue = (is_numeric($value) ? (string)$value : $dbvalue); + // The exception are array values. + $dbvalue = (!is_array($value) ? (string)$value : $value); $stored = self::get($family, $key); From 64d06b98b2c2910ecc133c01510ccf1cab131dab Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 27 Jan 2017 15:13:37 +0000 Subject: [PATCH 41/41] Added documentation --- include/poller.php | 1 + 1 file changed, 1 insertion(+) diff --git a/include/poller.php b/include/poller.php index 10bc70aae3..e8bfb88389 100644 --- a/include/poller.php +++ b/include/poller.php @@ -166,6 +166,7 @@ function poller_execute($queue) { * * @param array $queue Workerqueue entry * @param string $funcname name of the function + * @param array $argv Array of values to be passed to the function */ function poller_exec_function($queue, $funcname, $argv) {