From 4d966e1e213c7b6481e009454805a369416c5b0b Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 31 Oct 2016 21:32:08 +0000 Subject: [PATCH 1/2] Changed priority for onepoll/further enhancement for dbclean --- include/cron.php | 9 +++- include/dbclean.php | 119 ++++++++++++++++++++++++++++++++++++-------- include/follow.php | 2 +- mod/contacts.php | 2 +- 4 files changed, 109 insertions(+), 23 deletions(-) diff --git a/include/cron.php b/include/cron.php index c03745a442..5b588fee6f 100644 --- a/include/cron.php +++ b/include/cron.php @@ -131,6 +131,9 @@ function cron_run(&$argv, &$argc){ proc_run(PRIORITY_LOW, 'include/dbclean.php', 2); proc_run(PRIORITY_LOW, 'include/dbclean.php', 3); proc_run(PRIORITY_LOW, 'include/dbclean.php', 4); + proc_run(PRIORITY_LOW, 'include/dbclean.php', 5); + proc_run(PRIORITY_LOW, 'include/dbclean.php', 6); + proc_run(PRIORITY_LOW, 'include/dbclean.php', 7); } else { proc_run(PRIORITY_LOW, 'include/dbclean.php'); } @@ -330,7 +333,11 @@ function cron_poll_contacts($argc, $argv) { logger("Polling ".$contact["network"]." ".$contact["id"]." ".$contact["nick"]." ".$contact["name"]); - proc_run(PRIORITY_MEDIUM, 'include/onepoll.php', $contact['id']); + if ($contact["remote_self"]) { + proc_run(PRIORITY_MEDIUM, 'include/onepoll.php', $contact['id']); + } else { + proc_run(PRIORITY_LOW, 'include/onepoll.php', $contact['id']); + } if($interval) @time_sleep_until(microtime(true) + (float) $interval); diff --git a/include/dbclean.php b/include/dbclean.php index f69489109c..22a6d4e429 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -36,55 +36,134 @@ function remove_orphans($stage = 0) { global $db; if (($stage == 1) OR ($stage == 0)) { - logger("Deleting orphaned data from thread table"); - if ($db->q("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`)", true)) { - logger("found thread orphans: ".$db->num_rows()); + logger("Deleting old global item entries from item table without user copy"); + if ($db->q("SELECT `id` FROM `item` WHERE `uid` = 0 + AND NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) + AND `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY LIMIT 10000", true)) { + $count = $db->num_rows(); + logger("found global item orphans: ".$count); while ($orphan = $db->qfetch()) { - q("DELETE FROM `thread` WHERE `iid` = %d", intval($orphan["iid"])); + q("DELETE FROM `item` WHERE `id` = %d", intval($orphan["id"])); + } + + // Call it again if not all entries were purged + if (($count > 0) AND get_config("system", "worker")) { + proc_run(PRIORITY_LOW, 'include/dbclean.php', 1); } } $db->qclose(); + logger("Done deleting old global item entries from item table without user copy"); } if (($stage == 2) OR ($stage == 0)) { + logger("Deleting items without parents"); + if ($db->q("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT 10000", true)) { + $count = $db->num_rows(); + logger("found item orphans without parents: ".$count); + while ($orphan = $db->qfetch()) { + q("DELETE FROM `item` WHERE `id` = %d", intval($orphan["id"])); + } + + // Call it again if not all entries were purged + if (($count > 0) AND get_config("system", "worker")) { + proc_run(PRIORITY_LOW, 'include/dbclean.php', 2); + } + } + $db->qclose(); + logger("Done deleting items without parents"); + } + + if (($stage == 3) OR ($stage == 0)) { + logger("Deleting orphaned data from thread table"); + if ($db->q("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`)", true)) { + $count = $db->num_rows(); + logger("found thread orphans: ".$count); + while ($orphan = $db->qfetch()) { + q("DELETE FROM `thread` WHERE `iid` = %d", intval($orphan["iid"])); + } + + // Call it again if not all entries were purged + if (($count > 0) AND get_config("system", "worker")) { + proc_run(PRIORITY_LOW, 'include/dbclean.php', 3); + } + } + $db->qclose(); + logger("Done deleting orphaned data from thread table"); + } + + if (($stage == 4) OR ($stage == 0)) { logger("Deleting orphaned data from notify table"); if ($db->q("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`)", true)) { - logger("found notify orphans: ".$db->num_rows()); + $count = $db->num_rows(); + logger("found notify orphans: ".$count); while ($orphan = $db->qfetch()) { q("DELETE FROM `notify` WHERE `iid` = %d", intval($orphan["iid"])); } + + // Call it again if not all entries were purged + if (($count > 0) AND get_config("system", "worker")) { + proc_run(PRIORITY_LOW, 'include/dbclean.php', 4); + } } $db->qclose(); + logger("Done deleting orphaned data from notify table"); + } + + if (($stage == 5) OR ($stage == 0)) { + logger("Deleting orphaned data from notify-threads table"); + if ($db->q("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`)", true)) { + $count = $db->num_rows(); + logger("found notify-threads orphans: ".$count); + while ($orphan = $db->qfetch()) { + q("DELETE FROM `notify-threads` WHERE `id` = %d", intval($orphan["id"])); + } + + // Call it again if not all entries were purged + if (($count > 0) AND get_config("system", "worker")) { + proc_run(PRIORITY_LOW, 'include/dbclean.php', 5); + } + } + $db->qclose(); + logger("Done deleting orphaned data from notify-threads table"); } - if (($stage == 3) OR ($stage == 0)) { + if (($stage == 6) OR ($stage == 0)) { logger("Deleting orphaned data from sign table"); if ($db->q("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`)", true)) { - logger("found sign orphans: ".$db->num_rows()); + $count = $db->num_rows(); + logger("found sign orphans: ".$count); while ($orphan = $db->qfetch()) { q("DELETE FROM `sign` WHERE `iid` = %d", intval($orphan["iid"])); } - } - $db->qclose(); - } - - if (($stage == 4) OR ($stage == 0)) { - logger("Deleting orphaned data from term table"); - if ($db->q("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`)", true)) { - logger("found term orphans: ".$db->num_rows()); - while ($orphan = $db->qfetch()) { - q("DELETE FROM `term` WHERE `oid` = %d", intval($orphan["oid"])); + // Call it again if not all entries were purged + if (($count > 0) AND get_config("system", "worker")) { + proc_run(PRIORITY_LOW, 'include/dbclean.php', 6); } } $db->qclose(); + logger("Done deleting orphaned data from sign table"); } - /// @todo Based on the following query we should remove some more data - // SELECT `id`, `received`, `created`, `guid` FROM `item` WHERE `uid` = 0 AND NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) LIMIT 1; - logger("Done deleting orphaned data from tables"); + if (($stage == 7) OR ($stage == 0)) { + logger("Deleting orphaned data from term table"); + if ($db->q("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`)", true)) { + $count = $db->num_rows(); + logger("found term orphans: ".$count); + while ($orphan = $db->qfetch()) { + q("DELETE FROM `term` WHERE `oid` = %d", intval($orphan["oid"])); + } + + // Call it again if not all entries were purged + if (($count > 0) AND get_config("system", "worker")) { + proc_run(PRIORITY_LOW, 'include/dbclean.php', 7); + } + } + $db->qclose(); + logger("Done deleting orphaned data from term table"); + } } if (array_search(__file__,get_included_files())===0){ diff --git a/include/follow.php b/include/follow.php index 08e74e02f3..1c33edf80e 100644 --- a/include/follow.php +++ b/include/follow.php @@ -270,7 +270,7 @@ function new_contact($uid,$url,$interactive = false) { // pull feed and consume it, which should subscribe to the hub. - proc_run(PRIORITY_MEDIUM, "include/onepoll.php", $contact_id, "force"); + proc_run(PRIORITY_HIGH, "include/onepoll.php", $contact_id, "force"); // create a follow slap diff --git a/mod/contacts.php b/mod/contacts.php index 410cced8e6..23907669a6 100644 --- a/mod/contacts.php +++ b/mod/contacts.php @@ -237,7 +237,7 @@ function _contact_update($contact_id) { intval($contact_id)); } else // pull feed and consume it, which should subscribe to the hub. - proc_run(PRIORITY_MEDIUM, "include/onepoll.php", $contact_id, "force"); + proc_run(PRIORITY_HIGH, "include/onepoll.php", $contact_id, "force"); } function _contact_update_profile($contact_id) { From b5d004e1d7405317aa1bd0316f14c86dc9a64f78 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 1 Nov 2016 21:36:15 +0000 Subject: [PATCH 2/2] run dbclean as long as there is some data to be purged --- include/cron.php | 12 +--------- include/dbclean.php | 56 ++++++++++++++++----------------------------- 2 files changed, 21 insertions(+), 47 deletions(-) diff --git a/include/cron.php b/include/cron.php index 5b588fee6f..925bb5eab5 100644 --- a/include/cron.php +++ b/include/cron.php @@ -126,17 +126,7 @@ function cron_run(&$argv, &$argc){ proc_run(PRIORITY_LOW, 'include/expire.php'); - if (get_config("system", "worker")) { - proc_run(PRIORITY_LOW, 'include/dbclean.php', 1); - proc_run(PRIORITY_LOW, 'include/dbclean.php', 2); - proc_run(PRIORITY_LOW, 'include/dbclean.php', 3); - proc_run(PRIORITY_LOW, 'include/dbclean.php', 4); - proc_run(PRIORITY_LOW, 'include/dbclean.php', 5); - proc_run(PRIORITY_LOW, 'include/dbclean.php', 6); - proc_run(PRIORITY_LOW, 'include/dbclean.php', 7); - } else { - proc_run(PRIORITY_LOW, 'include/dbclean.php'); - } + proc_run(PRIORITY_LOW, 'include/dbclean.php'); cron_update_photo_albums(); } diff --git a/include/dbclean.php b/include/dbclean.php index 22a6d4e429..0baece8168 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -26,7 +26,18 @@ function dbclean_run(&$argv, &$argc) { } else { $stage = 0; } - remove_orphans($stage); + + if (get_config("system", "worker") AND ($stage == 0)) { + proc_run(PRIORITY_LOW, 'include/dbclean.php', 1); + proc_run(PRIORITY_LOW, 'include/dbclean.php', 2); + proc_run(PRIORITY_LOW, 'include/dbclean.php', 3); + proc_run(PRIORITY_LOW, 'include/dbclean.php', 4); + proc_run(PRIORITY_LOW, 'include/dbclean.php', 5); + proc_run(PRIORITY_LOW, 'include/dbclean.php', 6); + proc_run(PRIORITY_LOW, 'include/dbclean.php', 7); + } else { + remove_orphans($stage); + } } /** @@ -35,6 +46,8 @@ function dbclean_run(&$argv, &$argc) { function remove_orphans($stage = 0) { global $db; + $count = 0; + if (($stage == 1) OR ($stage == 0)) { logger("Deleting old global item entries from item table without user copy"); if ($db->q("SELECT `id` FROM `item` WHERE `uid` = 0 @@ -45,11 +58,6 @@ function remove_orphans($stage = 0) { while ($orphan = $db->qfetch()) { q("DELETE FROM `item` WHERE `id` = %d", intval($orphan["id"])); } - - // Call it again if not all entries were purged - if (($count > 0) AND get_config("system", "worker")) { - proc_run(PRIORITY_LOW, 'include/dbclean.php', 1); - } } $db->qclose(); logger("Done deleting old global item entries from item table without user copy"); @@ -63,11 +71,6 @@ function remove_orphans($stage = 0) { while ($orphan = $db->qfetch()) { q("DELETE FROM `item` WHERE `id` = %d", intval($orphan["id"])); } - - // Call it again if not all entries were purged - if (($count > 0) AND get_config("system", "worker")) { - proc_run(PRIORITY_LOW, 'include/dbclean.php', 2); - } } $db->qclose(); logger("Done deleting items without parents"); @@ -81,11 +84,6 @@ function remove_orphans($stage = 0) { while ($orphan = $db->qfetch()) { q("DELETE FROM `thread` WHERE `iid` = %d", intval($orphan["iid"])); } - - // Call it again if not all entries were purged - if (($count > 0) AND get_config("system", "worker")) { - proc_run(PRIORITY_LOW, 'include/dbclean.php', 3); - } } $db->qclose(); logger("Done deleting orphaned data from thread table"); @@ -99,11 +97,6 @@ function remove_orphans($stage = 0) { while ($orphan = $db->qfetch()) { q("DELETE FROM `notify` WHERE `iid` = %d", intval($orphan["iid"])); } - - // Call it again if not all entries were purged - if (($count > 0) AND get_config("system", "worker")) { - proc_run(PRIORITY_LOW, 'include/dbclean.php', 4); - } } $db->qclose(); logger("Done deleting orphaned data from notify table"); @@ -117,11 +110,6 @@ function remove_orphans($stage = 0) { while ($orphan = $db->qfetch()) { q("DELETE FROM `notify-threads` WHERE `id` = %d", intval($orphan["id"])); } - - // Call it again if not all entries were purged - if (($count > 0) AND get_config("system", "worker")) { - proc_run(PRIORITY_LOW, 'include/dbclean.php', 5); - } } $db->qclose(); logger("Done deleting orphaned data from notify-threads table"); @@ -136,11 +124,6 @@ function remove_orphans($stage = 0) { while ($orphan = $db->qfetch()) { q("DELETE FROM `sign` WHERE `iid` = %d", intval($orphan["iid"])); } - - // Call it again if not all entries were purged - if (($count > 0) AND get_config("system", "worker")) { - proc_run(PRIORITY_LOW, 'include/dbclean.php', 6); - } } $db->qclose(); logger("Done deleting orphaned data from sign table"); @@ -155,15 +138,16 @@ function remove_orphans($stage = 0) { while ($orphan = $db->qfetch()) { q("DELETE FROM `term` WHERE `oid` = %d", intval($orphan["oid"])); } - - // Call it again if not all entries were purged - if (($count > 0) AND get_config("system", "worker")) { - proc_run(PRIORITY_LOW, 'include/dbclean.php', 7); - } } $db->qclose(); logger("Done deleting orphaned data from term table"); } + + // Call it again if not all entries were purged + if (($stage != 0) AND ($count > 0) AND get_config("system", "worker")) { + proc_run(PRIORITY_LOW, 'include/dbclean.php'); + } + } if (array_search(__file__,get_included_files())===0){