clones/friendica
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #3460 from annando/1705-dbclean 

dbclean is now using the new database functions
This commit is contained in:
Hypolite Petovan 2017-05-12 11:45:27 -04:00 committed by GitHub
parent abc924bc8f acd9f453d7
commit 06ea23665c
7 changed files with 124 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
include/dba.php
View File

@ -22,6 +22,7 @@ class dba {
public $connected = false; public $connected = false;
public $error = false; public $error = false;
private $_server_info = ''; private $_server_info = '';
private static $in_transaction = false;
private static $dbo; private static $dbo;
private static $relation = array(); private static $relation = array();
@ -602,7 +603,8 @@ class dba {
} }
if (self::$dbo->errorno != 0) { if (self::$dbo->errorno != 0) {
logger('DB Error '.self::$dbo->errorno.': '.self::$dbo->error."\n".self::replace_parameters($sql, $args)); logger('DB Error '.self::$dbo->errorno.': '.self::$dbo->error."\n".
$a->callstack(8))."\n".self::replace_parameters($sql, $args);
} }
$a->save_timestamp($stamp1, 'database'); $a->save_timestamp($stamp1, 'database');
@ -779,6 +781,48 @@ class dba {
return self::e($sql, $param); return self::e($sql, $param);
} }
/**
* @brief Starts a transaction
*
* @return boolean Was the command executed successfully?
*/
static public function transaction() {
if (!self::e('COMMIT')) {
return false;
}
if (!self::e('START TRANSACTION')) {
return false;
}
self::$in_transaction = true;
return true;
}
/**
* @brief Does a commit
*
* @return boolean Was the command executed successfully?
*/
static public function commit() {
if (!self::e('COMMIT')) {
return false;
}
self::$in_transaction = false;
return true;
}
/**
* @brief Does a rollback
*
* @return boolean Was the command executed successfully?
*/
static public function rollback() {
if (!self::e('ROLLBACK')) {
return false;
}
self::$in_transaction = false;
return true;
}
/** /**
* @brief Build the array with the table relations * @brief Build the array with the table relations
* *
@ -805,12 +849,12 @@ class dba {
* *
* @param string $table Table name * @param string $table Table name
* @param array $param parameter array * @param array $param parameter array
* @param boolean $in_commit Internal use: Only do a commit after the last delete * @param boolean $in_process Internal use: Only do a commit after the last delete
* @param array $callstack Internal use: prevent endless loops * @param array $callstack Internal use: prevent endless loops
* *
* @return boolean|array was the delete successfull? When $in_commit is set: deletion data * @return boolean|array was the delete successfull? When $in_process is set: deletion data
*/ */
static public function delete($table, $param, $in_commit = false, &$callstack = array()) { static public function delete($table, $param, $in_process = false, &$callstack = array()) {
$commands = array(); $commands = array();
@ -872,10 +916,11 @@ class dba {
} }
} }
if (!$in_commit) { if (!$in_process) {
// Now we finalize the process // Now we finalize the process
self::p("COMMIT"); if (!self::$in_transaction) {
self::p("START TRANSACTION"); self::transaction();
}
$compacted = array(); $compacted = array();
$counter = array(); $counter = array();
@ -887,7 +932,9 @@ class dba {
logger(dba::replace_parameters($sql, $command['param']), LOGGER_DATA); logger(dba::replace_parameters($sql, $command['param']), LOGGER_DATA);
if (!self::e($sql, $command['param'])) { if (!self::e($sql, $command['param'])) {
self::p("ROLLBACK"); if (!self::$in_transaction) {
self::rollback();
}
return false; return false;
} }
} else { } else {
@ -915,13 +962,17 @@ class dba {
logger(dba::replace_parameters($sql, $field_values), LOGGER_DATA); logger(dba::replace_parameters($sql, $field_values), LOGGER_DATA);
if (!self::e($sql, $field_values)) { if (!self::e($sql, $field_values)) {
self::p("ROLLBACK"); if (!self::$in_transaction) {
self::rollback();
}
return false; return false;
} }
} }
} }
} }
self::p("COMMIT"); if (!self::$in_transaction) {
self::commit();
}
return true; return true;
} }

71
include/dbclean.php
View File

@ -41,7 +41,7 @@ function remove_orphans($stage = 0) {
// We split the deletion in many small tasks // We split the deletion in many small tasks
$limit = 1000; $limit = 1000;
if (($stage == 1) OR ($stage == 0)) { if ($stage == 1) {
logger("Deleting old global item entries from item table without user copy"); logger("Deleting old global item entries from item table without user copy");
$r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0 $r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0
AND NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) AND NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0)
@ -50,97 +50,98 @@ function remove_orphans($stage = 0) {
if ($count > 0) { if ($count > 0) {
logger("found global item orphans: ".$count); logger("found global item orphans: ".$count);
while ($orphan = dba::fetch($r)) { while ($orphan = dba::fetch($r)) {
q("DELETE FROM `item` WHERE `id` = %d", intval($orphan["id"])); dba::delete('item', array('id' => $orphan["id"]));
} }
} else {
logger("No global item orphans found");
} }
dba::close($r); dba::close($r);
logger("Done deleting old global item entries from item table without user copy"); logger("Done deleting ".$count." old global item entries from item table without user copy");
} } elseif ($stage == 2) {
if (($stage == 2) OR ($stage == 0)) {
logger("Deleting items without parents"); logger("Deleting items without parents");
$r = dba::p("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit)); $r = dba::p("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit));
$count = dba::num_rows($r); $count = dba::num_rows($r);
if ($count > 0) { if ($count > 0) {
logger("found item orphans without parents: ".$count); logger("found item orphans without parents: ".$count);
while ($orphan = dba::fetch($r)) { while ($orphan = dba::fetch($r)) {
q("DELETE FROM `item` WHERE `id` = %d", intval($orphan["id"])); dba::delete('item', array('id' => $orphan["id"]));
} }
} else {
logger("No item orphans without parents found");
} }
dba::close($r); dba::close($r);
logger("Done deleting items without parents"); logger("Done deleting ".$count." items without parents");
} } elseif ($stage == 3) {
if (($stage == 3) OR ($stage == 0)) {
logger("Deleting orphaned data from thread table"); logger("Deleting orphaned data from thread table");
$r = dba::p("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit)); $r = dba::p("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit));
$count = dba::num_rows($r); $count = dba::num_rows($r);
if ($count > 0) { if ($count > 0) {
logger("found thread orphans: ".$count); logger("found thread orphans: ".$count);
while ($orphan = dba::fetch($r)) { while ($orphan = dba::fetch($r)) {
q("DELETE FROM `thread` WHERE `iid` = %d", intval($orphan["iid"])); dba::delete('thread', array('iid' => $orphan["iid"]));
} }
} else {
logger("No thread orphans found");
} }
dba::close($r);
logger("Done deleting orphaned data from thread table");
}
if (($stage == 4) OR ($stage == 0)) { dba::close($r);
logger("Done deleting ".$count." orphaned data from thread table");
} elseif ($stage == 4) {
logger("Deleting orphaned data from notify table"); logger("Deleting orphaned data from notify table");
$r = dba::p("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit)); $r = dba::p("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit));
$count = dba::num_rows($r); $count = dba::num_rows($r);
if ($count > 0) { if ($count > 0) {
logger("found notify orphans: ".$count); logger("found notify orphans: ".$count);
while ($orphan = dba::fetch($r)) { while ($orphan = dba::fetch($r)) {
q("DELETE FROM `notify` WHERE `iid` = %d", intval($orphan["iid"])); dba::delete('notify', array('iid' => $orphan["iid"]));
} }
} else {
logger("No notify orphans found");
} }
dba::close($r); dba::close($r);
logger("Done deleting orphaned data from notify table"); logger("Done deleting ".$count." orphaned data from notify table");
} } elseif ($stage == 5) {
if (($stage == 5) OR ($stage == 0)) {
logger("Deleting orphaned data from notify-threads table"); logger("Deleting orphaned data from notify-threads table");
$r = dba::p("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit)); $r = dba::p("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit));
$count = dba::num_rows($r); $count = dba::num_rows($r);
if ($count > 0) { if ($count > 0) {
logger("found notify-threads orphans: ".$count); logger("found notify-threads orphans: ".$count);
while ($orphan = dba::fetch($r)) { while ($orphan = dba::fetch($r)) {
q("DELETE FROM `notify-threads` WHERE `id` = %d", intval($orphan["id"])); dba::delete('notify-threads', array('id' => $orphan["id"]));
} }
} else {
logger("No notify-threads orphans found");
} }
dba::close($r); dba::close($r);
logger("Done deleting orphaned data from notify-threads table"); logger("Done deleting ".$count." orphaned data from notify-threads table");
} } elseif ($stage == 6) {
if (($stage == 6) OR ($stage == 0)) {
logger("Deleting orphaned data from sign table"); logger("Deleting orphaned data from sign table");
$r = dba::p("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit)); $r = dba::p("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit));
$count = dba::num_rows($r); $count = dba::num_rows($r);
if ($count > 0) { if ($count > 0) {
logger("found sign orphans: ".$count); logger("found sign orphans: ".$count);
while ($orphan = dba::fetch($r)) { while ($orphan = dba::fetch($r)) {
q("DELETE FROM `sign` WHERE `iid` = %d", intval($orphan["iid"])); dba::delete('sign', array('iid' => $orphan["iid"]));
} }
} else {
logger("No sign orphans found");
} }
dba::close($r); dba::close($r);
logger("Done deleting orphaned data from sign table"); logger("Done deleting ".$count." orphaned data from sign table");
} } elseif ($stage == 7) {
if (($stage == 7) OR ($stage == 0)) {
logger("Deleting orphaned data from term table"); logger("Deleting orphaned data from term table");
$r = dba::p("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit)); $r = dba::p("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit));
$count = dba::num_rows($r); $count = dba::num_rows($r);
if ($count > 0) { if ($count > 0) {
logger("found term orphans: ".$count); logger("found term orphans: ".$count);
while ($orphan = dba::fetch($r)) { while ($orphan = dba::fetch($r)) {
q("DELETE FROM `term` WHERE `oid` = %d", intval($orphan["oid"])); dba::delete('term', array('oid' => $orphan["oid"]));
} }
} else {
logger("No term orphans found");
} }
dba::close($r); dba::close($r);
logger("Done deleting orphaned data from term table"); logger("Done deleting ".$count." orphaned data from term table");
} }
// Call it again if not all entries were purged // Call it again if not all entries were purged

9
include/expire.php
View File

@ -10,8 +10,13 @@ function expire_run(&$argv, &$argc){
require_once('include/Contact.php'); require_once('include/Contact.php');
// physically remove anything that has been deleted for more than two months // physically remove anything that has been deleted for more than two months
$r = dba::p("SELECT `id` FROM `item` WHERE `deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY");
$r = q("DELETE FROM `item` WHERE `deleted` = 1 AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY"); if (dbm::is_result($r)) {
while ($row = dba::fetch($r)) {
dba::delete('item', array('id' => $row['id']));
}
dba::close($r);
}
// make this optional as it could have a performance impact on large sites // make this optional as it could have a performance impact on large sites

39
include/items.php
View File

@ -951,8 +951,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
logger('item_store: ' . print_r($arr,true), LOGGER_DATA); logger('item_store: ' . print_r($arr,true), LOGGER_DATA);
q("COMMIT"); dba::transaction();
q("START TRANSACTION;");
$r = dbq("INSERT INTO `item` (`" $r = dbq("INSERT INTO `item` (`"
. implode("`, `", array_keys($arr)) . implode("`, `", array_keys($arr))
@ -974,7 +973,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
} }
} else { } else {
// This can happen - for example - if there are locking timeouts. // This can happen - for example - if there are locking timeouts.
q("ROLLBACK"); dba::rollback();
// Store the data into a spool file so that we can try again later. // Store the data into a spool file so that we can try again later.
@ -999,7 +998,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
if ($current_post == 0) { if ($current_post == 0) {
// This is one of these error messages that never should occur. // This is one of these error messages that never should occur.
logger("couldn't find created item - we better quit now."); logger("couldn't find created item - we better quit now.");
q("ROLLBACK"); dba::rollback();
return 0; return 0;
} }
@ -1014,7 +1013,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
if (!dbm::is_result($r)) { if (!dbm::is_result($r)) {
// This shouldn't happen, since COUNT always works when the database connection is there. // This shouldn't happen, since COUNT always works when the database connection is there.
logger("We couldn't count the stored entries. Very strange ..."); logger("We couldn't count the stored entries. Very strange ...");
q("ROLLBACK"); dba::rollback();
return 0; return 0;
} }
@ -1023,13 +1022,13 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
logger('Duplicated post occurred. uri = ' . $arr['uri'] . ' uid = ' . $arr['uid']); logger('Duplicated post occurred. uri = ' . $arr['uri'] . ' uid = ' . $arr['uid']);
// Yes, we could do a rollback here - but we are having many users with MyISAM. // Yes, we could do a rollback here - but we are having many users with MyISAM.
q("DELETE FROM `item` WHERE `id` = %d", intval($current_post)); dba::delete('item', array('id' => $current_post));
q("COMMIT"); dba::commit();
return 0; return 0;
} elseif ($r[0]["entries"] == 0) { } elseif ($r[0]["entries"] == 0) {
// This really should never happen since we quit earlier if there were problems. // This really should never happen since we quit earlier if there were problems.
logger("Something is terribly wrong. We haven't found our created entry."); logger("Something is terribly wrong. We haven't found our created entry.");
q("ROLLBACK"); dba::rollback();
return 0; return 0;
} }
@ -1109,7 +1108,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
update_thread($parent_id); update_thread($parent_id);
} }
q("COMMIT"); dba::commit();
/* /*
* Due to deadlock issues with the "term" table we are doing these steps after the commit. * Due to deadlock issues with the "term" table we are doing these steps after the commit.
@ -1378,10 +1377,7 @@ function tag_deliver($uid, $item_id) {
// mmh.. no mention.. community page or private group... no wall.. no origin.. top-post (not a comment) // mmh.. no mention.. community page or private group... no wall.. no origin.. top-post (not a comment)
// delete it! // delete it!
logger("tag_deliver: no-mention top-level post to communuty or private group. delete."); logger("tag_deliver: no-mention top-level post to communuty or private group. delete.");
q("DELETE FROM item WHERE id = %d and uid = %d", dba::delete('item', array('id' => $item_id));
intval($item_id),
intval($uid)
);
return true; return true;
} }
return; return;
@ -2237,23 +2233,6 @@ function drop_item($id, $interactive = true) {
// ignore the result // ignore the result
} }
// clean up item_id and sign meta-data tables
/*
/// @TODO Old code - caused very long queries and warning entries in the mysql logfiles:
$r = q("DELETE FROM item_id where iid in (select id from item where parent = %d and uid = %d)",
intval($item['id']),
intval($item['uid'])
);
$r = q("DELETE FROM sign where iid in (select id from item where parent = %d and uid = %d)",
intval($item['id']),
intval($item['uid'])
);
*/
// The new code splits the queries since the mysql optimizer really has bad problems with subqueries // The new code splits the queries since the mysql optimizer really has bad problems with subqueries
// Creating list of parents // Creating list of parents

12
include/poller.php
View File

@ -128,7 +128,7 @@ function poller_execute($queue) {
if (!$upd) { if (!$upd) {
logger("Couldn't update queue entry ".$queue["id"]." - skip this execution", LOGGER_DEBUG); logger("Couldn't update queue entry ".$queue["id"]." - skip this execution", LOGGER_DEBUG);
q("COMMIT"); dba::commit();
return true; return true;
} }
@ -136,18 +136,18 @@ function poller_execute($queue) {
$id = q("SELECT `pid`, `executed` FROM `workerqueue` WHERE `id` = %d", intval($queue["id"])); $id = q("SELECT `pid`, `executed` FROM `workerqueue` WHERE `id` = %d", intval($queue["id"]));
if (!$id) { if (!$id) {
logger("Queue item ".$queue["id"]." vanished - skip this execution", LOGGER_DEBUG); logger("Queue item ".$queue["id"]." vanished - skip this execution", LOGGER_DEBUG);
q("COMMIT"); dba::commit();
return true; return true;
} elseif ((strtotime($id[0]["executed"]) <= 0) OR ($id[0]["pid"] == 0)) { } elseif ((strtotime($id[0]["executed"]) <= 0) OR ($id[0]["pid"] == 0)) {
logger("Entry for queue item ".$queue["id"]." wasn't stored - skip this execution", LOGGER_DEBUG); logger("Entry for queue item ".$queue["id"]." wasn't stored - skip this execution", LOGGER_DEBUG);
q("COMMIT"); dba::commit();
return true; return true;
} elseif ($id[0]["pid"] != $mypid) { } elseif ($id[0]["pid"] != $mypid) {
logger("Queue item ".$queue["id"]." is to be executed by process ".$id[0]["pid"]." and not by me (".$mypid.") - skip this execution", LOGGER_DEBUG); logger("Queue item ".$queue["id"]." is to be executed by process ".$id[0]["pid"]." and not by me (".$mypid.") - skip this execution", LOGGER_DEBUG);
q("COMMIT"); dba::commit();
return true; return true;
} }
q("COMMIT"); dba::commit();
$argv = json_decode($queue["parameter"]); $argv = json_decode($queue["parameter"]);
@ -558,7 +558,7 @@ function poller_passing_slow(&$highest_priority) {
*/ */
function poller_worker_process() { function poller_worker_process() {
q("START TRANSACTION;"); dba::transaction();
// Check if we should pass some low priority process // Check if we should pass some low priority process
$highest_priority = 0; $highest_priority = 0;

7
mod/item.php
View File

@ -809,8 +809,7 @@ function item_post(App $a) {
$post_id = 0; $post_id = 0;
} }
q("COMMIT"); dba::transaction();
q("START TRANSACTION;");
$r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`, $r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`,
`owner-name`,`owner-link`,`owner-avatar`, `owner-id`, `owner-name`,`owner-link`,`owner-avatar`, `owner-id`,
@ -900,7 +899,7 @@ function item_post(App $a) {
} }
if ($post_id == 0) { if ($post_id == 0) {
q("COMMIT"); dba::commit();
logger('mod_item: unable to retrieve post that was just stored.'); logger('mod_item: unable to retrieve post that was just stored.');
notice(t('System error. Post not saved.') . EOL); notice(t('System error. Post not saved.') . EOL);
goaway($return_path); goaway($return_path);
@ -1026,7 +1025,7 @@ function item_post(App $a) {
update_thread($parent, true); update_thread($parent, true);
} }
q("COMMIT"); dba::commit();
create_tags_from_item($post_id); create_tags_from_item($post_id);
create_files_from_item($post_id); create_files_from_item($post_id);

4
src/App.php
View File

@ -729,8 +729,8 @@ class App {
* *
* @return string * @return string
*/ */
function callstack() { function callstack($depth = 4) {
$trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 6); $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, $depth + 2);
// We remove the first two items from the list since they contain data that we don't need. // We remove the first two items from the list since they contain data that we don't need.
array_shift($trace); array_shift($trace);