clones/friendica
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #3461 from annando/1705-some-more-dba 

DBA: poller / SQL errors / dbclean
This commit is contained in:
Hypolite Petovan 2017-05-13 00:22:01 -04:00 committed by GitHub
parent 2bbcbdc704 e884090efa
commit db0c3620c0
10 changed files with 72 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
boot.php
View File

@ -1077,14 +1077,10 @@ function proc_run($cmd) {
array_shift($argv); array_shift($argv);
$parameters = json_encode($argv); $parameters = json_encode($argv);
$found = q("SELECT `id` FROM `workerqueue` WHERE `parameter` = '%s'", dbesc($parameters)); $found = dba::select('workerqueue', array('id'), array('parameter' => $parameters), array('limit' => 1));
if (!dbm::is_result($found)) { if (!dbm::is_result($found)) {
q("INSERT INTO `workerqueue` (`parameter`, `created`, `priority`) dba::insert('workerqueue', array('parameter' => $parameters, 'created' => datetime_convert(), 'priority' => $priority));
VALUES ('%s', '%s', %d)",
dbesc($parameters),
dbesc(datetime_convert()),
intval($priority));
} }
// Should we quit and wait for the poller to be called as a cronjob? // Should we quit and wait for the poller to be called as a cronjob?

24
include/Contact.php
View File

@ -23,29 +23,7 @@ function user_remove($uid) {
$r[0]['nickname'] $r[0]['nickname']
); );
/// @todo Should be done in a background job since this likely will run into a time out // The user and related data will be deleted in "cron_expire_and_remove_users" (cronjobs.php)
// don't delete yet, will be done later when contacts have deleted my stuff
// q("DELETE FROM `contact` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `gcign` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `group` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `group_member` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `intro` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `event` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `item` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `item_id` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `mail` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `mailacct` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `manage` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `notify` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `photo` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `attach` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `profile` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `profile_check` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `pconfig` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `search` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `spam` WHERE `uid` = %d", intval($uid));
// don't delete yet, will be done later when contacts have deleted my stuff
// q("DELETE FROM `user` WHERE `uid` = %d", intval($uid));
q("UPDATE `user` SET `account_removed` = 1, `account_expires_on` = UTC_TIMESTAMP() WHERE `uid` = %d", intval($uid)); q("UPDATE `user` SET `account_removed` = 1, `account_expires_on` = UTC_TIMESTAMP() WHERE `uid` = %d", intval($uid));
proc_run(PRIORITY_HIGH, "include/notifier.php", "removeme", $uid); proc_run(PRIORITY_HIGH, "include/notifier.php", "removeme", $uid);

5
include/cronjobs.php
View File

@ -110,12 +110,11 @@ function cron_expire_and_remove_users() {
AND `account_expires_on` > '%s' AND `account_expires_on` > '%s'
AND `account_expires_on` < UTC_TIMESTAMP()", dbesc(NULL_DATE)); AND `account_expires_on` < UTC_TIMESTAMP()", dbesc(NULL_DATE));
// delete user and contact records for recently removed accounts // delete user records for recently removed accounts
$r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); $r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
if (dbm::is_result($r)) { if (dbm::is_result($r)) {
foreach ($r as $user) { foreach ($r as $user) {
q("DELETE FROM `contact` WHERE `uid` = %d", intval($user['uid'])); dba::delete('user', array('uid' => $user['uid']));
q("DELETE FROM `user` WHERE `uid` = %d", intval($user['uid']));
} }
} }
} }

43
include/dbclean.php
View File

@ -18,13 +18,11 @@ function dbclean_run(&$argv, &$argc) {
} }
if ($stage == 0) { if ($stage == 0) {
proc_run(PRIORITY_LOW, 'include/dbclean.php', 1); for ($i = 1; $i <= 7; $i++) {
proc_run(PRIORITY_LOW, 'include/dbclean.php', 2); if (!Config::get('system', 'finished-dbclean-'.$i)) {
proc_run(PRIORITY_LOW, 'include/dbclean.php', 3); proc_run(PRIORITY_LOW, 'include/dbclean.php', $i);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 4); }
proc_run(PRIORITY_LOW, 'include/dbclean.php', 5); }
proc_run(PRIORITY_LOW, 'include/dbclean.php', 6);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 7);
} else { } else {
remove_orphans($stage); remove_orphans($stage);
} }
@ -54,9 +52,15 @@ function remove_orphans($stage = 0) {
} }
} else { } else {
logger("No global item orphans found"); logger("No global item orphans found");
} }
dba::close($r); dba::close($r);
logger("Done deleting ".$count." old global item entries from item table without user copy"); logger("Done deleting ".$count." old global item entries from item table without user copy");
// We will eventually set this value when we found a good way to delete these items in another way.
// if ($count < $limit) {
// Config::set('system', 'finished-dbclean-1', true);
// }
} elseif ($stage == 2) { } elseif ($stage == 2) {
logger("Deleting items without parents"); logger("Deleting items without parents");
$r = dba::p("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit)); $r = dba::p("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit));
@ -71,6 +75,10 @@ function remove_orphans($stage = 0) {
} }
dba::close($r); dba::close($r);
logger("Done deleting ".$count." items without parents"); logger("Done deleting ".$count." items without parents");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-2', true);
}
} elseif ($stage == 3) { } elseif ($stage == 3) {
logger("Deleting orphaned data from thread table"); logger("Deleting orphaned data from thread table");
$r = dba::p("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit)); $r = dba::p("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit));
@ -83,9 +91,12 @@ function remove_orphans($stage = 0) {
} else { } else {
logger("No thread orphans found"); logger("No thread orphans found");
} }
dba::close($r); dba::close($r);
logger("Done deleting ".$count." orphaned data from thread table"); logger("Done deleting ".$count." orphaned data from thread table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-3', true);
}
} elseif ($stage == 4) { } elseif ($stage == 4) {
logger("Deleting orphaned data from notify table"); logger("Deleting orphaned data from notify table");
$r = dba::p("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit)); $r = dba::p("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit));
@ -100,6 +111,10 @@ function remove_orphans($stage = 0) {
} }
dba::close($r); dba::close($r);
logger("Done deleting ".$count." orphaned data from notify table"); logger("Done deleting ".$count." orphaned data from notify table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-4', true);
}
} elseif ($stage == 5) { } elseif ($stage == 5) {
logger("Deleting orphaned data from notify-threads table"); logger("Deleting orphaned data from notify-threads table");
$r = dba::p("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit)); $r = dba::p("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit));
@ -114,6 +129,10 @@ function remove_orphans($stage = 0) {
} }
dba::close($r); dba::close($r);
logger("Done deleting ".$count." orphaned data from notify-threads table"); logger("Done deleting ".$count." orphaned data from notify-threads table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-5', true);
}
} elseif ($stage == 6) { } elseif ($stage == 6) {
logger("Deleting orphaned data from sign table"); logger("Deleting orphaned data from sign table");
$r = dba::p("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit)); $r = dba::p("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit));
@ -128,6 +147,10 @@ function remove_orphans($stage = 0) {
} }
dba::close($r); dba::close($r);
logger("Done deleting ".$count." orphaned data from sign table"); logger("Done deleting ".$count." orphaned data from sign table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-6', true);
}
} elseif ($stage == 7) { } elseif ($stage == 7) {
logger("Deleting orphaned data from term table"); logger("Deleting orphaned data from term table");
$r = dba::p("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit)); $r = dba::p("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit));
@ -142,6 +165,10 @@ function remove_orphans($stage = 0) {
} }
dba::close($r); dba::close($r);
logger("Done deleting ".$count." orphaned data from term table"); logger("Done deleting ".$count." orphaned data from term table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-7', true);
}
} }
// Call it again if not all entries were purged // Call it again if not all entries were purged

27
include/items.php
View File

@ -501,6 +501,20 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
} }
} }
if ($notify) {
$guid_prefix = "";
} elseif ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "")) {
$arr['guid'] = uri_to_guid($arr['plink']);
} elseif ((trim($arr['guid']) == "") AND (trim($arr['uri']) != "")) {
$arr['guid'] = uri_to_guid($arr['uri']);
} else {
$parsed = parse_url($arr["author-link"]);
$guid_prefix = hash("crc32", $parsed["host"]);
}
$arr['guid'] = ((x($arr, 'guid')) ? notags(trim($arr['guid'])) : get_guid(32, $guid_prefix));
$arr['uri'] = ((x($arr, 'uri')) ? notags(trim($arr['uri'])) : item_new_uri($a->get_hostname(), $uid, $arr['guid']));
// Store conversation data // Store conversation data
$arr = store_conversation($arr); $arr = store_conversation($arr);
@ -585,20 +599,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
item_add_language_opt($arr); item_add_language_opt($arr);
if ($notify) {
$guid_prefix = "";
} elseif ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "")) {
$arr['guid'] = uri_to_guid($arr['plink']);
} elseif ((trim($arr['guid']) == "") AND (trim($arr['uri']) != "")) {
$arr['guid'] = uri_to_guid($arr['uri']);
} else {
$parsed = parse_url($arr["author-link"]);
$guid_prefix = hash("crc32", $parsed["host"]);
}
$arr['wall'] = ((x($arr, 'wall')) ? intval($arr['wall']) : 0); $arr['wall'] = ((x($arr, 'wall')) ? intval($arr['wall']) : 0);
$arr['guid'] = ((x($arr, 'guid')) ? notags(trim($arr['guid'])) : get_guid(32, $guid_prefix));
$arr['uri'] = ((x($arr, 'uri')) ? notags(trim($arr['uri'])) : item_new_uri($a->get_hostname(), $uid, $arr['guid']));
$arr['extid'] = ((x($arr, 'extid')) ? notags(trim($arr['extid'])) : ''); $arr['extid'] = ((x($arr, 'extid')) ? notags(trim($arr['extid'])) : '');
$arr['author-name'] = ((x($arr, 'author-name')) ? trim($arr['author-name']) : ''); $arr['author-name'] = ((x($arr, 'author-name')) ? trim($arr['author-name']) : '');
$arr['author-link'] = ((x($arr, 'author-link')) ? notags(trim($arr['author-link'])) : ''); $arr['author-link'] = ((x($arr, 'author-link')) ? notags(trim($arr['author-link'])) : '');

25
include/poller.php
View File

@ -121,12 +121,8 @@ function poller_execute($queue) {
return false; return false;
} }
$upd = q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `pid` = 0", if (!dba::update('workerqueue', array('executed' => datetime_convert(), 'pid' => $mypid),
dbesc(datetime_convert()), array('id' => $queue["id"], 'pid' => 0))) {
intval($mypid),
intval($queue["id"]));
if (!$upd) {
logger("Couldn't update queue entry ".$queue["id"]." - skip this execution", LOGGER_DEBUG); logger("Couldn't update queue entry ".$queue["id"]." - skip this execution", LOGGER_DEBUG);
dba::commit(); dba::commit();
return true; return true;
@ -156,7 +152,7 @@ function poller_execute($queue) {
if (!validate_include($include)) { if (!validate_include($include)) {
logger("Include file ".$argv[0]." is not valid!"); logger("Include file ".$argv[0]." is not valid!");
q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"])); dba::delete('workerqueue', array('id' => $queue["id"]));
return true; return true;
} }
@ -168,7 +164,7 @@ function poller_execute($queue) {
poller_exec_function($queue, $funcname, $argv); poller_exec_function($queue, $funcname, $argv);
q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"])); dba::delete('workerqueue', array('id' => $queue["id"]));
} else { } else {
logger("Function ".$funcname." does not exist"); logger("Function ".$funcname." does not exist");
} }
@ -376,8 +372,8 @@ function poller_kill_stale_workers() {
foreach ($r AS $pid) { foreach ($r AS $pid) {
if (!posix_kill($pid["pid"], 0)) { if (!posix_kill($pid["pid"], 0)) {
q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = 0 WHERE `pid` = %d", dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0),
dbesc(NULL_DATE), intval($pid["pid"])); array('pid' => $pid["pid"]));
} else { } else {
// Kill long running processes // Kill long running processes
@ -401,12 +397,9 @@ function poller_kill_stale_workers() {
// We killed the stale process. // We killed the stale process.
// To avoid a blocking situation we reschedule the process at the beginning of the queue. // To avoid a blocking situation we reschedule the process at the beginning of the queue.
// Additionally we are lowering the priority. // Additionally we are lowering the priority.
q("UPDATE `workerqueue` SET `executed` = '%s', `created` = '%s', dba::update('workerqueue',
`priority` = %d, `pid` = 0 WHERE `pid` = %d", array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => PRIORITY_NEGLIGIBLE, 'pid' => 0),
dbesc(NULL_DATE), array('pid' => $pid["pid"]));
dbesc(datetime_convert()),
intval(PRIORITY_NEGLIGIBLE),
intval($pid["pid"]));
} else { } else {
logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG); logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
} }

4
include/threads.php
View File

@ -245,9 +245,7 @@ function delete_thread($itemid, $itemuri = "") {
intval($item["uid"]) intval($item["uid"])
); );
if (!dbm::is_result($r)) { if (!dbm::is_result($r)) {
$r = q("DELETE FROM `item` WHERE `uri` = '%s' AND `uid` = 0", dba::delete('item', array('uri' => $itemuri, 'uid' => 0));
dbesc($itemuri)
);
logger("delete_thread: Deleted shadow for item ".$itemuri." - ".print_r($result, true), LOGGER_DEBUG); logger("delete_thread: Deleted shadow for item ".$itemuri." - ".print_r($result, true), LOGGER_DEBUG);
} }
} }

12
include/uimport.php
View File

@ -63,16 +63,6 @@ function db_import_assoc($table, $arr) {
return q($query); return q($query);
} }
function import_cleanup($newuid) {
q("DELETE FROM `user` WHERE uid = %d", $newuid);
q("DELETE FROM `contact` WHERE uid = %d", $newuid);
q("DELETE FROM `profile` WHERE uid = %d", $newuid);
q("DELETE FROM `photo` WHERE uid = %d", $newuid);
q("DELETE FROM `group` WHERE uid = %d", $newuid);
q("DELETE FROM `group_member` WHERE uid = %d", $newuid);
q("DELETE FROM `pconfig` WHERE uid = %d", $newuid);
}
/** /**
* @brief Import account file exported from mod/uexport * @brief Import account file exported from mod/uexport
* *
@ -174,7 +164,7 @@ function import_account(App $a, $file) {
if ($r === false) { if ($r === false) {
logger("uimport:insert profile " . $profile['profile-name'] . " : ERROR : " . last_error(), LOGGER_NORMAL); logger("uimport:insert profile " . $profile['profile-name'] . " : ERROR : " . last_error(), LOGGER_NORMAL);
info(t("User profile creation error")); info(t("User profile creation error"));
import_cleanup($newuid); dba::delete('user', array('uid' => $newuid));
return; return;
} }
} }

7
include/user.php
View File

@ -226,9 +226,7 @@ function create_user($arr) {
); );
if ((dbm::is_result($r)) && (count($r) > 1) && $newuid) { if ((dbm::is_result($r)) && (count($r) > 1) && $newuid) {
$result['message'] .= t('Nickname is already registered. Please choose another.') . EOL; $result['message'] .= t('Nickname is already registered. Please choose another.') . EOL;
q("DELETE FROM `user` WHERE `uid` = %d", dba::delete('user', array('uid' => $newuid));
intval($newuid)
);
return $result; return $result;
} }
@ -248,8 +246,7 @@ function create_user($arr) {
if ($r === false) { if ($r === false) {
$result['message'] .= t('An error occurred creating your default profile. Please try again.') . EOL; $result['message'] .= t('An error occurred creating your default profile. Please try again.') . EOL;
// Start fresh next time. // Start fresh next time.
$r = q("DELETE FROM `user` WHERE `uid` = %d", dba::delete('user', array('uid' => $newuid));
intval($newuid));
return $result; return $result;
} }

19
mod/regmod.php
View File

@ -74,27 +74,18 @@ function user_deny($hash) {
dbesc($hash) dbesc($hash)
); );
if(! dbm::is_result($register)) if (!dbm::is_result($register)) {
return false; return false;
}
$user = q("SELECT * FROM `user` WHERE `uid` = %d LIMIT 1", $user = q("SELECT * FROM `user` WHERE `uid` = %d LIMIT 1",
intval($register[0]['uid']) intval($register[0]['uid'])
); );
$r = q("DELETE FROM `user` WHERE `uid` = %d", dba::delete('user', array('uid' => $register[0]['uid']));
intval($register[0]['uid']) dba::delete('register', array('hash' => $register[0]['hash']));
);
$r = q("DELETE FROM `contact` WHERE `uid` = %d",
intval($register[0]['uid'])
);
$r = q("DELETE FROM `profile` WHERE `uid` = %d",
intval($register[0]['uid'])
);
$r = q("DELETE FROM `register` WHERE `hash` = '%s'", notice(sprintf(t('Registration revoked for %s'), $user[0]['username']) . EOL);
dbesc($register[0]['hash'])
);
notice( sprintf(t('Registration revoked for %s'), $user[0]['username']) . EOL);
return true; return true;
} }