From 405753d1c335cfda94bdf1c8799f7e7b3e94a177 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 14 Nov 2017 21:50:16 +0000 Subject: [PATCH] cron.php is reworked, other worker files had been moved temporarily --- include/cli_startup.php | 29 --- include/cron.php | 258 -------------------- src/Core/Worker.php | 4 +- src/Worker/Cron.php | 263 +++++++++++++++++++++ {include => worker}/checkversion.php | 0 {include => worker}/create_shadowentry.php | 0 {include => worker}/cronjobs.php | 0 {include => worker}/dbclean.php | 0 {include => worker}/dbupdate.php | 0 {include => worker}/delivery.php | 0 {include => worker}/directory.php | 0 {include => worker}/discover_poco.php | 0 {include => worker}/expire.php | 0 {include => worker}/gprobe.php | 0 {include => worker}/notifier.php | 0 {include => worker}/profile_update.php | 0 {include => worker}/pubsubpublish.php | 0 {include => worker}/queue.php | 0 {include => worker}/remove_contact.php | 0 {include => worker}/shadowupdate.php | 0 {include => worker}/spool_post.php | 0 {include => worker}/tagupdate.php | 0 {include => worker}/threadupdate.php | 0 {include => worker}/update_gcontact.php | 0 24 files changed, 265 insertions(+), 289 deletions(-) delete mode 100644 include/cli_startup.php delete mode 100644 include/cron.php create mode 100644 src/Worker/Cron.php rename {include => worker}/checkversion.php (100%) rename {include => worker}/create_shadowentry.php (100%) rename {include => worker}/cronjobs.php (100%) rename {include => worker}/dbclean.php (100%) rename {include => worker}/dbupdate.php (100%) rename {include => worker}/delivery.php (100%) rename {include => worker}/directory.php (100%) rename {include => worker}/discover_poco.php (100%) rename {include => worker}/expire.php (100%) rename {include => worker}/gprobe.php (100%) rename {include => worker}/notifier.php (100%) rename {include => worker}/profile_update.php (100%) rename {include => worker}/pubsubpublish.php (100%) rename {include => worker}/queue.php (100%) rename {include => worker}/remove_contact.php (100%) rename {include => worker}/shadowupdate.php (100%) rename {include => worker}/spool_post.php (100%) rename {include => worker}/tagupdate.php (100%) rename {include => worker}/threadupdate.php (100%) rename {include => worker}/update_gcontact.php (100%) diff --git a/include/cli_startup.php b/include/cli_startup.php deleted file mode 100644 index 8cbc7db378..0000000000 --- a/include/cli_startup.php +++ /dev/null @@ -1,29 +0,0 @@ -set_baseurl(Config::get('system','url')); - - load_hooks(); -} diff --git a/include/cron.php b/include/cron.php deleted file mode 100644 index 01989826da..0000000000 --- a/include/cron.php +++ /dev/null @@ -1,258 +0,0 @@ - 1) { - cron_poll_contacts($argc, $argv); - return; - } - - $last = Config::get('system', 'last_cron'); - - $poll_interval = intval(Config::get('system', 'cron_interval')); - if (! $poll_interval) { - $poll_interval = 10; - } - - if ($last) { - $next = $last + ($poll_interval * 60); - if ($next > time()) { - logger('cron intervall not reached'); - return; - } - } - - logger('cron: start'); - - // run queue delivery process in the background - Worker::add(PRIORITY_NEGLIGIBLE, "queue"); - - // run the process to discover global contacts in the background - Worker::add(PRIORITY_LOW, "discover_poco"); - - // run the process to update locally stored global contacts in the background - Worker::add(PRIORITY_LOW, "discover_poco", "checkcontact"); - - // Expire and remove user entries - Worker::add(PRIORITY_MEDIUM, "cronjobs", "expire_and_remove_users"); - - // Call possible post update functions - Worker::add(PRIORITY_LOW, "cronjobs", "post_update"); - - // update nodeinfo data - Worker::add(PRIORITY_LOW, "cronjobs", "nodeinfo"); - - // Clear cache entries - Worker::add(PRIORITY_LOW, "cronjobs", "clear_cache"); - - // Repair missing Diaspora values in contacts - Worker::add(PRIORITY_LOW, "cronjobs", "repair_diaspora"); - - // Repair entries in the database - Worker::add(PRIORITY_LOW, "cronjobs", "repair_database"); - - // once daily run birthday_updates and then expire in background - $d1 = Config::get('system', 'last_expire_day'); - $d2 = intval(datetime_convert('UTC', 'UTC', 'now', 'd')); - - if ($d2 != intval($d1)) { - - Worker::add(PRIORITY_LOW, "cronjobs", "update_contact_birthdays"); - - Worker::add(PRIORITY_LOW, "discover_poco", "update_server"); - - Worker::add(PRIORITY_LOW, "discover_poco", "suggestions"); - - Config::set('system', 'last_expire_day', $d2); - - Worker::add(PRIORITY_LOW, 'expire'); - - Worker::add(PRIORITY_MEDIUM, 'dbclean'); - - Worker::add(PRIORITY_LOW, "cronjobs", "update_photo_albums"); - - // Delete all done workerqueue entries - dba::delete('workerqueue', array('`done` AND `executed` < UTC_TIMESTAMP() - INTERVAL 12 HOUR')); - - // check upstream version? - Worker::add(PRIORITY_LOW, 'checkversion'); - } - - // Poll contacts - cron_poll_contacts($argc, $argv); - - logger('cron: end'); - - Config::set('system', 'last_cron', time()); - - return; -} - -/** - * @brief Poll contacts for unreceived messages - * - * @param Integer $argc Number of command line arguments - * @param Array $argv Array of command line arguments - */ -function cron_poll_contacts($argc, $argv) { - $manual_id = 0; - $generation = 0; - $force = false; - $restart = false; - - if (($argc > 1) && ($argv[1] == 'force')) { - $force = true; - } - if (($argc > 1) && ($argv[1] == 'restart')) { - $restart = true; - $generation = intval($argv[2]); - if (!$generation) { - killme(); - } - } - - if (($argc > 1) && intval($argv[1])) { - $manual_id = intval($argv[1]); - $force = true; - } - - $min_poll_interval = Config::get('system', 'min_poll_interval', 1); - - $sql_extra = (($manual_id) ? " AND `id` = $manual_id " : ""); - - reload_plugins(); - - $d = datetime_convert(); - - // Only poll from those with suitable relationships, - // and which have a polling address and ignore Diaspora since - // we are unable to match those posts with a Diaspora GUID and prevent duplicates. - - $abandon_days = intval(Config::get('system', 'account_abandon_days')); - if ($abandon_days < 1) { - $abandon_days = 0; - } - $abandon_sql = (($abandon_days) - ? sprintf(" AND `user`.`login_date` > UTC_TIMESTAMP() - INTERVAL %d DAY ", intval($abandon_days)) - : '' - ); - - $contacts = q("SELECT `contact`.`id` FROM `user` - STRAIGHT_JOIN `contact` - ON `contact`.`uid` = `user`.`uid` AND `contact`.`rel` IN (%d, %d) AND `contact`.`poll` != '' - AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s', '%s') $sql_extra - AND NOT `contact`.`self` AND NOT `contact`.`blocked` AND NOT `contact`.`readonly` - AND NOT `contact`.`archive` - WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql ORDER BY RAND()", - intval(CONTACT_IS_SHARING), - intval(CONTACT_IS_FRIEND), - dbesc(NETWORK_DFRN), - dbesc(NETWORK_ZOT), - dbesc(NETWORK_OSTATUS), - dbesc(NETWORK_FEED), - dbesc(NETWORK_MAIL), - dbesc(NETWORK_MAIL2) - ); - - if (!DBM::is_result($contacts)) { - return; - } - - foreach ($contacts as $c) { - - $res = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1", - intval($c['id']) - ); - - if (!DBM::is_result($res)) { - continue; - } - - foreach ($res as $contact) { - - $xml = false; - - if ($manual_id) { - $contact['last-update'] = NULL_DATE; - } - - if (in_array($contact['network'], array(NETWORK_DFRN, NETWORK_ZOT, NETWORK_OSTATUS))) { - $contact['priority'] = 2; - } - - if ($contact['subhub'] && in_array($contact['network'], array(NETWORK_DFRN, NETWORK_ZOT, NETWORK_OSTATUS))) { - /* - * We should be getting everything via a hub. But just to be sure, let's check once a day. - * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately) - * This also lets us update our subscription to the hub, and add or replace hubs in case it - * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'. - */ - $poll_interval = Config::get('system', 'pushpoll_frequency'); - $contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3); - } - - if (($contact['priority'] >= 0) && !$force) { - $update = false; - - $t = $contact['last-update']; - - /* - * Based on $contact['priority'], should we poll this site now? Or later? - */ - switch ($contact['priority']) { - case 5: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 month")) { - $update = true; - } - break; - case 4: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 week")) { - $update = true; - } - break; - case 3: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 day")) { - $update = true; - } - break; - case 2: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 12 hour")) { - $update = true; - } - break; - case 1: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 hour")) { - $update = true; - } - break; - case 0: - default: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + ".$min_poll_interval." minute")) { - $update = true; - } - break; - } - if (!$update) { - continue; - } - } - - logger("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact["nick"] . " " . $contact["name"]); - - if (($contact['network'] == NETWORK_FEED) && ($contact['priority'] <= 3)) { - $priority = PRIORITY_MEDIUM; - } else { - $priority = PRIORITY_LOW; - } - Worker::add(array('priority' => $priority, 'dont_fork' => true), 'OnePoll', (int)$contact['id']); - } - } -} diff --git a/src/Core/Worker.php b/src/Core/Worker.php index ffc6b29db2..db93adddaa 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -241,7 +241,7 @@ class Worker { // The script could be provided as full path or only with the function name if ($include == basename($include)) { - $include = "include/".$include.".php"; + $include = "worker/".$include.".php"; } if (!validate_include($include)) { @@ -902,7 +902,7 @@ class Worker { self::add(PRIORITY_HIGH, "spool_post"); // Run the cron job that calls all other jobs - self::add(PRIORITY_MEDIUM, "cron"); + self::add(PRIORITY_MEDIUM, "Cron"); // Run the cronhooks job separately from cron for being able to use a different timing self::add(PRIORITY_MEDIUM, "CronHooks"); diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php new file mode 100644 index 0000000000..c859824755 --- /dev/null +++ b/src/Worker/Cron.php @@ -0,0 +1,263 @@ + time()) { + logger('cron intervall not reached'); + return; + } + } + + logger('cron: start'); + + // run queue delivery process in the background + Worker::add(PRIORITY_NEGLIGIBLE, "queue"); + + // run the process to discover global contacts in the background + Worker::add(PRIORITY_LOW, "discover_poco"); + + // run the process to update locally stored global contacts in the background + Worker::add(PRIORITY_LOW, "discover_poco", "checkcontact"); + + // Expire and remove user entries + Worker::add(PRIORITY_MEDIUM, "cronjobs", "expire_and_remove_users"); + + // Call possible post update functions + Worker::add(PRIORITY_LOW, "cronjobs", "post_update"); + + // update nodeinfo data + Worker::add(PRIORITY_LOW, "cronjobs", "nodeinfo"); + + // Clear cache entries + Worker::add(PRIORITY_LOW, "cronjobs", "clear_cache"); + + // Repair missing Diaspora values in contacts + Worker::add(PRIORITY_LOW, "cronjobs", "repair_diaspora"); + + // Repair entries in the database + Worker::add(PRIORITY_LOW, "cronjobs", "repair_database"); + + // once daily run birthday_updates and then expire in background + $d1 = Config::get('system', 'last_expire_day'); + $d2 = intval(datetime_convert('UTC', 'UTC', 'now', 'd')); + + if ($d2 != intval($d1)) { + + Worker::add(PRIORITY_LOW, "cronjobs", "update_contact_birthdays"); + + Worker::add(PRIORITY_LOW, "discover_poco", "update_server"); + + Worker::add(PRIORITY_LOW, "discover_poco", "suggestions"); + + Config::set('system', 'last_expire_day', $d2); + + Worker::add(PRIORITY_LOW, 'expire'); + + Worker::add(PRIORITY_MEDIUM, 'dbclean'); + + Worker::add(PRIORITY_LOW, "cronjobs", "update_photo_albums"); + + // Delete all done workerqueue entries + dba::delete('workerqueue', array('`done` AND `executed` < UTC_TIMESTAMP() - INTERVAL 12 HOUR')); + + // check upstream version? + Worker::add(PRIORITY_LOW, 'checkversion'); + } + + // Poll contacts + self::pollContacts($parameter, $generation); + + logger('cron: end'); + + Config::set('system', 'last_cron', time()); + + return; + } + + /** + * @brief Poll contacts for unreceived messages + * + * @todo Currently it seems as if the following parameter aren't used at all ... + * + * @param string $parameter Parameter (force, restart, ...) for the contact polling + * @param integer $generation + */ + private static function pollContacts($parameter, $generation) { + $manual_id = 0; + $generation = 0; + $force = false; + $restart = false; + + if ($parameter == 'force') { + $force = true; + } + if ($parameter == 'restart') { + $restart = true; + $generation = intval($generation); + if (!$generation) { + killme(); + } + } + + if (intval($parameter)) { + $manual_id = intval($parameter); + $force = true; + } + + $min_poll_interval = Config::get('system', 'min_poll_interval', 1); + + $sql_extra = (($manual_id) ? " AND `id` = $manual_id " : ""); + + reload_plugins(); + + $d = datetime_convert(); + + // Only poll from those with suitable relationships, + // and which have a polling address and ignore Diaspora since + // we are unable to match those posts with a Diaspora GUID and prevent duplicates. + + $abandon_days = intval(Config::get('system', 'account_abandon_days')); + if ($abandon_days < 1) { + $abandon_days = 0; + } + $abandon_sql = (($abandon_days) + ? sprintf(" AND `user`.`login_date` > UTC_TIMESTAMP() - INTERVAL %d DAY ", intval($abandon_days)) + : '' + ); + + $contacts = q("SELECT `contact`.`id` FROM `user` + STRAIGHT_JOIN `contact` + ON `contact`.`uid` = `user`.`uid` AND `contact`.`rel` IN (%d, %d) AND `contact`.`poll` != '' + AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s', '%s') $sql_extra + AND NOT `contact`.`self` AND NOT `contact`.`blocked` AND NOT `contact`.`readonly` + AND NOT `contact`.`archive` + WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql ORDER BY RAND()", + intval(CONTACT_IS_SHARING), + intval(CONTACT_IS_FRIEND), + dbesc(NETWORK_DFRN), + dbesc(NETWORK_ZOT), + dbesc(NETWORK_OSTATUS), + dbesc(NETWORK_FEED), + dbesc(NETWORK_MAIL), + dbesc(NETWORK_MAIL2) + ); + + if (!DBM::is_result($contacts)) { + return; + } + + foreach ($contacts as $c) { + + $res = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1", + intval($c['id']) + ); + + if (!DBM::is_result($res)) { + continue; + } + + foreach ($res as $contact) { + + $xml = false; + + if ($manual_id) { + $contact['last-update'] = NULL_DATE; + } + + if (in_array($contact['network'], array(NETWORK_DFRN, NETWORK_ZOT, NETWORK_OSTATUS))) { + $contact['priority'] = 2; + } + + if ($contact['subhub'] && in_array($contact['network'], array(NETWORK_DFRN, NETWORK_ZOT, NETWORK_OSTATUS))) { + /* + * We should be getting everything via a hub. But just to be sure, let's check once a day. + * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately) + * This also lets us update our subscription to the hub, and add or replace hubs in case it + * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'. + */ + $poll_interval = Config::get('system', 'pushpoll_frequency'); + $contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3); + } + + if (($contact['priority'] >= 0) && !$force) { + $update = false; + + $t = $contact['last-update']; + + /* + * Based on $contact['priority'], should we poll this site now? Or later? + */ + switch ($contact['priority']) { + case 5: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 month")) { + $update = true; + } + break; + case 4: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 week")) { + $update = true; + } + break; + case 3: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 day")) { + $update = true; + } + break; + case 2: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 12 hour")) { + $update = true; + } + break; + case 1: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 hour")) { + $update = true; + } + break; + case 0: + default: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + ".$min_poll_interval." minute")) { + $update = true; + } + break; + } + if (!$update) { + continue; + } + } + + logger("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact["nick"] . " " . $contact["name"]); + + if (($contact['network'] == NETWORK_FEED) && ($contact['priority'] <= 3)) { + $priority = PRIORITY_MEDIUM; + } else { + $priority = PRIORITY_LOW; + } + Worker::add(array('priority' => $priority, 'dont_fork' => true), 'OnePoll', (int)$contact['id']); + } + } + } +} diff --git a/include/checkversion.php b/worker/checkversion.php similarity index 100% rename from include/checkversion.php rename to worker/checkversion.php diff --git a/include/create_shadowentry.php b/worker/create_shadowentry.php similarity index 100% rename from include/create_shadowentry.php rename to worker/create_shadowentry.php diff --git a/include/cronjobs.php b/worker/cronjobs.php similarity index 100% rename from include/cronjobs.php rename to worker/cronjobs.php diff --git a/include/dbclean.php b/worker/dbclean.php similarity index 100% rename from include/dbclean.php rename to worker/dbclean.php diff --git a/include/dbupdate.php b/worker/dbupdate.php similarity index 100% rename from include/dbupdate.php rename to worker/dbupdate.php diff --git a/include/delivery.php b/worker/delivery.php similarity index 100% rename from include/delivery.php rename to worker/delivery.php diff --git a/include/directory.php b/worker/directory.php similarity index 100% rename from include/directory.php rename to worker/directory.php diff --git a/include/discover_poco.php b/worker/discover_poco.php similarity index 100% rename from include/discover_poco.php rename to worker/discover_poco.php diff --git a/include/expire.php b/worker/expire.php similarity index 100% rename from include/expire.php rename to worker/expire.php diff --git a/include/gprobe.php b/worker/gprobe.php similarity index 100% rename from include/gprobe.php rename to worker/gprobe.php diff --git a/include/notifier.php b/worker/notifier.php similarity index 100% rename from include/notifier.php rename to worker/notifier.php diff --git a/include/profile_update.php b/worker/profile_update.php similarity index 100% rename from include/profile_update.php rename to worker/profile_update.php diff --git a/include/pubsubpublish.php b/worker/pubsubpublish.php similarity index 100% rename from include/pubsubpublish.php rename to worker/pubsubpublish.php diff --git a/include/queue.php b/worker/queue.php similarity index 100% rename from include/queue.php rename to worker/queue.php diff --git a/include/remove_contact.php b/worker/remove_contact.php similarity index 100% rename from include/remove_contact.php rename to worker/remove_contact.php diff --git a/include/shadowupdate.php b/worker/shadowupdate.php similarity index 100% rename from include/shadowupdate.php rename to worker/shadowupdate.php diff --git a/include/spool_post.php b/worker/spool_post.php similarity index 100% rename from include/spool_post.php rename to worker/spool_post.php diff --git a/include/tagupdate.php b/worker/tagupdate.php similarity index 100% rename from include/tagupdate.php rename to worker/tagupdate.php diff --git a/include/threadupdate.php b/worker/threadupdate.php similarity index 100% rename from include/threadupdate.php rename to worker/threadupdate.php diff --git a/include/update_gcontact.php b/worker/update_gcontact.php similarity index 100% rename from include/update_gcontact.php rename to worker/update_gcontact.php