Merge pull request #2737 from annando/1608-worker-again

Performance improvements for the poller

boot.php

@@ -1266,8 +1266,20 @@ class App {
 	function proc_run($args) {
 
 		// Add the php path if it is a php call
-		if (count($args) && ($args[0] === 'php' OR is_int($args[0])))
+		if (count($args) && ($args[0] === 'php' OR is_int($args[0]))) {
+
+			// If the last worker fork was less than 10 seconds before then don't fork another one.
+			// This should prevent the forking of masses of workers.
+			if (get_config("system", "worker")) {
+				if ((time() - get_config("system", "proc_run_started")) < 10)
+					return;
+
+				// Set the timestamp of the last proc_run
+				set_config("system", "proc_run_started", time());
+			}
+
 			$args[0] = ((x($this->config,'php_path')) && (strlen($this->config['php_path'])) ? $this->config['php_path'] : 'php');
+		}
 
 		// add baseurl to args. cli scripts can't construct it
 		$args[] = $this->get_baseurl();

include/poller.php

@@ -239,10 +239,13 @@ function poller_kill_stale_workers() {
 			$max_duration_defaults = array(PRIORITY_SYSTEM => 360, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180);
 			$max_duration = $max_duration_defaults[$pid["priority"]];
 
+			$argv = json_decode($pid["parameter"]);
+			$argv[0] = basename($argv[0]);
+
 			// How long is the process already running?
 			$duration = (time() - strtotime($pid["executed"])) / 60;
 			if ($duration > $max_duration) {
-				logger("Worker process ".$pid["pid"]." (".$pid["parameter"].") took more than ".$max_duration." minutes. It will be killed now.");
+				logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") took more than ".$max_duration." minutes. It will be killed now.");
 				posix_kill($pid["pid"], SIGTERM);
 
 				// We killed the stale process.
@@ -254,7 +257,7 @@ function poller_kill_stale_workers() {
 					intval(PRIORITY_LOW),
 					intval($pid["pid"]));
 			} else
-				logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
+				logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
 		}
 }
 

include/post_update.php

@@ -159,7 +159,7 @@ function post_update_1198() {
 	logger("Start", LOGGER_DEBUG);
 
 	// Check if the first step is done (Setting "author-id" and "owner-id" in the item table)
-	$r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 1000");
+	$r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 100");
 	if (!$r) {
 		// Are there unfinished entries in the thread table?
 		$r = q("SELECT COUNT(*) AS `total` FROM `thread`

include/queue.php

@@ -45,6 +45,8 @@ function queue_run(&$argv, &$argc){
 	$deadservers = array();
 	$serverlist = array();
 
+	if (!$queue_id) {
+
 		logger('queue: start');
 
 		// Handling the pubsubhubbub requests
@@ -77,23 +79,23 @@ function queue_run(&$argv, &$argc){
 			q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
 		}
 
-	if($queue_id) {
-		$r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1",
-			intval($queue_id)
-		);
-	}
-	else {
-
 		// For the first 12 hours we'll try to deliver every 15 minutes
 		// After that, we'll only attempt delivery once per hour.
 
 		$r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`");
+	} else {
+		logger('queue: start for id '.$queue_id);
+
+		$r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1",
+			intval($queue_id)
+		);
 	}
-	if(! $r){
+
+	if (!$r){
 		return;
 	}
 
-	if(! $queue_id)
+	if (!$queue_id)
 		call_hooks('queue_predeliver', $a, $r);
 
 
@@ -107,16 +109,17 @@ function queue_run(&$argv, &$argc){
 		// queue_predeliver hooks may have changed the queue db details,
 		// so check again if this entry still needs processing
 
-		if($queue_id) {
+		if($queue_id)
 			$qi = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1",
-				intval($queue_id)
+				intval($queue_id));
-			);
+		elseif (get_config("system", "worker")) {
-		}
+			logger('Call queue for id '.$q_item['id']);
-		else {
+			proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']);
+			continue;
+		} else
 			$qi = q("SELECT * FROM `queue` WHERE `id` = %d AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ",
-				intval($q_item['id'])
+				intval($q_item['id']));
-			);
+
-		}
 		if(! count($qi))
 			continue;