From e1341b6ad3b51ee849217cba8c1326e4aaec39b8 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Thu, 22 Sep 2022 05:45:42 +0000
Subject: [PATCH] Fallback for load detection, additional checks

---
 src/Core/System.php | 19 +++++++++++++------
 src/Core/Worker.php | 36 +++++++++++++++++++++++++++++++++---
 2 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/src/Core/System.php b/src/Core/System.php
index 55b4bb8cf9..0c1e499d93 100644
--- a/src/Core/System.php
+++ b/src/Core/System.php
@@ -442,17 +442,24 @@ class System
 	 */
 	public static function getLoadAvg(): array
 	{
-		$content = file_get_contents('/proc/loadavg');
+		$content = @file_get_contents('/proc/loadavg');
 		if (empty($content)) {
 			$content = shell_exec('cat /proc/loadavg');
 		}
-		if (empty($content)) {
-			return [];
+		if (empty($content) || !preg_match("#([.\d]+)\s([.\d]+)\s([.\d]+)\s(\d+)/(\d+)#", $content, $matches)) {
+			$load_arr = sys_getloadavg();
+			if (empty($load_arr)) {
+				return [];
+			}
+			return [
+				'average1'  => $load_arr[0],
+				'average5'  => $load_arr[1],
+				'average15' => $load_arr[2],
+				'runnable'  => 0,
+				'scheduled' => 0
+			];
 		}
 
-		if (!preg_match("#([.\d]+)\s([.\d]+)\s([.\d]+)\s(\d+)/(\d+)#", $content, $matches)) {
-			return [];
-		}
 		return [
 			'average1'  => (float)$matches[1],
 			'average5'  => (float)$matches[2],
diff --git a/src/Core/Worker.php b/src/Core/Worker.php
index 4496caf920..6bf6168b2f 100644
--- a/src/Core/Worker.php
+++ b/src/Core/Worker.php
@@ -143,7 +143,7 @@ class Worker
 			}
 
 			// Quit the worker once every cron interval
-			if (time() > ($starttime + (DI::config()->get('system', 'cron_interval') * 60))) {
+			if (time() > ($starttime + (DI::config()->get('system', 'cron_interval') * 60)) && !self::systemLimitReached()) {
 				Logger::info('Process lifetime reached, respawning.');
 				self::unclaimProcess($process);
 				if (Worker\Daemon::isMode()) {
@@ -444,6 +444,36 @@ class Worker
 		return true;
 	}
 
+	/**
+	 * Checks if system limits are reached.
+	 *
+	 * @return boolean
+	 */
+	private static function systemLimitReached(): bool
+	{
+		$load_cooldown      = DI::config()->get('system', 'worker_load_cooldown');
+		$processes_cooldown = DI::config()->get('system', 'worker_processes_cooldown');
+
+		if (($load_cooldown == 0) && ($processes_cooldown == 0)) {
+			return false;
+		}
+
+		$load = System::getLoadAvg();
+		if (empty($load)) {
+			return false;
+		}
+
+		if (($load_cooldown > 0) && ($load['average1'] > $load_cooldown)) {
+			return true;
+		}
+
+		if (($processes_cooldown > 0) && ($load['scheduled'] > $processes_cooldown)) {
+			return true;
+		}
+
+		return false;
+	}
+
 	/**
 	 * Slow the execution down if the system load is too high
 	 *
@@ -772,7 +802,7 @@ class Worker
 			Logger::notice('Load: ' . $load . '/' . $maxsysload . ' - processes: ' . $deferred . '/' . $active . '/' . $waiting_processes . $processlist . ' - maximum: ' . $queues . '/' . $maxqueues);
 
 			// Are there fewer workers running as possible? Then fork a new one.
-			if (!DI::config()->get('system', 'worker_dont_fork', false) && ($queues > ($active + 1)) && self::entriesExists()) {
+			if (!DI::config()->get('system', 'worker_dont_fork', false) && ($queues > ($active + 1)) && self::entriesExists() && !self::systemLimitReached()) {
 				Logger::info('There are fewer workers as possible, fork a new worker.', ['active' => $active, 'queues' => $queues]);
 				if (Worker\Daemon::isMode()) {
 					Worker\IPC::SetJobState(true);
@@ -1250,7 +1280,7 @@ class Worker
 		Worker\Daemon::checkState();
 
 		// Should we quit and wait for the worker to be called as a cronjob?
-		if ($dont_fork) {
+		if ($dont_fork || self::systemLimitReached()) {
 			return $added;
 		}