diff --git a/src/Core/System.php b/src/Core/System.php index 55b4bb8cf9..0c1e499d93 100644 --- a/src/Core/System.php +++ b/src/Core/System.php @@ -442,17 +442,24 @@ class System */ public static function getLoadAvg(): array { - $content = file_get_contents('/proc/loadavg'); + $content = @file_get_contents('/proc/loadavg'); if (empty($content)) { $content = shell_exec('cat /proc/loadavg'); } - if (empty($content)) { - return []; + if (empty($content) || !preg_match("#([.\d]+)\s([.\d]+)\s([.\d]+)\s(\d+)/(\d+)#", $content, $matches)) { + $load_arr = sys_getloadavg(); + if (empty($load_arr)) { + return []; + } + return [ + 'average1' => $load_arr[0], + 'average5' => $load_arr[1], + 'average15' => $load_arr[2], + 'runnable' => 0, + 'scheduled' => 0 + ]; } - if (!preg_match("#([.\d]+)\s([.\d]+)\s([.\d]+)\s(\d+)/(\d+)#", $content, $matches)) { - return []; - } return [ 'average1' => (float)$matches[1], 'average5' => (float)$matches[2], diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 4496caf920..6bf6168b2f 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -143,7 +143,7 @@ class Worker } // Quit the worker once every cron interval - if (time() > ($starttime + (DI::config()->get('system', 'cron_interval') * 60))) { + if (time() > ($starttime + (DI::config()->get('system', 'cron_interval') * 60)) && !self::systemLimitReached()) { Logger::info('Process lifetime reached, respawning.'); self::unclaimProcess($process); if (Worker\Daemon::isMode()) { @@ -444,6 +444,36 @@ class Worker return true; } + /** + * Checks if system limits are reached. + * + * @return boolean + */ + private static function systemLimitReached(): bool + { + $load_cooldown = DI::config()->get('system', 'worker_load_cooldown'); + $processes_cooldown = DI::config()->get('system', 'worker_processes_cooldown'); + + if (($load_cooldown == 0) && ($processes_cooldown == 0)) { + return false; + } + + $load = System::getLoadAvg(); + if (empty($load)) { + return false; + } + + if (($load_cooldown > 0) && ($load['average1'] > $load_cooldown)) { + return true; + } + + if (($processes_cooldown > 0) && ($load['scheduled'] > $processes_cooldown)) { + return true; + } + + return false; + } + /** * Slow the execution down if the system load is too high * @@ -772,7 +802,7 @@ class Worker Logger::notice('Load: ' . $load . '/' . $maxsysload . ' - processes: ' . $deferred . '/' . $active . '/' . $waiting_processes . $processlist . ' - maximum: ' . $queues . '/' . $maxqueues); // Are there fewer workers running as possible? Then fork a new one. - if (!DI::config()->get('system', 'worker_dont_fork', false) && ($queues > ($active + 1)) && self::entriesExists()) { + if (!DI::config()->get('system', 'worker_dont_fork', false) && ($queues > ($active + 1)) && self::entriesExists() && !self::systemLimitReached()) { Logger::info('There are fewer workers as possible, fork a new worker.', ['active' => $active, 'queues' => $queues]); if (Worker\Daemon::isMode()) { Worker\IPC::SetJobState(true); @@ -1250,7 +1280,7 @@ class Worker Worker\Daemon::checkState(); // Should we quit and wait for the worker to be called as a cronjob? - if ($dont_fork) { + if ($dont_fork || self::systemLimitReached()) { return $added; }