From b7e8c1595de3b1b4eeeadf1eabd5a471d3841b92 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 16 Aug 2020 17:59:37 +0000 Subject: [PATCH 01/14] Automatically adjust feed frequencies --- src/Protocol/Feed.php | 140 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 130 insertions(+), 10 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index ba0adb8347..61ef846800 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -300,6 +300,7 @@ class Feed } $items = []; + $creation_dates = []; // Limit the number of items that are about to be fetched $total_items = ($entries->length - 1); @@ -354,16 +355,6 @@ class Feed $item["parent-uri"] = $item["uri"]; - if (!$dryRun) { - $condition = ["`uid` = ? AND `uri` = ? AND `network` IN (?, ?)", - $importer["uid"], $item["uri"], Protocol::FEED, Protocol::DFRN]; - $previous = Item::selectFirst(['id'], $condition); - if (DBA::isResult($previous)) { - Logger::info("Item with uri " . $item["uri"] . " for user " . $importer["uid"] . " already existed under id " . $previous["id"]); - continue; - } - } - $item["title"] = XML::getFirstNodeValue($xpath, 'atom:title/text()', $entry); if (empty($item["title"])) { @@ -403,6 +394,19 @@ class Feed $item["edited"] = $updated; } + if (!$dryRun) { + $condition = ["`uid` = ? AND `uri` = ? AND `network` IN (?, ?)", + $importer["uid"], $item["uri"], Protocol::FEED, Protocol::DFRN]; + $previous = Item::selectFirst(['id', 'created'], $condition); + if (DBA::isResult($previous)) { + // Use the creation date when the post had been stored. It can happen this date changes in the feed. + $creation_dates[] = $previous['created']; + Logger::info("Item with uri " . $item["uri"] . " for user " . $importer["uid"] . " already existed under id " . $previous["id"]); + continue; + } + $creation_dates[] = DateTimeFormat::utc($item['created']); + } + $creator = XML::getFirstNodeValue($xpath, 'author/text()', $entry); if (empty($creator)) { @@ -598,9 +602,125 @@ class Feed } } + if (!$dryRun) { + self::adjustPollFrequency($contact, $creation_dates); + } + return ["header" => $author, "items" => $items]; } + /** + * Automatically adjust the poll frequency according to the post frequency + * + * @param array $contact + * @param array $creation_dates + * @return void + */ + private static function adjustPollFrequency(array $contact, array $creation_dates) + { + if (($contact['priority'] > 3) || ($contact['network'] != Protocol::FEED)) { + Logger::info('Contact is no feed or has a low priority, skip.', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'priority' => $contact['priority'], 'network' => $contact['network']]); + return; + } + + if (!empty($creation_dates)) { + // Count the post frequency and the earliest and latest post date + $frequency = []; + $oldest = time(); + $newest = 0; + $oldest_date = $newest_date = ''; + + foreach ($creation_dates as $date) { + $timestamp = strtotime($date); + $day = intdiv($timestamp, 86400); + $hour = $timestamp % 86400; + + // Only have a look at values from the last seven days + if (((time() / 86400) - $day) < 7) { + if (empty($frequency[$day])) { + $frequency[$day] = ['count' => 1, 'low' => $hour, 'high' => $hour]; + } else { + ++$frequency[$day]['count']; + if ($frequency[$day]['low'] > $hour) { + $frequency[$day]['low'] = $hour; + } + if ($frequency[$day]['high'] < $hour) { + $frequency[$day]['high'] = $hour; + } + } + } + if ($oldest > $day) { + $oldest = $day; + $oldest_date = $date; + } + + if ($newest < $day) { + $newest = $day; + $newest_date = $date; + } + } + + if (($newest == $oldest) && count($creation_dates) > 1) { + Logger::info('Feed has no different creation dates, quitting', ['date' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + return; + } + + if (((time() / 86400) - $newest) > 30) { + Logger::info('Feed had not posted for a month, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 3; // Poll once a day + } + + if (count($creation_dates) == 1) { + Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 3; // Poll once a day + } + + if (empty($priority) && (($newest - $oldest) > count($creation_dates))) { + Logger::info('Less than a post per day, switching to daily polling', ['posts' => count($creation_dates), 'oldest' => $oldest_date, 'newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 3; // Poll once a day + } + + if (empty($priority)) { + // Calculate the highest "posts per day" value + $max = 1; + foreach ($frequency as $entry) { + if (($entry['count'] == 1) || ($entry['high'] == $entry['low'])) { + continue; + } + + // We take the earliest and latest post day and interpolate the number of post per day + // that would had been created with this post frequency + + // Assume at least four hours between oldest and newest post per day - should be okay for news outlets + $duration = max($entry['high'] - $entry['low'], 14400); + $ppd = (86400 / $duration) * $entry['count']; + if ($ppd > $max) { + $max = $ppd; + } + } + + if ($max >= 24) { + $priority = 0; // Poll with the minimum poll intervall + } elseif ($max >= 12) { + $priority = 1; // Poll hourly + } elseif ($max > 1) { + $priority = 2; // Poll twice a day + } else { + /// @todo In the future we could calculate the days between the posts to set even lower priorities + $priority = 3; // Poll once a day + } + Logger::info('Calculated priority by the posts per day', ['priority' => $priority, 'max' => round($max, 2), 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + } + } else { + Logger::info('No posts, switching to daily polling', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 3; // Poll once a day + } + + if ($contact['priority'] != $priority) { + Logger::info('Adjusting priority', ['old' => $contact['priority'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + } + } + /** * Convert a tag array to a tag string * From 112dd37a09b121d00197173a24f9b5e41abcc63c Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 16 Aug 2020 18:05:34 +0000 Subject: [PATCH 02/14] Switch to daily after a week --- src/Protocol/Feed.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 61ef846800..9ba2089c13 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -665,12 +665,12 @@ class Feed return; } - if (((time() / 86400) - $newest) > 30) { - Logger::info('Feed had not posted for a month, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + if (empty($frequency)) { + Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); $priority = 3; // Poll once a day } - if (count($creation_dates) == 1) { + if (empty($priority) && (count($creation_dates) == 1)) { Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); $priority = 3; // Poll once a day } From c4aebffd3b29a82306c76307f7c4a93f95efcd92 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 16 Aug 2020 18:07:36 +0000 Subject: [PATCH 03/14] Setting priority to "notice" --- src/Protocol/Feed.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 9ba2089c13..78830c5603 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -717,7 +717,7 @@ class Feed } if ($contact['priority'] != $priority) { - Logger::info('Adjusting priority', ['old' => $contact['priority'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + Logger::notice('Adjusting priority', ['old' => $contact['priority'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); } } From df75a24dc7a501e9081d285f7f09060af7655583 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 16 Aug 2020 21:38:26 +0000 Subject: [PATCH 04/14] Use config value to trigger automated rating --- src/Protocol/Feed.php | 7 ++++--- src/Worker/Cron.php | 7 ++++++- static/defaults.config.php | 4 ++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 78830c5603..915d78ac39 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -602,7 +602,7 @@ class Feed } } - if (!$dryRun) { + if (!$dryRun && DI::config()->get('system', 'adjust_poll_frequency')) { self::adjustPollFrequency($contact, $creation_dates); } @@ -716,8 +716,9 @@ class Feed $priority = 3; // Poll once a day } - if ($contact['priority'] != $priority) { - Logger::notice('Adjusting priority', ['old' => $contact['priority'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + if ($contact['rating'] != $priority) { + Logger::notice('Adjusting priority', ['old' => $contact['rating'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + DBA::update('contact', ['rating' => $priority], ['id' => $contact['id']]); } } diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index ecfd2e7ddc..286a19cc97 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -186,7 +186,7 @@ class Cron Addon::reload(); $sql = "SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`, `contact`.`archive`, - `contact`.`last-update`, `contact`.`priority`, `contact`.`rel`, `contact`.`subhub` + `contact`.`last-update`, `contact`.`priority`, `contact`.`rating`, `contact`.`rel`, `contact`.`subhub` FROM `user` STRAIGHT_JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`poll` != '' @@ -217,6 +217,11 @@ class Cron } while ($contact = DBA::fetch($contacts)) { + // Use the "rating" field when auto adjusting the poll intervall + if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { + $contact['priority'] = max($contact['rating'], $contact['priority']); + } + // Friendica and OStatus are checked once a day if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { $contact['priority'] = 3; diff --git a/static/defaults.config.php b/static/defaults.config.php index 5e5b48c683..018e0dc635 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -82,6 +82,10 @@ return [ 'php_path' => 'php', ], 'system' => [ + // adjust_poll_frequency (Boolean) + // Automatically detect and set the best feed poll frequency. + 'adjust_poll_frequency' => false, + // allowed_link_protocols (Array) // Allowed protocols in links URLs, add at your own risk. http(s) is always allowed. 'allowed_link_protocols' => ['ftp://', 'ftps://', 'mailto:', 'cid:', 'gopher://'], From 975bb76291536f3c20c136fd4f15e8721158a612 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 05:26:40 +0000 Subject: [PATCH 05/14] Removed check for number of posts --- src/Protocol/Feed.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 915d78ac39..9222d36057 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -675,11 +675,6 @@ class Feed $priority = 3; // Poll once a day } - if (empty($priority) && (($newest - $oldest) > count($creation_dates))) { - Logger::info('Less than a post per day, switching to daily polling', ['posts' => count($creation_dates), 'oldest' => $oldest_date, 'newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); - $priority = 3; // Poll once a day - } - if (empty($priority)) { // Calculate the highest "posts per day" value $max = 1; From 7370a075f8185fb2084cabb43a99c821f4a4a678 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 06:47:29 +0000 Subject: [PATCH 06/14] New rating scale --- src/Protocol/Feed.php | 21 ++++++++++----- src/Worker/Cron.php | 59 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 9222d36057..c67dfcdda9 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -693,16 +693,23 @@ class Feed $max = $ppd; } } - - if ($max >= 24) { - $priority = 0; // Poll with the minimum poll intervall - } elseif ($max >= 12) { - $priority = 1; // Poll hourly + if ($max > 48) { + $priority = 1; // Poll every quarter hour + } elseif ($max > 24) { + $priority = 2; // Poll half an hour + } elseif ($max > 12) { + $priority = 3; // Poll hourly + } elseif ($max > 8) { + $priority = 4; // Poll every two hours + } elseif ($max > 4) { + $priority = 5; // Poll every three hours + } elseif ($max > 2) { + $priority = 6; // Poll every six hours } elseif ($max > 1) { - $priority = 2; // Poll twice a day + $priority = 7; // Poll twice a day } else { /// @todo In the future we could calculate the days between the posts to set even lower priorities - $priority = 3; // Poll once a day + $priority = 8; // Poll once a day } Logger::info('Calculated priority by the posts per day', ['priority' => $priority, 'max' => round($max, 2), 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); } diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 286a19cc97..a67d99407f 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -219,25 +219,37 @@ class Cron while ($contact = DBA::fetch($contacts)) { // Use the "rating" field when auto adjusting the poll intervall if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { - $contact['priority'] = max($contact['rating'], $contact['priority']); + $rating = $contact['rating']; + } elseif ($contact['priority'] == 1) { + $rating = 3; + } elseif ($contact['priority'] == 2) { + $rating = 7; + } elseif ($contact['priority'] == 3) { + $rating = 8; + } elseif ($contact['priority'] == 4) { + $rating = 9; + } elseif ($contact['priority'] == 5) { + $rating = 10; + } else { + $rating = -1; } // Friendica and OStatus are checked once a day if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { - $contact['priority'] = 3; + $rating = 8; } // ActivityPub is checked once a week if ($contact['network'] == Protocol::ACTIVITYPUB) { - $contact['priority'] = 4; + $rating = 9; } // Check archived contacts once a month if ($contact['archive']) { - $contact['priority'] = 5; + $rating = 10; } - if ($contact['priority'] >= 0) { + if ($rating >= 0) { $update = false; $t = $contact['last-update']; @@ -245,32 +257,57 @@ class Cron /* * Based on $contact['priority'], should we poll this site now? Or later? */ - switch ($contact['priority']) { - case 5: + switch ($rating) { + case 10: if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 month")) { $update = true; } break; - case 4: + case 9: if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 week")) { $update = true; } break; - case 3: + case 8: if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 day")) { $update = true; } break; - case 2: + case 7: if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 12 hour")) { $update = true; } break; - case 1: + case 6: + if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 6 hour")) { + $update = true; + } + break; + case 5: + if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 3 hour")) { + $update = true; + } + break; + case 4: + if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 2 hour")) { + $update = true; + } + break; + case 3: if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 hour")) { $update = true; } break; + case 2: + if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 30 minute")) { + $update = true; + } + break; + case 1: + if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 15 minute")) { + $update = true; + } + break; case 0: default: if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + " . $min_poll_interval . " minute")) { From ad2674aac69137cf74fc9d70aba8ec30ecdf2e4c Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 06:50:51 +0000 Subject: [PATCH 07/14] Once a day is 8 --- src/Protocol/Feed.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index c67dfcdda9..0681b863a5 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -667,12 +667,12 @@ class Feed if (empty($frequency)) { Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); - $priority = 3; // Poll once a day + $priority = 8; // Poll once a day } if (empty($priority) && (count($creation_dates) == 1)) { Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); - $priority = 3; // Poll once a day + $priority = 8; // Poll once a day } if (empty($priority)) { @@ -715,7 +715,7 @@ class Feed } } else { Logger::info('No posts, switching to daily polling', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); - $priority = 3; // Poll once a day + $priority = 8; // Poll once a day } if ($contact['rating'] != $priority) { From 224ef91715beacec7dd3bb9a7335fbc5196d8b6a Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 07:44:59 +0000 Subject: [PATCH 08/14] Simplify code --- src/Worker/Cron.php | 92 +++++++-------------------------------------- 1 file changed, 14 insertions(+), 78 deletions(-) diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index a67d99407f..89a2b7373d 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -217,19 +217,11 @@ class Cron } while ($contact = DBA::fetch($contacts)) { - // Use the "rating" field when auto adjusting the poll intervall + $ratings = [0, 3, 7, 8, 9, 10]; if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { $rating = $contact['rating']; - } elseif ($contact['priority'] == 1) { - $rating = 3; - } elseif ($contact['priority'] == 2) { - $rating = 7; - } elseif ($contact['priority'] == 3) { - $rating = 8; - } elseif ($contact['priority'] == 4) { - $rating = 9; - } elseif ($contact['priority'] == 5) { - $rating = 10; + } elseif (array_key_exists($ratings, $contact['priority'])) { + $rating = $ratings[$contact['priority']]; } else { $rating = -1; } @@ -249,75 +241,19 @@ class Cron $rating = 10; } - if ($rating >= 0) { - $update = false; + if ($rating < 0) { + continue; + } + /* + * Based on $contact['priority'], should we poll this site now? Or later? + */ + $t = $contact['last-update']; - $t = $contact['last-update']; + $poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute', + '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month']; - /* - * Based on $contact['priority'], should we poll this site now? Or later? - */ - switch ($rating) { - case 10: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 month")) { - $update = true; - } - break; - case 9: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 week")) { - $update = true; - } - break; - case 8: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 day")) { - $update = true; - } - break; - case 7: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 12 hour")) { - $update = true; - } - break; - case 6: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 6 hour")) { - $update = true; - } - break; - case 5: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 3 hour")) { - $update = true; - } - break; - case 4: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 2 hour")) { - $update = true; - } - break; - case 3: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 hour")) { - $update = true; - } - break; - case 2: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 30 minute")) { - $update = true; - } - break; - case 1: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 15 minute")) { - $update = true; - } - break; - case 0: - default: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + " . $min_poll_interval . " minute")) { - $update = true; - } - break; - } - if (!$update) { - continue; - } + if (empty($poll_intervals[$rating]) || (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . ' + ' . $poll_intervals[$rating]))) { + continue; } if ((($contact['network'] == Protocol::FEED) && ($contact['priority'] <= 3)) || ($contact['network'] == Protocol::MAIL)) { From ec35e2827b59dd1c942591b3f4bedb1c284ddb9a Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 07:51:15 +0000 Subject: [PATCH 09/14] Remove check --- src/Protocol/Feed.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 0681b863a5..917352793b 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -660,11 +660,6 @@ class Feed } } - if (($newest == $oldest) && count($creation_dates) > 1) { - Logger::info('Feed has no different creation dates, quitting', ['date' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); - return; - } - if (empty($frequency)) { Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); $priority = 8; // Poll once a day From af47a9f36c60851f8b2b7262c8cf169a21f006c9 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 08:55:20 +0000 Subject: [PATCH 10/14] Fix parameter order --- src/Worker/Cron.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 89a2b7373d..d7cd7c0546 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -220,7 +220,7 @@ class Cron $ratings = [0, 3, 7, 8, 9, 10]; if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { $rating = $contact['rating']; - } elseif (array_key_exists($ratings, $contact['priority'])) { + } elseif (array_key_exists($contact['priority'], $ratings)) { $rating = $ratings[$contact['priority']]; } else { $rating = -1; From fb9832bbd0b76da844721ac65d482851ef5764e5 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 09:53:49 +0000 Subject: [PATCH 11/14] Poll twice a day when there had been posts in the last 7 days --- src/Protocol/Feed.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 917352793b..8a6725d19a 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -672,7 +672,7 @@ class Feed if (empty($priority)) { // Calculate the highest "posts per day" value - $max = 1; + $max = 0; foreach ($frequency as $entry) { if (($entry['count'] == 1) || ($entry['high'] == $entry['low'])) { continue; @@ -700,7 +700,7 @@ class Feed $priority = 5; // Poll every three hours } elseif ($max > 2) { $priority = 6; // Poll every six hours - } elseif ($max > 1) { + } elseif ($max > 0) { $priority = 7; // Poll twice a day } else { /// @todo In the future we could calculate the days between the posts to set even lower priorities From e0490aff5c6d204f3bd3f7516049bdb46e1a6d29 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 12:25:55 +0000 Subject: [PATCH 12/14] Added weekly and monthly polling --- src/Protocol/Feed.php | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 8a6725d19a..28e0493dae 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -660,13 +660,23 @@ class Feed } } - if (empty($frequency)) { - Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + if (count($creation_dates) == 1) { + Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); $priority = 8; // Poll once a day } - if (empty($priority) && (count($creation_dates) == 1)) { - Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + if (empty($priority) && (((time() / 86400) - $newest) > 730)) { + Logger::info('Feed had not posted for two years, switching to monthly polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 10; // Poll every month + } + + if (empty($priority) && (((time() / 86400) - $newest) > 365)) { + Logger::info('Feed had not posted for a year, switching to weekly polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 9; // Poll every week + } + + if (empty($priority) && empty($frequency)) { + Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); $priority = 8; // Poll once a day } @@ -700,11 +710,8 @@ class Feed $priority = 5; // Poll every three hours } elseif ($max > 2) { $priority = 6; // Poll every six hours - } elseif ($max > 0) { - $priority = 7; // Poll twice a day } else { - /// @todo In the future we could calculate the days between the posts to set even lower priorities - $priority = 8; // Poll once a day + $priority = 7; // Poll twice a day } Logger::info('Calculated priority by the posts per day', ['priority' => $priority, 'max' => round($max, 2), 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); } From 312ac73ae3eef377d6273025294ff2338d18755f Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 15:49:34 +0000 Subject: [PATCH 13/14] Removed check for priority --- src/Protocol/Feed.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 28e0493dae..560fe005e6 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -618,8 +618,8 @@ class Feed */ private static function adjustPollFrequency(array $contact, array $creation_dates) { - if (($contact['priority'] > 3) || ($contact['network'] != Protocol::FEED)) { - Logger::info('Contact is no feed or has a low priority, skip.', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'priority' => $contact['priority'], 'network' => $contact['network']]); + if ($contact['network'] != Protocol::FEED) { + Logger::info('Contact is no feed, skip.', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'network' => $contact['network']]); return; } From 2cae485e0d18d4b4a3056f8bc2ef0fdf59212694 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Aug 2020 17:40:31 +0000 Subject: [PATCH 14/14] Don't show poll frequency on auto adjust --- src/Module/Contact.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Module/Contact.php b/src/Module/Contact.php index 6425df8df1..4a5aeefe4f 100644 --- a/src/Module/Contact.php +++ b/src/Module/Contact.php @@ -558,7 +558,7 @@ class Contact extends BaseModule } $poll_interval = null; - if (in_array($contact['network'], [Protocol::FEED, Protocol::MAIL])) { + if ((($contact['network'] == Protocol::FEED) && !DI::config()->get('system', 'adjust_poll_frequency')) || ($contact['network']== Protocol::MAIL)) { $poll_interval = ContactSelector::pollInterval($contact['priority'], !$poll_enabled); }