From b7e8c1595de3b1b4eeeadf1eabd5a471d3841b92 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sun, 16 Aug 2020 17:59:37 +0000
Subject: [PATCH 01/14] Automatically adjust feed frequencies
---
src/Protocol/Feed.php | 140 +++++++++++++++++++++++++++++++++++++++---
1 file changed, 130 insertions(+), 10 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index ba0adb8347..61ef846800 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -300,6 +300,7 @@ class Feed
}
$items = [];
+ $creation_dates = [];
// Limit the number of items that are about to be fetched
$total_items = ($entries->length - 1);
@@ -354,16 +355,6 @@ class Feed
$item["parent-uri"] = $item["uri"];
- if (!$dryRun) {
- $condition = ["`uid` = ? AND `uri` = ? AND `network` IN (?, ?)",
- $importer["uid"], $item["uri"], Protocol::FEED, Protocol::DFRN];
- $previous = Item::selectFirst(['id'], $condition);
- if (DBA::isResult($previous)) {
- Logger::info("Item with uri " . $item["uri"] . " for user " . $importer["uid"] . " already existed under id " . $previous["id"]);
- continue;
- }
- }
-
$item["title"] = XML::getFirstNodeValue($xpath, 'atom:title/text()', $entry);
if (empty($item["title"])) {
@@ -403,6 +394,19 @@ class Feed
$item["edited"] = $updated;
}
+ if (!$dryRun) {
+ $condition = ["`uid` = ? AND `uri` = ? AND `network` IN (?, ?)",
+ $importer["uid"], $item["uri"], Protocol::FEED, Protocol::DFRN];
+ $previous = Item::selectFirst(['id', 'created'], $condition);
+ if (DBA::isResult($previous)) {
+ // Use the creation date when the post had been stored. It can happen this date changes in the feed.
+ $creation_dates[] = $previous['created'];
+ Logger::info("Item with uri " . $item["uri"] . " for user " . $importer["uid"] . " already existed under id " . $previous["id"]);
+ continue;
+ }
+ $creation_dates[] = DateTimeFormat::utc($item['created']);
+ }
+
$creator = XML::getFirstNodeValue($xpath, 'author/text()', $entry);
if (empty($creator)) {
@@ -598,9 +602,125 @@ class Feed
}
}
+ if (!$dryRun) {
+ self::adjustPollFrequency($contact, $creation_dates);
+ }
+
return ["header" => $author, "items" => $items];
}
+ /**
+ * Automatically adjust the poll frequency according to the post frequency
+ *
+ * @param array $contact
+ * @param array $creation_dates
+ * @return void
+ */
+ private static function adjustPollFrequency(array $contact, array $creation_dates)
+ {
+ if (($contact['priority'] > 3) || ($contact['network'] != Protocol::FEED)) {
+ Logger::info('Contact is no feed or has a low priority, skip.', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'priority' => $contact['priority'], 'network' => $contact['network']]);
+ return;
+ }
+
+ if (!empty($creation_dates)) {
+ // Count the post frequency and the earliest and latest post date
+ $frequency = [];
+ $oldest = time();
+ $newest = 0;
+ $oldest_date = $newest_date = '';
+
+ foreach ($creation_dates as $date) {
+ $timestamp = strtotime($date);
+ $day = intdiv($timestamp, 86400);
+ $hour = $timestamp % 86400;
+
+ // Only have a look at values from the last seven days
+ if (((time() / 86400) - $day) < 7) {
+ if (empty($frequency[$day])) {
+ $frequency[$day] = ['count' => 1, 'low' => $hour, 'high' => $hour];
+ } else {
+ ++$frequency[$day]['count'];
+ if ($frequency[$day]['low'] > $hour) {
+ $frequency[$day]['low'] = $hour;
+ }
+ if ($frequency[$day]['high'] < $hour) {
+ $frequency[$day]['high'] = $hour;
+ }
+ }
+ }
+ if ($oldest > $day) {
+ $oldest = $day;
+ $oldest_date = $date;
+ }
+
+ if ($newest < $day) {
+ $newest = $day;
+ $newest_date = $date;
+ }
+ }
+
+ if (($newest == $oldest) && count($creation_dates) > 1) {
+ Logger::info('Feed has no different creation dates, quitting', ['date' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ return;
+ }
+
+ if (((time() / 86400) - $newest) > 30) {
+ Logger::info('Feed had not posted for a month, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ $priority = 3; // Poll once a day
+ }
+
+ if (count($creation_dates) == 1) {
+ Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ $priority = 3; // Poll once a day
+ }
+
+ if (empty($priority) && (($newest - $oldest) > count($creation_dates))) {
+ Logger::info('Less than a post per day, switching to daily polling', ['posts' => count($creation_dates), 'oldest' => $oldest_date, 'newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ $priority = 3; // Poll once a day
+ }
+
+ if (empty($priority)) {
+ // Calculate the highest "posts per day" value
+ $max = 1;
+ foreach ($frequency as $entry) {
+ if (($entry['count'] == 1) || ($entry['high'] == $entry['low'])) {
+ continue;
+ }
+
+ // We take the earliest and latest post day and interpolate the number of post per day
+ // that would had been created with this post frequency
+
+ // Assume at least four hours between oldest and newest post per day - should be okay for news outlets
+ $duration = max($entry['high'] - $entry['low'], 14400);
+ $ppd = (86400 / $duration) * $entry['count'];
+ if ($ppd > $max) {
+ $max = $ppd;
+ }
+ }
+
+ if ($max >= 24) {
+ $priority = 0; // Poll with the minimum poll intervall
+ } elseif ($max >= 12) {
+ $priority = 1; // Poll hourly
+ } elseif ($max > 1) {
+ $priority = 2; // Poll twice a day
+ } else {
+ /// @todo In the future we could calculate the days between the posts to set even lower priorities
+ $priority = 3; // Poll once a day
+ }
+ Logger::info('Calculated priority by the posts per day', ['priority' => $priority, 'max' => round($max, 2), 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ }
+ } else {
+ Logger::info('No posts, switching to daily polling', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ $priority = 3; // Poll once a day
+ }
+
+ if ($contact['priority'] != $priority) {
+ Logger::info('Adjusting priority', ['old' => $contact['priority'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ }
+ }
+
/**
* Convert a tag array to a tag string
*
From 112dd37a09b121d00197173a24f9b5e41abcc63c Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sun, 16 Aug 2020 18:05:34 +0000
Subject: [PATCH 02/14] Switch to daily after a week
---
src/Protocol/Feed.php | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 61ef846800..9ba2089c13 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -665,12 +665,12 @@ class Feed
return;
}
- if (((time() / 86400) - $newest) > 30) {
- Logger::info('Feed had not posted for a month, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ if (empty($frequency)) {
+ Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
$priority = 3; // Poll once a day
}
- if (count($creation_dates) == 1) {
+ if (empty($priority) && (count($creation_dates) == 1)) {
Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
$priority = 3; // Poll once a day
}
From c4aebffd3b29a82306c76307f7c4a93f95efcd92 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sun, 16 Aug 2020 18:07:36 +0000
Subject: [PATCH 03/14] Setting priority to "notice"
---
src/Protocol/Feed.php | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 9ba2089c13..78830c5603 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -717,7 +717,7 @@ class Feed
}
if ($contact['priority'] != $priority) {
- Logger::info('Adjusting priority', ['old' => $contact['priority'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ Logger::notice('Adjusting priority', ['old' => $contact['priority'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
}
}
From df75a24dc7a501e9081d285f7f09060af7655583 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sun, 16 Aug 2020 21:38:26 +0000
Subject: [PATCH 04/14] Use config value to trigger automated rating
---
src/Protocol/Feed.php | 7 ++++---
src/Worker/Cron.php | 7 ++++++-
static/defaults.config.php | 4 ++++
3 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 78830c5603..915d78ac39 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -602,7 +602,7 @@ class Feed
}
}
- if (!$dryRun) {
+ if (!$dryRun && DI::config()->get('system', 'adjust_poll_frequency')) {
self::adjustPollFrequency($contact, $creation_dates);
}
@@ -716,8 +716,9 @@ class Feed
$priority = 3; // Poll once a day
}
- if ($contact['priority'] != $priority) {
- Logger::notice('Adjusting priority', ['old' => $contact['priority'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ if ($contact['rating'] != $priority) {
+ Logger::notice('Adjusting priority', ['old' => $contact['rating'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ DBA::update('contact', ['rating' => $priority], ['id' => $contact['id']]);
}
}
diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php
index ecfd2e7ddc..286a19cc97 100644
--- a/src/Worker/Cron.php
+++ b/src/Worker/Cron.php
@@ -186,7 +186,7 @@ class Cron
Addon::reload();
$sql = "SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`, `contact`.`archive`,
- `contact`.`last-update`, `contact`.`priority`, `contact`.`rel`, `contact`.`subhub`
+ `contact`.`last-update`, `contact`.`priority`, `contact`.`rating`, `contact`.`rel`, `contact`.`subhub`
FROM `user`
STRAIGHT_JOIN `contact`
ON `contact`.`uid` = `user`.`uid` AND `contact`.`poll` != ''
@@ -217,6 +217,11 @@ class Cron
}
while ($contact = DBA::fetch($contacts)) {
+ // Use the "rating" field when auto adjusting the poll intervall
+ if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) {
+ $contact['priority'] = max($contact['rating'], $contact['priority']);
+ }
+
// Friendica and OStatus are checked once a day
if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) {
$contact['priority'] = 3;
diff --git a/static/defaults.config.php b/static/defaults.config.php
index 5e5b48c683..018e0dc635 100644
--- a/static/defaults.config.php
+++ b/static/defaults.config.php
@@ -82,6 +82,10 @@ return [
'php_path' => 'php',
],
'system' => [
+ // adjust_poll_frequency (Boolean)
+ // Automatically detect and set the best feed poll frequency.
+ 'adjust_poll_frequency' => false,
+
// allowed_link_protocols (Array)
// Allowed protocols in links URLs, add at your own risk. http(s) is always allowed.
'allowed_link_protocols' => ['ftp://', 'ftps://', 'mailto:', 'cid:', 'gopher://'],
From 975bb76291536f3c20c136fd4f15e8721158a612 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 05:26:40 +0000
Subject: [PATCH 05/14] Removed check for number of posts
---
src/Protocol/Feed.php | 5 -----
1 file changed, 5 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 915d78ac39..9222d36057 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -675,11 +675,6 @@ class Feed
$priority = 3; // Poll once a day
}
- if (empty($priority) && (($newest - $oldest) > count($creation_dates))) {
- Logger::info('Less than a post per day, switching to daily polling', ['posts' => count($creation_dates), 'oldest' => $oldest_date, 'newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
- $priority = 3; // Poll once a day
- }
-
if (empty($priority)) {
// Calculate the highest "posts per day" value
$max = 1;
From 7370a075f8185fb2084cabb43a99c821f4a4a678 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 06:47:29 +0000
Subject: [PATCH 06/14] New rating scale
---
src/Protocol/Feed.php | 21 ++++++++++-----
src/Worker/Cron.php | 59 +++++++++++++++++++++++++++++++++++--------
2 files changed, 62 insertions(+), 18 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 9222d36057..c67dfcdda9 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -693,16 +693,23 @@ class Feed
$max = $ppd;
}
}
-
- if ($max >= 24) {
- $priority = 0; // Poll with the minimum poll intervall
- } elseif ($max >= 12) {
- $priority = 1; // Poll hourly
+ if ($max > 48) {
+ $priority = 1; // Poll every quarter hour
+ } elseif ($max > 24) {
+ $priority = 2; // Poll half an hour
+ } elseif ($max > 12) {
+ $priority = 3; // Poll hourly
+ } elseif ($max > 8) {
+ $priority = 4; // Poll every two hours
+ } elseif ($max > 4) {
+ $priority = 5; // Poll every three hours
+ } elseif ($max > 2) {
+ $priority = 6; // Poll every six hours
} elseif ($max > 1) {
- $priority = 2; // Poll twice a day
+ $priority = 7; // Poll twice a day
} else {
/// @todo In the future we could calculate the days between the posts to set even lower priorities
- $priority = 3; // Poll once a day
+ $priority = 8; // Poll once a day
}
Logger::info('Calculated priority by the posts per day', ['priority' => $priority, 'max' => round($max, 2), 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
}
diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php
index 286a19cc97..a67d99407f 100644
--- a/src/Worker/Cron.php
+++ b/src/Worker/Cron.php
@@ -219,25 +219,37 @@ class Cron
while ($contact = DBA::fetch($contacts)) {
// Use the "rating" field when auto adjusting the poll intervall
if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) {
- $contact['priority'] = max($contact['rating'], $contact['priority']);
+ $rating = $contact['rating'];
+ } elseif ($contact['priority'] == 1) {
+ $rating = 3;
+ } elseif ($contact['priority'] == 2) {
+ $rating = 7;
+ } elseif ($contact['priority'] == 3) {
+ $rating = 8;
+ } elseif ($contact['priority'] == 4) {
+ $rating = 9;
+ } elseif ($contact['priority'] == 5) {
+ $rating = 10;
+ } else {
+ $rating = -1;
}
// Friendica and OStatus are checked once a day
if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) {
- $contact['priority'] = 3;
+ $rating = 8;
}
// ActivityPub is checked once a week
if ($contact['network'] == Protocol::ACTIVITYPUB) {
- $contact['priority'] = 4;
+ $rating = 9;
}
// Check archived contacts once a month
if ($contact['archive']) {
- $contact['priority'] = 5;
+ $rating = 10;
}
- if ($contact['priority'] >= 0) {
+ if ($rating >= 0) {
$update = false;
$t = $contact['last-update'];
@@ -245,32 +257,57 @@ class Cron
/*
* Based on $contact['priority'], should we poll this site now? Or later?
*/
- switch ($contact['priority']) {
- case 5:
+ switch ($rating) {
+ case 10:
if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 month")) {
$update = true;
}
break;
- case 4:
+ case 9:
if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 week")) {
$update = true;
}
break;
- case 3:
+ case 8:
if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 day")) {
$update = true;
}
break;
- case 2:
+ case 7:
if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 12 hour")) {
$update = true;
}
break;
- case 1:
+ case 6:
+ if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 6 hour")) {
+ $update = true;
+ }
+ break;
+ case 5:
+ if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 3 hour")) {
+ $update = true;
+ }
+ break;
+ case 4:
+ if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 2 hour")) {
+ $update = true;
+ }
+ break;
+ case 3:
if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 hour")) {
$update = true;
}
break;
+ case 2:
+ if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 30 minute")) {
+ $update = true;
+ }
+ break;
+ case 1:
+ if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 15 minute")) {
+ $update = true;
+ }
+ break;
case 0:
default:
if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + " . $min_poll_interval . " minute")) {
From ad2674aac69137cf74fc9d70aba8ec30ecdf2e4c Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 06:50:51 +0000
Subject: [PATCH 07/14] Once a day is 8
---
src/Protocol/Feed.php | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index c67dfcdda9..0681b863a5 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -667,12 +667,12 @@ class Feed
if (empty($frequency)) {
Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
- $priority = 3; // Poll once a day
+ $priority = 8; // Poll once a day
}
if (empty($priority) && (count($creation_dates) == 1)) {
Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
- $priority = 3; // Poll once a day
+ $priority = 8; // Poll once a day
}
if (empty($priority)) {
@@ -715,7 +715,7 @@ class Feed
}
} else {
Logger::info('No posts, switching to daily polling', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
- $priority = 3; // Poll once a day
+ $priority = 8; // Poll once a day
}
if ($contact['rating'] != $priority) {
From 224ef91715beacec7dd3bb9a7335fbc5196d8b6a Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 07:44:59 +0000
Subject: [PATCH 08/14] Simplify code
---
src/Worker/Cron.php | 92 +++++++--------------------------------------
1 file changed, 14 insertions(+), 78 deletions(-)
diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php
index a67d99407f..89a2b7373d 100644
--- a/src/Worker/Cron.php
+++ b/src/Worker/Cron.php
@@ -217,19 +217,11 @@ class Cron
}
while ($contact = DBA::fetch($contacts)) {
- // Use the "rating" field when auto adjusting the poll intervall
+ $ratings = [0, 3, 7, 8, 9, 10];
if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) {
$rating = $contact['rating'];
- } elseif ($contact['priority'] == 1) {
- $rating = 3;
- } elseif ($contact['priority'] == 2) {
- $rating = 7;
- } elseif ($contact['priority'] == 3) {
- $rating = 8;
- } elseif ($contact['priority'] == 4) {
- $rating = 9;
- } elseif ($contact['priority'] == 5) {
- $rating = 10;
+ } elseif (array_key_exists($ratings, $contact['priority'])) {
+ $rating = $ratings[$contact['priority']];
} else {
$rating = -1;
}
@@ -249,75 +241,19 @@ class Cron
$rating = 10;
}
- if ($rating >= 0) {
- $update = false;
+ if ($rating < 0) {
+ continue;
+ }
+ /*
+ * Based on $contact['priority'], should we poll this site now? Or later?
+ */
+ $t = $contact['last-update'];
- $t = $contact['last-update'];
+ $poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute',
+ '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month'];
- /*
- * Based on $contact['priority'], should we poll this site now? Or later?
- */
- switch ($rating) {
- case 10:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 month")) {
- $update = true;
- }
- break;
- case 9:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 week")) {
- $update = true;
- }
- break;
- case 8:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 day")) {
- $update = true;
- }
- break;
- case 7:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 12 hour")) {
- $update = true;
- }
- break;
- case 6:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 6 hour")) {
- $update = true;
- }
- break;
- case 5:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 3 hour")) {
- $update = true;
- }
- break;
- case 4:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 2 hour")) {
- $update = true;
- }
- break;
- case 3:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 hour")) {
- $update = true;
- }
- break;
- case 2:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 30 minute")) {
- $update = true;
- }
- break;
- case 1:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 15 minute")) {
- $update = true;
- }
- break;
- case 0:
- default:
- if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + " . $min_poll_interval . " minute")) {
- $update = true;
- }
- break;
- }
- if (!$update) {
- continue;
- }
+ if (empty($poll_intervals[$rating]) || (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . ' + ' . $poll_intervals[$rating]))) {
+ continue;
}
if ((($contact['network'] == Protocol::FEED) && ($contact['priority'] <= 3)) || ($contact['network'] == Protocol::MAIL)) {
From ec35e2827b59dd1c942591b3f4bedb1c284ddb9a Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 07:51:15 +0000
Subject: [PATCH 09/14] Remove check
---
src/Protocol/Feed.php | 5 -----
1 file changed, 5 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 0681b863a5..917352793b 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -660,11 +660,6 @@ class Feed
}
}
- if (($newest == $oldest) && count($creation_dates) > 1) {
- Logger::info('Feed has no different creation dates, quitting', ['date' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
- return;
- }
-
if (empty($frequency)) {
Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
$priority = 8; // Poll once a day
From af47a9f36c60851f8b2b7262c8cf169a21f006c9 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 08:55:20 +0000
Subject: [PATCH 10/14] Fix parameter order
---
src/Worker/Cron.php | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php
index 89a2b7373d..d7cd7c0546 100644
--- a/src/Worker/Cron.php
+++ b/src/Worker/Cron.php
@@ -220,7 +220,7 @@ class Cron
$ratings = [0, 3, 7, 8, 9, 10];
if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) {
$rating = $contact['rating'];
- } elseif (array_key_exists($ratings, $contact['priority'])) {
+ } elseif (array_key_exists($contact['priority'], $ratings)) {
$rating = $ratings[$contact['priority']];
} else {
$rating = -1;
From fb9832bbd0b76da844721ac65d482851ef5764e5 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 09:53:49 +0000
Subject: [PATCH 11/14] Poll twice a day when there had been posts in the last
7 days
---
src/Protocol/Feed.php | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 917352793b..8a6725d19a 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -672,7 +672,7 @@ class Feed
if (empty($priority)) {
// Calculate the highest "posts per day" value
- $max = 1;
+ $max = 0;
foreach ($frequency as $entry) {
if (($entry['count'] == 1) || ($entry['high'] == $entry['low'])) {
continue;
@@ -700,7 +700,7 @@ class Feed
$priority = 5; // Poll every three hours
} elseif ($max > 2) {
$priority = 6; // Poll every six hours
- } elseif ($max > 1) {
+ } elseif ($max > 0) {
$priority = 7; // Poll twice a day
} else {
/// @todo In the future we could calculate the days between the posts to set even lower priorities
From e0490aff5c6d204f3bd3f7516049bdb46e1a6d29 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 12:25:55 +0000
Subject: [PATCH 12/14] Added weekly and monthly polling
---
src/Protocol/Feed.php | 23 +++++++++++++++--------
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 8a6725d19a..28e0493dae 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -660,13 +660,23 @@ class Feed
}
}
- if (empty($frequency)) {
- Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ if (count($creation_dates) == 1) {
+ Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
$priority = 8; // Poll once a day
}
- if (empty($priority) && (count($creation_dates) == 1)) {
- Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ if (empty($priority) && (((time() / 86400) - $newest) > 730)) {
+ Logger::info('Feed had not posted for two years, switching to monthly polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ $priority = 10; // Poll every month
+ }
+
+ if (empty($priority) && (((time() / 86400) - $newest) > 365)) {
+ Logger::info('Feed had not posted for a year, switching to weekly polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
+ $priority = 9; // Poll every week
+ }
+
+ if (empty($priority) && empty($frequency)) {
+ Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
$priority = 8; // Poll once a day
}
@@ -700,11 +710,8 @@ class Feed
$priority = 5; // Poll every three hours
} elseif ($max > 2) {
$priority = 6; // Poll every six hours
- } elseif ($max > 0) {
- $priority = 7; // Poll twice a day
} else {
- /// @todo In the future we could calculate the days between the posts to set even lower priorities
- $priority = 8; // Poll once a day
+ $priority = 7; // Poll twice a day
}
Logger::info('Calculated priority by the posts per day', ['priority' => $priority, 'max' => round($max, 2), 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]);
}
From 312ac73ae3eef377d6273025294ff2338d18755f Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 15:49:34 +0000
Subject: [PATCH 13/14] Removed check for priority
---
src/Protocol/Feed.php | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php
index 28e0493dae..560fe005e6 100644
--- a/src/Protocol/Feed.php
+++ b/src/Protocol/Feed.php
@@ -618,8 +618,8 @@ class Feed
*/
private static function adjustPollFrequency(array $contact, array $creation_dates)
{
- if (($contact['priority'] > 3) || ($contact['network'] != Protocol::FEED)) {
- Logger::info('Contact is no feed or has a low priority, skip.', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'priority' => $contact['priority'], 'network' => $contact['network']]);
+ if ($contact['network'] != Protocol::FEED) {
+ Logger::info('Contact is no feed, skip.', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'network' => $contact['network']]);
return;
}
From 2cae485e0d18d4b4a3056f8bc2ef0fdf59212694 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Mon, 17 Aug 2020 17:40:31 +0000
Subject: [PATCH 14/14] Don't show poll frequency on auto adjust
---
src/Module/Contact.php | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/Module/Contact.php b/src/Module/Contact.php
index 6425df8df1..4a5aeefe4f 100644
--- a/src/Module/Contact.php
+++ b/src/Module/Contact.php
@@ -558,7 +558,7 @@ class Contact extends BaseModule
}
$poll_interval = null;
- if (in_array($contact['network'], [Protocol::FEED, Protocol::MAIL])) {
+ if ((($contact['network'] == Protocol::FEED) && !DI::config()->get('system', 'adjust_poll_frequency')) || ($contact['network']== Protocol::MAIL)) {
$poll_interval = ContactSelector::pollInterval($contact['priority'], !$poll_enabled);
}