diff --git a/src/Module/Contact.php b/src/Module/Contact.php index 6425df8df1..4a5aeefe4f 100644 --- a/src/Module/Contact.php +++ b/src/Module/Contact.php @@ -558,7 +558,7 @@ class Contact extends BaseModule } $poll_interval = null; - if (in_array($contact['network'], [Protocol::FEED, Protocol::MAIL])) { + if ((($contact['network'] == Protocol::FEED) && !DI::config()->get('system', 'adjust_poll_frequency')) || ($contact['network']== Protocol::MAIL)) { $poll_interval = ContactSelector::pollInterval($contact['priority'], !$poll_enabled); } diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index ba0adb8347..560fe005e6 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -300,6 +300,7 @@ class Feed } $items = []; + $creation_dates = []; // Limit the number of items that are about to be fetched $total_items = ($entries->length - 1); @@ -354,16 +355,6 @@ class Feed $item["parent-uri"] = $item["uri"]; - if (!$dryRun) { - $condition = ["`uid` = ? AND `uri` = ? AND `network` IN (?, ?)", - $importer["uid"], $item["uri"], Protocol::FEED, Protocol::DFRN]; - $previous = Item::selectFirst(['id'], $condition); - if (DBA::isResult($previous)) { - Logger::info("Item with uri " . $item["uri"] . " for user " . $importer["uid"] . " already existed under id " . $previous["id"]); - continue; - } - } - $item["title"] = XML::getFirstNodeValue($xpath, 'atom:title/text()', $entry); if (empty($item["title"])) { @@ -403,6 +394,19 @@ class Feed $item["edited"] = $updated; } + if (!$dryRun) { + $condition = ["`uid` = ? AND `uri` = ? AND `network` IN (?, ?)", + $importer["uid"], $item["uri"], Protocol::FEED, Protocol::DFRN]; + $previous = Item::selectFirst(['id', 'created'], $condition); + if (DBA::isResult($previous)) { + // Use the creation date when the post had been stored. It can happen this date changes in the feed. + $creation_dates[] = $previous['created']; + Logger::info("Item with uri " . $item["uri"] . " for user " . $importer["uid"] . " already existed under id " . $previous["id"]); + continue; + } + $creation_dates[] = DateTimeFormat::utc($item['created']); + } + $creator = XML::getFirstNodeValue($xpath, 'author/text()', $entry); if (empty($creator)) { @@ -598,9 +602,130 @@ class Feed } } + if (!$dryRun && DI::config()->get('system', 'adjust_poll_frequency')) { + self::adjustPollFrequency($contact, $creation_dates); + } + return ["header" => $author, "items" => $items]; } + /** + * Automatically adjust the poll frequency according to the post frequency + * + * @param array $contact + * @param array $creation_dates + * @return void + */ + private static function adjustPollFrequency(array $contact, array $creation_dates) + { + if ($contact['network'] != Protocol::FEED) { + Logger::info('Contact is no feed, skip.', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'network' => $contact['network']]); + return; + } + + if (!empty($creation_dates)) { + // Count the post frequency and the earliest and latest post date + $frequency = []; + $oldest = time(); + $newest = 0; + $oldest_date = $newest_date = ''; + + foreach ($creation_dates as $date) { + $timestamp = strtotime($date); + $day = intdiv($timestamp, 86400); + $hour = $timestamp % 86400; + + // Only have a look at values from the last seven days + if (((time() / 86400) - $day) < 7) { + if (empty($frequency[$day])) { + $frequency[$day] = ['count' => 1, 'low' => $hour, 'high' => $hour]; + } else { + ++$frequency[$day]['count']; + if ($frequency[$day]['low'] > $hour) { + $frequency[$day]['low'] = $hour; + } + if ($frequency[$day]['high'] < $hour) { + $frequency[$day]['high'] = $hour; + } + } + } + if ($oldest > $day) { + $oldest = $day; + $oldest_date = $date; + } + + if ($newest < $day) { + $newest = $day; + $newest_date = $date; + } + } + + if (count($creation_dates) == 1) { + Logger::info('Feed had posted a single time, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 8; // Poll once a day + } + + if (empty($priority) && (((time() / 86400) - $newest) > 730)) { + Logger::info('Feed had not posted for two years, switching to monthly polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 10; // Poll every month + } + + if (empty($priority) && (((time() / 86400) - $newest) > 365)) { + Logger::info('Feed had not posted for a year, switching to weekly polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 9; // Poll every week + } + + if (empty($priority) && empty($frequency)) { + Logger::info('Feed had not posted for at least a week, switching to daily polling', ['newest' => $newest_date, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 8; // Poll once a day + } + + if (empty($priority)) { + // Calculate the highest "posts per day" value + $max = 0; + foreach ($frequency as $entry) { + if (($entry['count'] == 1) || ($entry['high'] == $entry['low'])) { + continue; + } + + // We take the earliest and latest post day and interpolate the number of post per day + // that would had been created with this post frequency + + // Assume at least four hours between oldest and newest post per day - should be okay for news outlets + $duration = max($entry['high'] - $entry['low'], 14400); + $ppd = (86400 / $duration) * $entry['count']; + if ($ppd > $max) { + $max = $ppd; + } + } + if ($max > 48) { + $priority = 1; // Poll every quarter hour + } elseif ($max > 24) { + $priority = 2; // Poll half an hour + } elseif ($max > 12) { + $priority = 3; // Poll hourly + } elseif ($max > 8) { + $priority = 4; // Poll every two hours + } elseif ($max > 4) { + $priority = 5; // Poll every three hours + } elseif ($max > 2) { + $priority = 6; // Poll every six hours + } else { + $priority = 7; // Poll twice a day + } + Logger::info('Calculated priority by the posts per day', ['priority' => $priority, 'max' => round($max, 2), 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + } + } else { + Logger::info('No posts, switching to daily polling', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + $priority = 8; // Poll once a day + } + + if ($contact['rating'] != $priority) { + Logger::notice('Adjusting priority', ['old' => $contact['rating'], 'new' => $priority, 'id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url']]); + DBA::update('contact', ['rating' => $priority], ['id' => $contact['id']]); + } + } + /** * Convert a tag array to a tag string * diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index ecfd2e7ddc..d7cd7c0546 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -186,7 +186,7 @@ class Cron Addon::reload(); $sql = "SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`, `contact`.`archive`, - `contact`.`last-update`, `contact`.`priority`, `contact`.`rel`, `contact`.`subhub` + `contact`.`last-update`, `contact`.`priority`, `contact`.`rating`, `contact`.`rel`, `contact`.`subhub` FROM `user` STRAIGHT_JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`poll` != '' @@ -217,65 +217,43 @@ class Cron } while ($contact = DBA::fetch($contacts)) { + $ratings = [0, 3, 7, 8, 9, 10]; + if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { + $rating = $contact['rating']; + } elseif (array_key_exists($contact['priority'], $ratings)) { + $rating = $ratings[$contact['priority']]; + } else { + $rating = -1; + } + // Friendica and OStatus are checked once a day if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { - $contact['priority'] = 3; + $rating = 8; } // ActivityPub is checked once a week if ($contact['network'] == Protocol::ACTIVITYPUB) { - $contact['priority'] = 4; + $rating = 9; } // Check archived contacts once a month if ($contact['archive']) { - $contact['priority'] = 5; + $rating = 10; } - if ($contact['priority'] >= 0) { - $update = false; + if ($rating < 0) { + continue; + } + /* + * Based on $contact['priority'], should we poll this site now? Or later? + */ + $t = $contact['last-update']; - $t = $contact['last-update']; + $poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute', + '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month']; - /* - * Based on $contact['priority'], should we poll this site now? Or later? - */ - switch ($contact['priority']) { - case 5: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 month")) { - $update = true; - } - break; - case 4: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 week")) { - $update = true; - } - break; - case 3: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 day")) { - $update = true; - } - break; - case 2: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 12 hour")) { - $update = true; - } - break; - case 1: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 hour")) { - $update = true; - } - break; - case 0: - default: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + " . $min_poll_interval . " minute")) { - $update = true; - } - break; - } - if (!$update) { - continue; - } + if (empty($poll_intervals[$rating]) || (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . ' + ' . $poll_intervals[$rating]))) { + continue; } if ((($contact['network'] == Protocol::FEED) && ($contact['priority'] <= 3)) || ($contact['network'] == Protocol::MAIL)) { diff --git a/static/defaults.config.php b/static/defaults.config.php index 5e5b48c683..018e0dc635 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -82,6 +82,10 @@ return [ 'php_path' => 'php', ], 'system' => [ + // adjust_poll_frequency (Boolean) + // Automatically detect and set the best feed poll frequency. + 'adjust_poll_frequency' => false, + // allowed_link_protocols (Array) // Allowed protocols in links URLs, add at your own risk. http(s) is always allowed. 'allowed_link_protocols' => ['ftp://', 'ftps://', 'mailto:', 'cid:', 'gopher://'],