From cd55c742e8252c91e01520ebecdc1d9fcfe18be3 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 3 Dec 2017 08:59:24 +0000 Subject: [PATCH] Check Diaspora contacts, don't poll unreachable contacts too often --- src/Worker/Cron.php | 164 +++++++++++++++++++---------------------- src/Worker/OnePoll.php | 55 +++++++++++--- 2 files changed, 123 insertions(+), 96 deletions(-) diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 354cb35853..1a7a5784dc 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -153,17 +153,20 @@ Class Cron { : '' ); - $contacts = q("SELECT `contact`.`id` FROM `user` + $contacts = q("SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`, + `contact`.`last-update`, `contact`.`priority`, `contact`.`subhub` + FROM `user` STRAIGHT_JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`rel` IN (%d, %d) AND `contact`.`poll` != '' - AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s') $sql_extra + AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s', '%s') $sql_extra AND NOT `contact`.`self` AND NOT `contact`.`blocked` AND NOT `contact`.`readonly` AND NOT `contact`.`archive` - WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql ORDER BY RAND()", + WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql", intval(CONTACT_IS_SHARING), intval(CONTACT_IS_FRIEND), dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), + dbesc(NETWORK_DIASPORA), dbesc(NETWORK_FEED), dbesc(NETWORK_MAIL), dbesc(NETWORK_MAIL2) @@ -173,94 +176,81 @@ Class Cron { return; } - foreach ($contacts as $c) { + foreach ($contacts as $contact) { - $res = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1", - intval($c['id']) - ); - - if (!DBM::is_result($res)) { - continue; + if ($manual_id) { + $contact['last-update'] = NULL_DATE; } - foreach ($res as $contact) { - - $xml = false; - - if ($manual_id) { - $contact['last-update'] = NULL_DATE; - } - - if (in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) { - $contact['priority'] = 2; - } - - if ($contact['subhub'] && in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) { - /* - * We should be getting everything via a hub. But just to be sure, let's check once a day. - * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately) - * This also lets us update our subscription to the hub, and add or replace hubs in case it - * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'. - */ - $poll_interval = Config::get('system', 'pushpoll_frequency'); - $contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3); - } - - if (($contact['priority'] >= 0) && !$force) { - $update = false; - - $t = $contact['last-update']; - - /* - * Based on $contact['priority'], should we poll this site now? Or later? - */ - switch ($contact['priority']) { - case 5: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 month")) { - $update = true; - } - break; - case 4: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 week")) { - $update = true; - } - break; - case 3: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 day")) { - $update = true; - } - break; - case 2: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 12 hour")) { - $update = true; - } - break; - case 1: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 hour")) { - $update = true; - } - break; - case 0: - default: - if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + ".$min_poll_interval." minute")) { - $update = true; - } - break; - } - if (!$update) { - continue; - } - } - - logger("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact["nick"] . " " . $contact["name"]); - - if (($contact['network'] == NETWORK_FEED) && ($contact['priority'] <= 3)) { - $priority = PRIORITY_MEDIUM; - } else { - $priority = PRIORITY_LOW; - } - Worker::add(array('priority' => $priority, 'dont_fork' => true), 'OnePoll', (int)$contact['id']); + if (in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) { + $contact['priority'] = 2; } + + if ($contact['subhub'] && in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) { + /* + * We should be getting everything via a hub. But just to be sure, let's check once a day. + * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately) + * This also lets us update our subscription to the hub, and add or replace hubs in case it + * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'. + */ + $poll_interval = Config::get('system', 'pushpoll_frequency'); + $contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3); + } + + if (($contact['priority'] >= 0) && !$force) { + $update = false; + + $t = $contact['last-update']; + + /* + * Based on $contact['priority'], should we poll this site now? Or later? + */ + switch ($contact['priority']) { + case 5: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 month")) { + $update = true; + } + break; + case 4: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 week")) { + $update = true; + } + break; + case 3: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 day")) { + $update = true; + } + break; + case 2: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 12 hour")) { + $update = true; + } + break; + case 1: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 hour")) { + $update = true; + } + break; + case 0: + default: + if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + ".$min_poll_interval." minute")) { + $update = true; + } + break; + } + if (!$update) { + continue; + } + } + + logger("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact["nick"] . " " . $contact["name"]); + + if (($contact['network'] == NETWORK_FEED) && ($contact['priority'] <= 3)) { + $priority = PRIORITY_MEDIUM; + } else { + $priority = PRIORITY_LOW; + } + Worker::add(array('priority' => $priority, 'dont_fork' => true), 'OnePoll', (int)$contact['id']); } } } diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index f7f7bcccf8..c5a62e9826 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -23,7 +23,7 @@ Class OnePoll require_once 'include/items.php'; require_once 'include/queue_fn.php'; - logger('onepoll: start'); + logger('start'); $manual_id = 0; $generation = 0; @@ -36,7 +36,7 @@ Class OnePoll } if (!$contact_id) { - logger('onepoll: no contact'); + logger('no contact'); return; } @@ -88,10 +88,20 @@ Class OnePoll $last_updated = PortableContact::lastUpdated($contact["url"]); $updated = datetime_convert(); if ($last_updated) { + logger('Diaspora contact '.$contact['id'].' had last update on '.$last_updated, LOGGER_DEBUG); + + // The last public item can be older than the last item we got + if ($last_updated < $contact['last-item']) { + $last_updated = $contact['last-item']; + } + $fields = array('last-item' => $last_updated, 'last-update' => $updated, 'success_update' => $updated); dba::update('contact', $fields, array('id' => $contact['id'])); + Contact::unmarkForArchival($contact); } else { dba::update('contact', array('last-update' => $updated, 'failure_update' => $updated), array('id' => $contact['id'])); + Contact::markForArchival($contact); + logger('Diaspora contact '.$contact['id'].' is marked for archival', LOGGER_DEBUG); } } return; @@ -122,12 +132,18 @@ Class OnePoll if (($contact['network'] === NETWORK_OSTATUS) || ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DFRN)) { if (!PortableContact::reachable($contact['url'])) { logger("Skipping probably dead contact ".$contact['url']); + + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); return; } if (!update_contact($contact["id"])) { Contact::markForArchival($contact); logger('Contact is marked dead'); + + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); return; } else { Contact::unmarkForArchival($contact); @@ -136,6 +152,9 @@ Class OnePoll if ($importer_uid == 0) { logger('Ignore public contacts'); + + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); return; } @@ -145,12 +164,15 @@ Class OnePoll if (!DBM::is_result($r)) { logger('No self contact for user '.$importer_uid); + + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); return; } $importer = $r[0]; - logger("onepoll: poll: ({$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}"); + logger("poll: ({$contact['network']}-{$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}"); if ($contact['network'] === NETWORK_DFRN) { $idtosend = $orig_id = (($contact['dfrn-id']) ? $contact['dfrn-id'] : $contact['issued-id']); @@ -179,6 +201,9 @@ Class OnePoll $ret = z_fetch_url($url); if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) { + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); + Contact::markForArchival($contact); return; } @@ -186,7 +211,7 @@ Class OnePoll $html_code = $a->get_curl_code(); - logger('onepoll: handshake with url ' . $url . ' returns xml: ' . $handshake_xml, LOGGER_DATA); + logger('handshake with url ' . $url . ' returns xml: ' . $handshake_xml, LOGGER_DATA); if (!strlen($handshake_xml) || ($html_code >= 400) || !$html_code) { @@ -201,7 +226,6 @@ Class OnePoll // set the last-update so we don't keep polling $fields = array('last-update' => datetime_convert(), 'failure_update' => datetime_convert()); dba::update('contact', $fields, array('id' => $contact['id'])); - return; } @@ -212,7 +236,6 @@ Class OnePoll $fields = array('last-update' => datetime_convert(), 'failure_update' => datetime_convert()); dba::update('contact', $fields, array('id' => $contact['id'])); - return; } @@ -234,6 +257,8 @@ Class OnePoll } if ((intval($res->status) != 0) || !strlen($res->challenge) || !strlen($res->dfrn_id)) { + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); return; } @@ -264,8 +289,12 @@ Class OnePoll } if ($final_dfrn_id != $orig_id) { - logger('ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id); // did not decode properly - cannot trust this site + logger('ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id); + + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); + Contact::markForArchival($contact); return; } @@ -298,6 +327,8 @@ Class OnePoll // Are we allowed to import from this person? if ($contact['rel'] == CONTACT_IS_FOLLOWER || $contact['blocked'] || $contact['readonly']) { + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); return; } @@ -305,6 +336,9 @@ Class OnePoll $ret = z_fetch_url($contact['poll'], false, $redirects, array('cookiejar' => $cookiejar)); if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) { + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); + Contact::markForArchival($contact); return; } @@ -317,6 +351,9 @@ Class OnePoll $mail_disabled = ((function_exists('imap_open') && (! Config::get('system', 'imap_disabled'))) ? 0 : 1); if ($mail_disabled) { + // set the last-update so we don't keep polling + dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]); + Contact::markForArchival($contact); return; } @@ -351,7 +388,7 @@ Class OnePoll $metas = Email::messageMeta($mbox, implode(',', $msgs)); if (count($metas) != count($msgs)) { - logger("onepoll: for " . $mailconf['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", LOGGER_DEBUG); + logger("for " . $mailconf['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", LOGGER_DEBUG); } else { $msgs = array_combine($msgs, $metas); @@ -560,7 +597,7 @@ Class OnePoll $fields = array('last-update' => datetime_convert(), 'failure_update' => datetime_convert()); dba::update('contact', $fields, array('id' => $contact['id'])); - + Contact::markForArchival($contact); return; }