clones/friendica
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #9535 from annando/ap-relay 

Relay code reworked to support AP delivery
This commit is contained in:
Hypolite Petovan 2020-11-15 20:27:08 -05:00 committed by GitHub
parent 42be636118 0384bf3e76
commit a69c98e32f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 285 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/Model/Contact.php
View File

@ -176,6 +176,14 @@ class Contact
*/
public static function insert(array $fields, bool $on_duplicate_update = false)
{
if (!empty($fields['baseurl']) && empty($fields['gsid'])) {
$fields['gsid'] = GServer::getID($fields['baseurl'], true);
}
if (empty($fields['created'])) {
$fields['created'] = DateTimeFormat::utcNow();
}
$ret = DBA::insert('contact', $fields, $on_duplicate_update);
$contact = DBA::selectFirst('contact', ['nurl', 'uid'], ['id' => DBA::lastInsertId()]);
if (!DBA::isResult($contact)) {

20
src/Model/GServer.php
View File

@ -31,7 +31,7 @@ use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Module\Register;
use Friendica\Network\CurlResult;
use Friendica\Protocol\Diaspora;
use Friendica\Protocol\Relay;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Network;
use Friendica\Util\Strings;
@ -518,7 +518,7 @@ class GServer
$data['tags'] = [];
}
$gserver = DBA::selectFirst('gserver', ['id', 'relay-subscribe', 'relay-scope'], ['nurl' => Strings::normaliseLink($server_url)]);
$gserver = DBA::selectFirst('gserver', ['id', 'url', 'network', 'relay-subscribe', 'relay-scope'], ['nurl' => Strings::normaliseLink($server_url)]);
if (!DBA::isResult($gserver)) {
return;
}
@ -567,8 +567,22 @@ class GServer
$fields['batch'] = $data['protocols']['dfrn'];
}
}
if (isset($data['protocols']['activitypub'])) {
$fields['network'] = Protocol::ACTIVITYPUB;
if (!empty($data['protocols']['activitypub']['actor'])) {
$fields['url'] = $data['protocols']['activitypub']['actor'];
}
if (!empty($data['protocols']['activitypub']['receive'])) {
$fields['batch'] = $data['protocols']['activitypub']['receive'];
}
}
}
Diaspora::setRelayContact($server_url, $fields);
Logger::info('Discovery ended', ['server' => $server_url, 'data' => $fields]);
Relay::updateContact($gserver, $fields);
}
/**

9
src/Module/WellKnown/XSocialRelay.php
View File

@ -67,8 +67,9 @@ class XSocialRelay extends BaseModule
'scope' => $scope,
'tags' => $tagList,
'protocols' => [
'diaspora' => [
'receive' => DI::baseUrl()->get() . '/receive/public'
'activitypub' => [
'actor' => DI::baseUrl()->get() . '/friendica',
'receive' => DI::baseUrl()->get() . '/inbox'
],
'dfrn' => [
'receive' => DI::baseUrl()->get() . '/dfrn_notify'
@ -76,6 +77,10 @@ class XSocialRelay extends BaseModule
]
];
if (DI::config()->get("system", "diaspora_enabled")) {
$relay['protocols']['diaspora'] = ['receive' => DI::baseUrl()->get() . '/receive/public'];
}
header('Content-type: application/json; charset=utf-8');
echo json_encode($relay, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
exit;

22
src/Protocol/ActivityPub/Transmitter.php
View File

@ -42,6 +42,7 @@ use Friendica\Model\Tag;
use Friendica\Model\User;
use Friendica\Protocol\Activity;
use Friendica\Protocol\ActivityPub;
use Friendica\Protocol\Relay;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\HTTPSignature;
use Friendica\Util\JsonLD;
@ -80,6 +81,27 @@ class Transmitter
return $inboxes;
}
/**
* Add relay servers to the list of inboxes
*
* @param array $inboxes
* @return array inboxes with added relay servers
*/
public static function addRelayServerInboxesForItem(int $item_id, array $inboxes = [])
{
$relays = Relay::getList($item_id, [], [Protocol::ACTIVITYPUB]);
if (empty($relays)) {
return $inboxes;
}
foreach ($relays as $relay) {
if (!in_array($relay['batch'], $inboxes)) {
$inboxes[] = $relay['batch'];
}
}
return $inboxes;
}
/**
* Subscribe to a relay
*

190
src/Protocol/Diaspora.php
View File

@ -56,196 +56,6 @@ use SimpleXMLElement;
*/
class Diaspora
{
/**
* Mark the relay contact of the given contact for archival
* This is called whenever there is a communication issue with the server.
* It avoids sending stuff to servers who don't exist anymore.
* The relay contact is a technical contact entry that exists once per server.
*
* @param array $contact of the relay contact
*/
public static function markRelayForArchival(array $contact)
{
if (!empty($contact['contact-type']) && ($contact['contact-type'] == Contact::TYPE_RELAY)) {
// This is already the relay contact, we don't need to fetch it
$relay_contact = $contact;
} elseif (empty($contact['baseurl'])) {
if (!empty($contact['batch'])) {
$condition = ['uid' => 0, 'network' => Protocol::FEDERATED, 'batch' => $contact['batch'], 'contact-type' => Contact::TYPE_RELAY];
$relay_contact = DBA::selectFirst('contact', [], $condition);
} else {
return;
}
} else {
$relay_contact = self::getRelayContact($contact['baseurl'], []);
}
if (!empty($relay_contact)) {
Logger::info('Relay contact will be marked for archival', ['id' => $relay_contact['id'], 'url' => $relay_contact['url']]);
Contact::markForArchival($relay_contact);
}
}
/**
* Return a list of relay servers
*
* The list contains not only the official relays but also servers that we serve directly
*
* @param integer $item_id The id of the item that is sent
* @param array $contacts The previously fetched contacts
*
* @return array of relay servers
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
public static function relayList($item_id, array $contacts = [])
{
$serverlist = [];
// Fetching relay servers
$serverdata = DI::config()->get("system", "relay_server");
if (!empty($serverdata)) {
$servers = explode(",", $serverdata);
foreach ($servers as $server) {
$serverlist[$server] = trim($server);
}
}
if (DI::config()->get("system", "relay_directly", false)) {
// We distribute our stuff based on the parent to ensure that the thread will be complete
$parent = Item::selectFirst(['uri-id'], ['id' => $item_id]);
if (!DBA::isResult($parent)) {
return;
}
// Servers that want to get all content
$servers = DBA::select('gserver', ['url'], ['relay-subscribe' => true, 'relay-scope' => 'all']);
while ($server = DBA::fetch($servers)) {
$serverlist[$server['url']] = $server['url'];
}
DBA::close($servers);
// All tags of the current post
$tags = DBA::select('tag-view', ['name'], ['uri-id' => $parent['uri-id'], 'type' => Tag::HASHTAG]);
$taglist = [];
while ($tag = DBA::fetch($tags)) {
$taglist[] = $tag['name'];
}
DBA::close($tags);
// All servers who wants content with this tag
$tagserverlist = [];
if (!empty($taglist)) {
$tagserver = DBA::select('gserver-tag', ['gserver-id'], ['tag' => $taglist]);
while ($server = DBA::fetch($tagserver)) {
$tagserverlist[] = $server['gserver-id'];
}
DBA::close($tagserver);
}
// All adresses with the given id
if (!empty($tagserverlist)) {
$servers = DBA::select('gserver', ['url'], ['relay-subscribe' => true, 'relay-scope' => 'tags', 'id' => $tagserverlist]);
while ($server = DBA::fetch($servers)) {
$serverlist[$server['url']] = $server['url'];
}
DBA::close($servers);
}
}
// Now we are collecting all relay contacts
foreach ($serverlist as $server_url) {
// We don't send messages to ourselves
if (Strings::compareLink($server_url, DI::baseUrl())) {
continue;
}
$contact = self::getRelayContact($server_url);
if (is_bool($contact)) {
continue;
}
$exists = false;
foreach ($contacts as $entry) {
if ($entry['batch'] == $contact['batch']) {
$exists = true;
}
}
if (!$exists) {
$contacts[] = $contact;
}
}
return $contacts;
}
/**
* Return a contact for a given server address or creates a dummy entry
*
* @param string $server_url The url of the server
* @param array $fields Fieldlist
* @return array with the contact
* @throws \Exception
*/
private static function getRelayContact(string $server_url, array $fields = ['batch', 'id', 'url', 'name', 'network', 'protocol', 'archive', 'blocked'])
{
// Fetch the relay contact
$condition = ['uid' => 0, 'nurl' => Strings::normaliseLink($server_url),
'contact-type' => Contact::TYPE_RELAY];
$contact = DBA::selectFirst('contact', $fields, $condition);
if (DBA::isResult($contact)) {
if ($contact['archive'] || $contact['blocked']) {
return false;
}
return $contact;
} else {
self::setRelayContact($server_url);
$contact = DBA::selectFirst('contact', $fields, $condition);
if (DBA::isResult($contact)) {
return $contact;
}
}
// It should never happen that we arrive here
return [];
}
/**
* Update or insert a relay contact
*
* @param string $server_url The url of the server
* @param array $network_fields Optional network specific fields
* @throws \Exception
*/
public static function setRelayContact($server_url, array $network_fields = [])
{
$fields = ['created' => DateTimeFormat::utcNow(),
'name' => 'relay', 'nick' => 'relay', 'url' => $server_url,
'nurl' => Strings::normaliseLink($server_url),
'network' => Protocol::DIASPORA, 'uid' => 0,
'batch' => $server_url . '/receive/public',
'rel' => Contact::FOLLOWER, 'blocked' => false,
'pending' => false, 'writable' => true,
'baseurl' => $server_url, 'contact-type' => Contact::TYPE_RELAY];
$fields = array_merge($fields, $network_fields);
$condition = ['uid' => 0, 'nurl' => Strings::normaliseLink($server_url)];
$old = DBA::selectFirst('contact', [], $condition);
if (DBA::isResult($old)) {
unset($fields['created']);
$condition = ['id' => $old['id']];
Logger::info('Update relay contact', ['fields' => $fields, 'condition' => $condition]);
DBA::update('contact', $fields, $condition, $old);
} else {
Logger::info('Create relay contact', ['fields' => $fields]);
Contact::insert($fields);
}
}
/**
* Return a list of participating contacts for a thread
*

218
src/Protocol/Relay.php
View File

@ -23,9 +23,17 @@ namespace Friendica\Protocol;
use Friendica\Content\Text\BBCode;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\APContact;
use Friendica\Model\Contact;
use Friendica\Model\GServer;
use Friendica\Model\Item;
use Friendica\Model\Search;
use Friendica\Model\Tag;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Strings;
/**
* Base class for relay handling
@ -124,4 +132,214 @@ class Relay
Logger::info('No matching hashtags found - rejected', ['network' => $network, 'url' => $url]);
return false;
}
/**
* Update or insert a relay contact
*
* @param array $gserver Global server record
* @param array $fields Optional network specific fields
* @throws \Exception
*/
public static function updateContact(array $gserver, array $fields = [])
{
if (in_array($gserver['network'], [Protocol::ACTIVITYPUB, Protocol::DFRN])) {
$system = APContact::getByURL($gserver['url'] . '/friendica');
if (!empty($system['sharedinbox'])) {
Logger::info('Sucessfully probed for relay contact', ['server' => $gserver['url']]);
$id = Contact::updateFromProbeByURL($system['url']);
Logger::info('Updated relay contact', ['server' => $gserver['url'], 'id' => $id]);
return;
}
}
$condition = ['uid' => 0, 'gsid' => $gserver['id'], 'contact-type' => Contact::TYPE_RELAY];
$old = DBA::selectFirst('contact', [], $condition);
if (!DBA::isResult($old)) {
$condition = ['uid' => 0, 'nurl' => Strings::normaliseLink($gserver['url'])];
$old = DBA::selectFirst('contact', [], $condition);
if (DBA::isResult($old)) {
$fields['gsid'] = $gserver['id'];
$fields['contact-type'] = Contact::TYPE_RELAY;
Logger::info('Assigning missing data for relay contact', ['server' => $gserver['url'], 'id' => $old['id']]);
}
} elseif (empty($fields)) {
Logger::info('No content to update, quitting', ['server' => $gserver['url']]);
return;
}
if (DBA::isResult($old)) {
$fields['updated'] = DateTimeFormat::utcNow();
Logger::info('Update relay contact', ['server' => $gserver['url'], 'id' => $old['id'], 'fields' => $fields]);
DBA::update('contact', $fields, ['id' => $old['id']], $old);
} else {
$default = ['created' => DateTimeFormat::utcNow(),
'name' => 'relay', 'nick' => 'relay', 'url' => $gserver['url'],
'nurl' => Strings::normaliseLink($gserver['url']),
'network' => Protocol::DIASPORA, 'uid' => 0,
'batch' => $gserver['url'] . '/receive/public',
'rel' => Contact::FOLLOWER, 'blocked' => false,
'pending' => false, 'writable' => true,
'gsid' => $gserver['id'],
'baseurl' => $gserver['url'], 'contact-type' => Contact::TYPE_RELAY];
$fields = array_merge($default, $fields);
Logger::info('Create relay contact', ['server' => $gserver['url'], 'fields' => $fields]);
Contact::insert($fields);
}
}
/**
* Mark the relay contact of the given contact for archival
* This is called whenever there is a communication issue with the server.
* It avoids sending stuff to servers who don't exist anymore.
* The relay contact is a technical contact entry that exists once per server.
*
* @param array $contact of the relay contact
*/
public static function markForArchival(array $contact)
{
if (!empty($contact['contact-type']) && ($contact['contact-type'] == Contact::TYPE_RELAY)) {
// This is already the relay contact, we don't need to fetch it
$relay_contact = $contact;
} elseif (empty($contact['baseurl'])) {
if (!empty($contact['batch'])) {
$condition = ['uid' => 0, 'network' => Protocol::FEDERATED, 'batch' => $contact['batch'], 'contact-type' => Contact::TYPE_RELAY];
$relay_contact = DBA::selectFirst('contact', [], $condition);
} else {
return;
}
} else {
$gserver = ['id' => $contact['gsid'] ?: GServer::getID($contact['baseurl'], true),
'url' => $contact['baseurl'], 'network' => $contact['network']];
$relay_contact = self::getContact($gserver, []);
}
if (!empty($relay_contact)) {
Logger::info('Relay contact will be marked for archival', ['id' => $relay_contact['id'], 'url' => $relay_contact['url']]);
Contact::markForArchival($relay_contact);
}
}
/**
* Return a list of relay servers
*
* The list contains not only the official relays but also servers that we serve directly
*
* @param integer $item_id id of the item that is sent
* @param array $contacts Previously fetched contacts
* @param array $networks Networks of the relay servers
*
* @return array of relay servers
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
public static function getList(int $item_id, array $contacts, array $networks)
{
$serverlist = [];
// Fetching relay servers
$serverdata = DI::config()->get("system", "relay_server");
if (!empty($serverdata)) {
$servers = explode(",", $serverdata);
foreach ($servers as $server) {
$gserver = DBA::selectFirst('gserver', ['id', 'url', 'network'], ['nurl' => Strings::normaliseLink($server)]);
if (DBA::isResult($gserver)) {
$serverlist[$gserver['id']] = $gserver;
}
}
}
if (DI::config()->get("system", "relay_directly", false)) {
// We distribute our stuff based on the parent to ensure that the thread will be complete
$parent = Item::selectFirst(['uri-id'], ['id' => $item_id]);
if (!DBA::isResult($parent)) {
return;
}
// Servers that want to get all content
$servers = DBA::select('gserver', ['id', 'url', 'network'], ['relay-subscribe' => true, 'relay-scope' => 'all']);
while ($server = DBA::fetch($servers)) {
$serverlist[$server['id']] = $server;
}
DBA::close($servers);
// All tags of the current post
$tags = DBA::select('tag-view', ['name'], ['uri-id' => $parent['uri-id'], 'type' => Tag::HASHTAG]);
$taglist = [];
while ($tag = DBA::fetch($tags)) {
$taglist[] = $tag['name'];
}
DBA::close($tags);
// All servers who wants content with this tag
$tagserverlist = [];
if (!empty($taglist)) {
$tagserver = DBA::select('gserver-tag', ['gserver-id'], ['tag' => $taglist]);
while ($server = DBA::fetch($tagserver)) {
$tagserverlist[] = $server['gserver-id'];
}
DBA::close($tagserver);
}
// All adresses with the given id
if (!empty($tagserverlist)) {
$servers = DBA::select('gserver', ['id', 'url', 'network'], ['relay-subscribe' => true, 'relay-scope' => 'tags', 'id' => $tagserverlist]);
while ($server = DBA::fetch($servers)) {
$serverlist[$server['id']] = $server;
}
DBA::close($servers);
}
}
// Now we are collecting all relay contacts
foreach ($serverlist as $gserver) {
// We don't send messages to ourselves
if (Strings::compareLink($gserver['url'], DI::baseUrl())) {
continue;
}
$contact = self::getContact($gserver);
if (empty($contact)) {
continue;
}
if (in_array($contact['network'], $networks) && !in_array($contact['batch'], array_column($contacts, 'batch'))) {
$contacts[] = $contact;
}
}
return $contacts;
}
/**
* Return a contact for a given server address or creates a dummy entry
*
* @param array $gserver Global server record
* @param array $fields Fieldlist
* @return array with the contact
* @throws \Exception
*/
private static function getContact(array $gserver, array $fields = ['batch', 'id', 'url', 'name', 'network', 'protocol', 'archive', 'blocked'])
{
// Fetch the relay contact
$condition = ['uid' => 0, 'gsid' => $gserver['id'], 'contact-type' => Contact::TYPE_RELAY];
$contact = DBA::selectFirst('contact', $fields, $condition);
if (DBA::isResult($contact)) {
if ($contact['archive'] || $contact['blocked']) {
return false;
}
return $contact;
} else {
self::updateContact($gserver);
$contact = DBA::selectFirst('contact', $fields, $condition);
if (DBA::isResult($contact)) {
return $contact;
}
}
// It should never happen that we arrive here
return [];
}
}

3
src/Worker/Delivery.php
View File

@ -34,6 +34,7 @@ use Friendica\Util\Strings;
use Friendica\Util\Network;
use Friendica\Core\Worker;
use Friendica\Model\FContact;
use Friendica\Protocol\Relay;
class Delivery
{
@ -484,7 +485,7 @@ class Delivery
// When it is delivered to the public endpoint, we do mark the relay contact for archival as well
if ($public_message) {
Diaspora::markRelayForArchival($contact);
Relay::markForArchival($contact);
}
if (empty($contact['contact-type']) || ($contact['contact-type'] != Model\Contact::TYPE_RELAY)) {

21
src/Worker/Notifier.php
View File

@ -40,6 +40,7 @@ use Friendica\Protocol\Activity;
use Friendica\Protocol\ActivityPub;
use Friendica\Protocol\Diaspora;
use Friendica\Protocol\OStatus;
use Friendica\Protocol\Relay;
use Friendica\Protocol\Salmon;
/*
@ -414,12 +415,10 @@ class Notifier
$relay_list_stmt = DBA::p(
"SELECT
`batch`,
`batch`, `network`, `protocol`,
ANY_VALUE(`id`) AS `id`,
ANY_VALUE(`url`) AS `url`,
ANY_VALUE(`name`) AS `name`,
ANY_VALUE(`network`) AS `network`,
ANY_VALUE(`protocol`) AS `protocol`
ANY_VALUE(`name`) AS `name`
FROM `contact`
WHERE `network` = ?
AND `batch` != ''
@ -428,7 +427,7 @@ class Notifier
AND NOT `blocked`
AND NOT `pending`
AND NOT `archive`
GROUP BY `batch`",
GROUP BY `batch`, `network`, `protocol`",
Protocol::DIASPORA,
$owner['uid'],
Contact::SHARING
@ -442,7 +441,7 @@ class Notifier
// Add the relay to the list, avoid duplicates.
// Don't send community posts to the relay. Forum posts via the Diaspora protocol are looking ugly.
if (!$followup && !Item::isForumPost($target_item, $owner) && !self::isForumPost($target_item)) {
$relay_list = Diaspora::relayList($target_id, $relay_list);
$relay_list = Relay::getList($target_id, $relay_list, [Protocol::DFRN, Protocol::DIASPORA]);
}
}
@ -461,7 +460,7 @@ class Notifier
}
if (!empty($rr['addr']) && ($rr['network'] == Protocol::ACTIVITYPUB) && !DBA::exists('fcontact', ['addr' => $rr['addr']])) {
Logger::info('Contact is AP omly, so skip delivery via legacy DFRN/Diaspora', ['target' => $target_id, 'contact' => $rr['url']]);
Logger::info('Contact is AP only, so skip delivery via legacy DFRN/Diaspora', ['target' => $target_id, 'contact' => $rr['url']]);
continue;
}
@ -476,7 +475,7 @@ class Notifier
}
if (self::skipDFRN($rr, $target_item, $parent, $thr_parent, $owner, $cmd)) {
Logger::info('Contact can be delivered via AP, so skip delivery via legacy DFRN/Diaspora', ['id' => $target_id, 'url' => $rr['url']]);
Logger::info('Content will be delivered via AP, so skip delivery via legacy DFRN/Diaspora', ['id' => $target_id, 'url' => $rr['url']]);
continue;
}
@ -516,7 +515,7 @@ class Notifier
}
if (!empty($contact['addr']) && ($contact['network'] == Protocol::ACTIVITYPUB) && !DBA::exists('fcontact', ['addr' => $contact['addr']])) {
Logger::info('Contact is AP omly, so skip delivery via legacy DFRN/Diaspora', ['target' => $target_id, 'contact' => $contact['url']]);
Logger::info('Contact is AP only, so skip delivery via legacy DFRN/Diaspora', ['target' => $target_id, 'contact' => $contact['url']]);
continue;
}
@ -531,7 +530,7 @@ class Notifier
}
if (self::skipDFRN($contact, $target_item, $parent, $thr_parent, $owner, $cmd)) {
Logger::info('Contact can be delivered via AP, so skip delivery via legacy DFRN/Diaspora', ['target' => $target_id, 'url' => $contact['url']]);
Logger::info('Content will be delivered via AP, so skip delivery via legacy DFRN/Diaspora', ['target' => $target_id, 'url' => $contact['url']]);
continue;
}
@ -789,6 +788,7 @@ class Notifier
$inboxes = ActivityPub\Transmitter::fetchTargetInboxes($target_item, $uid);
if (in_array($target_item['private'], [Item::PUBLIC])) {
$inboxes = ActivityPub\Transmitter::addRelayServerInboxesForItem($target_item['id'], $inboxes);
$relay_inboxes = ActivityPub\Transmitter::addRelayServerInboxes();
}
@ -805,6 +805,7 @@ class Notifier
$inboxes = ActivityPub\Transmitter::fetchTargetInboxes($parent, $uid, true, $target_item['id']);
if (in_array($target_item['private'], [Item::PUBLIC])) {
$inboxes = ActivityPub\Transmitter::addRelayServerInboxesForItem($parent['id'], $inboxes);
$relay_inboxes = ActivityPub\Transmitter::addRelayServerInboxes([]);
}