Merge remote-tracking branch 'upstream/2022.09-rc' into duplicates
This commit is contained in:
Michael
commit
36668dfdb1
File diff suppressed because it is too large
Load Diff
|
|
@ -639,6 +639,7 @@ CREATE TABLE IF NOT EXISTS `fcontact` (
|
|||
`network` char(4) NOT NULL DEFAULT '' COMMENT '',
|
||||
`alias` varbinary(383) NOT NULL DEFAULT '' COMMENT '',
|
||||
`pubkey` text COMMENT '',
|
||||
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
|
||||
`updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
|
||||
`interacting_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts this contact interactes with',
|
||||
`interacted_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts that interacted with this contact',
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ Fields
|
|||
| network | | char(4) | NO | | | |
|
||||
| alias | | varbinary(383) | NO | | | |
|
||||
| pubkey | | text | YES | | NULL | |
|
||||
| created | | datetime | NO | | 0001-01-01 00:00:00 | |
|
||||
| updated | | datetime | NO | | 0001-01-01 00:00:00 | |
|
||||
| interacting_count | Number of contacts this contact interactes with | int unsigned | YES | | 0 | |
|
||||
| interacted_count | Number of contacts that interacted with this contact | int unsigned | YES | | 0 | |
|
||||
|
|
|
|||
|
|
@ -586,7 +586,7 @@ class App
|
|||
$this->profiler->set(microtime(true), 'classinit');
|
||||
|
||||
$moduleName = $this->args->getModuleName();
|
||||
$page->setLogging($this->args->getCommand(), $this->args->getMethod());
|
||||
$page->setLogging($this->args->getMethod(), $this->args->getModuleName(), $this->args->getCommand());
|
||||
|
||||
try {
|
||||
// Missing DB connection: ERROR
|
||||
|
|
|
|||
|
|
@ -80,8 +80,9 @@ class Page implements ArrayAccess
|
|||
private $basePath;
|
||||
|
||||
private $timestamp = 0;
|
||||
private $command = '';
|
||||
private $method = '';
|
||||
private $module = '';
|
||||
private $command = '';
|
||||
|
||||
/**
|
||||
* @param string $basepath The Page basepath
|
||||
|
|
@ -92,21 +93,25 @@ class Page implements ArrayAccess
|
|||
$this->basePath = $basepath;
|
||||
}
|
||||
|
||||
public function setLogging(string $command, string $method)
|
||||
public function setLogging(string $method, string $module, string $command)
|
||||
{
|
||||
$this->command = $command;
|
||||
$this->method = $method;
|
||||
$this->module = $module;
|
||||
$this->command = $command;
|
||||
}
|
||||
|
||||
public function logRuntime(IManageConfigValues $config, string $origin = '')
|
||||
{
|
||||
if (in_array($this->command, $config->get('system', 'runtime_ignore'))) {
|
||||
$ignore = $config->get('system', 'runtime_ignore');
|
||||
if (in_array($this->module, $ignore) || in_array($this->command, $ignore)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$runtime = number_format(microtime(true) - $this->timestamp, 3);
|
||||
$signature = !empty($_SERVER['HTTP_SIGNATURE']);
|
||||
$load = number_format(System::currentLoad(), 2);
|
||||
$runtime = number_format(microtime(true) - $this->timestamp, 3);
|
||||
if ($runtime > $config->get('system', 'runtime_loglimit')) {
|
||||
Logger::debug('Runtime', ['method' => $this->method, 'command' => $this->command, 'runtime' => $runtime, 'origin' => $origin]);
|
||||
Logger::debug('Runtime', ['method' => $this->method, 'module' => $this->module, 'runtime' => $runtime, 'load' => $load, 'origin' => $origin, 'signature' => $signature, 'request' => $_SERVER['REQUEST_URI'] ?? '']);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ class Contact
|
|||
// Add internal fields
|
||||
$removal = [];
|
||||
if (!empty($fields)) {
|
||||
foreach (['id', 'avatar', 'created', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) {
|
||||
foreach (['id', 'next-update', 'network'] as $internal) {
|
||||
if (!in_array($internal, $fields)) {
|
||||
$fields[] = $internal;
|
||||
$removal[] = $internal;
|
||||
|
|
@ -307,9 +307,8 @@ class Contact
|
|||
}
|
||||
|
||||
// Update the contact in the background if needed
|
||||
$updated = max($contact['success_update'], $contact['created'], $contact['updated'], $contact['last-update'], $contact['failure_update']);
|
||||
if (($updated < DateTimeFormat::utc('now -7 days')) && in_array($contact['network'], Protocol::FEDERATED) && !self::isLocalById($contact['id'])) {
|
||||
Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']);
|
||||
if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
|
||||
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
|
||||
}
|
||||
|
||||
// Remove the internal fields
|
||||
|
|
@ -962,7 +961,6 @@ class Contact
|
|||
|
||||
DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'followers:' . $uid);
|
||||
DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'following:' . $uid);
|
||||
DI::cache()->delete(NoScrape::CACHEKEY . $uid);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1221,11 +1219,15 @@ class Contact
|
|||
return 0;
|
||||
}
|
||||
|
||||
$contact = self::getByURL($url, false, ['id', 'network', 'uri-id'], $uid);
|
||||
$contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update'], $uid);
|
||||
|
||||
if (!empty($contact)) {
|
||||
$contact_id = $contact['id'];
|
||||
|
||||
if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
|
||||
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
|
||||
}
|
||||
|
||||
if (empty($update) && (!empty($contact['uri-id']) || is_bool($update))) {
|
||||
Logger::debug('Contact found', ['url' => $url, 'uid' => $uid, 'update' => $update, 'cid' => $contact_id]);
|
||||
return $contact_id;
|
||||
|
|
@ -2524,7 +2526,7 @@ class Contact
|
|||
|
||||
$has_local_data = self::hasLocalData($id, $contact);
|
||||
|
||||
if (!in_array($ret['network'], array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM]))) {
|
||||
if (!Probe::isProbable($ret['network'])) {
|
||||
// Periodical checks are only done on federated contacts
|
||||
$failed_next_update = null;
|
||||
$success_next_update = null;
|
||||
|
|
@ -3382,12 +3384,12 @@ class Contact
|
|||
if (empty($url) || !is_string($url)) {
|
||||
continue;
|
||||
}
|
||||
$contact = self::getByURL($url, false, ['id', 'updated']);
|
||||
$contact = self::getByURL($url, false, ['id', 'network', 'next-update']);
|
||||
if (empty($contact['id']) && Network::isValidHttpUrl($url)) {
|
||||
Worker::add(PRIORITY_LOW, 'AddContact', 0, $url);
|
||||
++$added;
|
||||
} elseif ($contact['updated'] < DateTimeFormat::utc('now -7 days')) {
|
||||
Worker::add(PRIORITY_LOW, 'UpdateContact', $contact['id']);
|
||||
} elseif (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
|
||||
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
|
||||
++$updated;
|
||||
} else {
|
||||
++$unchanged;
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ namespace Friendica\Model;
|
|||
|
||||
use Friendica\Core\Logger;
|
||||
use Friendica\Core\Protocol;
|
||||
use Friendica\Core\Worker;
|
||||
use Friendica\Database\DBA;
|
||||
use Friendica\DI;
|
||||
use Friendica\Network\Probe;
|
||||
|
|
@ -43,6 +44,7 @@ class FContact
|
|||
*/
|
||||
public static function getByURL(string $handle, $update = null): array
|
||||
{
|
||||
Logger::debug('Fetch fcontact', ['handle' => $handle, 'update' => $update]);
|
||||
$person = DBA::selectFirst('fcontact', [], ['network' => Protocol::DIASPORA, 'addr' => $handle]);
|
||||
if (!DBA::isResult($person)) {
|
||||
$urls = [$handle, str_replace('http://', 'https://', $handle), Strings::normaliseLink($handle)];
|
||||
|
|
@ -50,21 +52,17 @@ class FContact
|
|||
}
|
||||
|
||||
if (DBA::isResult($person)) {
|
||||
Logger::debug('In cache', ['person' => $person]);
|
||||
Logger::debug('In cache', ['handle' => $handle]);
|
||||
|
||||
if (is_null($update)) {
|
||||
// update record occasionally so it doesn't get stale
|
||||
$d = strtotime($person['updated'] . ' +00:00');
|
||||
if ($d < strtotime('now - 14 days')) {
|
||||
$update = true;
|
||||
}
|
||||
|
||||
if (empty($person['guid']) || empty($person['uri-id'])) {
|
||||
$update = true;
|
||||
$update = empty($person['guid']) || empty($person['uri-id']) || ($person['created'] <= DBA::NULL_DATETIME);
|
||||
if (GServer::getNextUpdateDate(true, $person['created'], $person['updated'], false) < DateTimeFormat::utcNow()) {
|
||||
Logger::debug('Start background update', ['handle' => $handle]);
|
||||
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateFContact', $handle);
|
||||
}
|
||||
}
|
||||
} elseif (is_null($update)) {
|
||||
$update = !DBA::isResult($person);
|
||||
$update = true;
|
||||
} else {
|
||||
$person = [];
|
||||
}
|
||||
|
|
@ -95,7 +93,8 @@ class FContact
|
|||
{
|
||||
$uriid = ItemURI::insert(['uri' => $arr['url'], 'guid' => $arr['guid']]);
|
||||
|
||||
$contact = Contact::getByUriId($uriid, ['id']);
|
||||
$fcontact = DBA::selectFirst('fcontact', ['created'], ['url' => $arr['url'], 'network' => $arr['network']]);
|
||||
$contact = Contact::getByUriId($uriid, ['id', 'created']);
|
||||
$apcontact = APContact::getByURL($arr['url'], false);
|
||||
if (!empty($apcontact)) {
|
||||
$interacted = $apcontact['following_count'];
|
||||
|
|
@ -129,10 +128,14 @@ class FContact
|
|||
'updated' => DateTimeFormat::utcNow(),
|
||||
];
|
||||
|
||||
$condition = ['url' => $arr['url'], 'network' => $arr['network']];
|
||||
if (empty($fcontact['created'])) {
|
||||
$fields['created'] = $fields['updated'];
|
||||
} elseif (!empty($contact['created']) && ($fcontact['created'] <= DBA::NULL_DATETIME)) {
|
||||
$fields['created'] = $contact['created'];
|
||||
}
|
||||
|
||||
$fields = DI::dbaDefinition()->truncateFieldsForTable('fcontact', $fields);
|
||||
DBA::update('fcontact', $fields, $condition, true);
|
||||
DBA::update('fcontact', $fields, ['url' => $arr['url'], 'network' => $arr['network']], true);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1870,23 +1870,40 @@ class Item
|
|||
return '';
|
||||
}
|
||||
|
||||
// Convert attachments to links
|
||||
$naked_body = BBCode::removeAttachment($item['body']);
|
||||
if (empty($naked_body)) {
|
||||
$languages = self::getLanguageArray(trim($item['title'] . "\n" . $item['body']), 3);
|
||||
if (empty($languages)) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return json_encode($languages);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a language array from a given text
|
||||
*
|
||||
* @param string $body
|
||||
* @param integer $count
|
||||
* @return array
|
||||
*/
|
||||
public static function getLanguageArray(string $body, int $count): array
|
||||
{
|
||||
// Convert attachments to links
|
||||
$naked_body = BBCode::removeAttachment($body);
|
||||
if (empty($naked_body)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Remove links and pictures
|
||||
$naked_body = BBCode::removeLinks($naked_body);
|
||||
|
||||
// Convert the title and the body to plain text
|
||||
$naked_body = trim($item['title'] . "\n" . BBCode::toPlaintext($naked_body));
|
||||
$naked_body = BBCode::toPlaintext($naked_body);
|
||||
|
||||
// Remove possibly remaining links
|
||||
$naked_body = preg_replace(Strings::autoLinkRegEx(), '', $naked_body);
|
||||
|
||||
if (empty($naked_body)) {
|
||||
return '';
|
||||
return [];
|
||||
}
|
||||
|
||||
$naked_body = self::getDominantLanguage($naked_body);
|
||||
|
|
@ -1898,12 +1915,7 @@ class Item
|
|||
$availableLanguages['fa'] = 'fa';
|
||||
|
||||
$ld = new Language(array_keys($availableLanguages));
|
||||
$languages = $ld->detect($naked_body)->limit(0, 3)->close();
|
||||
if (is_array($languages)) {
|
||||
return json_encode($languages);
|
||||
}
|
||||
|
||||
return '';
|
||||
return $ld->detect($naked_body)->limit(0, $count)->close() ?: [];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -22,11 +22,10 @@
|
|||
namespace Friendica\Module;
|
||||
|
||||
use Friendica\BaseModule;
|
||||
use Friendica\Core\Cache\Enum\Duration;
|
||||
use Friendica\Core\Protocol;
|
||||
use Friendica\Core\System;
|
||||
use Friendica\Database\DBA;
|
||||
use Friendica\DI;
|
||||
use Friendica\Model\APContact;
|
||||
use Friendica\Model\User;
|
||||
|
||||
/**
|
||||
|
|
@ -36,8 +35,6 @@ use Friendica\Model\User;
|
|||
*/
|
||||
class NoScrape extends BaseModule
|
||||
{
|
||||
const CACHEKEY = 'noscrape:';
|
||||
|
||||
protected function rawContent(array $request = [])
|
||||
{
|
||||
$a = DI::app();
|
||||
|
|
@ -58,12 +55,6 @@ class NoScrape extends BaseModule
|
|||
System::jsonError(404, 'Profile not found');
|
||||
}
|
||||
|
||||
$cachekey = self::CACHEKEY . $owner['uid'];
|
||||
$result = DI::cache()->get($cachekey);
|
||||
if (!is_null($result)) {
|
||||
System::jsonExit($result);
|
||||
}
|
||||
|
||||
$json_info = [
|
||||
'addr' => $owner['addr'],
|
||||
'nick' => $which,
|
||||
|
|
@ -98,16 +89,8 @@ class NoScrape extends BaseModule
|
|||
}
|
||||
|
||||
if (!($owner['hide-friends'] ?? false)) {
|
||||
$json_info['contacts'] = DBA::count('contact',
|
||||
[
|
||||
'uid' => $owner['uid'],
|
||||
'self' => 0,
|
||||
'blocked' => 0,
|
||||
'pending' => 0,
|
||||
'hidden' => 0,
|
||||
'archive' => 0,
|
||||
'network' => [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS]
|
||||
]);
|
||||
$apcontact = APContact::getByURL($owner['url']);
|
||||
$json_info['contacts'] = max($apcontact['following_count'], $apcontact['followers_count']);
|
||||
}
|
||||
|
||||
// We display the last activity (post or login), reduced to year and week number
|
||||
|
|
@ -135,8 +118,6 @@ class NoScrape extends BaseModule
|
|||
}
|
||||
}
|
||||
|
||||
DI::cache()->set($cachekey, $json_info, Duration::DAY);
|
||||
|
||||
System::jsonExit($json_info);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,17 @@ class Probe
|
|||
private static $baseurl;
|
||||
private static $istimeout;
|
||||
|
||||
/**
|
||||
* Checks if the provided network can be probed
|
||||
*
|
||||
* @param string $network
|
||||
* @return boolean
|
||||
*/
|
||||
public static function isProbable(string $network): bool
|
||||
{
|
||||
return (in_array($network, array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM])));
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove stuff from an URI that doesn't belong there
|
||||
*
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ class Processor
|
|||
* @param string $body
|
||||
* @return string
|
||||
*/
|
||||
protected static function normalizeMentionLinks(string $body): string
|
||||
public static function normalizeMentionLinks(string $body): string
|
||||
{
|
||||
return preg_replace('%\[url=([^\[\]]*)]([#@!])(.*?)\[/url]%ism', '$2[url=$1]$3[/url]', $body);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ use Friendica\DI;
|
|||
use Friendica\Model\APContact;
|
||||
use Friendica\Model\Contact;
|
||||
use Friendica\Model\GServer;
|
||||
use Friendica\Model\Item;
|
||||
use Friendica\Model\Post;
|
||||
use Friendica\Model\Search;
|
||||
use Friendica\Model\Tag;
|
||||
|
|
@ -76,6 +77,8 @@ class Relay
|
|||
return false;
|
||||
}
|
||||
|
||||
$body = ActivityPub\Processor::normalizeMentionLinks($body);
|
||||
|
||||
$systemTags = [];
|
||||
$userTags = [];
|
||||
$denyTags = [];
|
||||
|
|
@ -125,6 +128,25 @@ class Relay
|
|||
}
|
||||
}
|
||||
|
||||
$languages = [];
|
||||
foreach (Item::getLanguageArray($body, 10) as $language => $reliability) {
|
||||
if ($reliability > 0) {
|
||||
$languages[] = $language;
|
||||
}
|
||||
}
|
||||
|
||||
Logger::debug('Got languages', ['languages' => $languages, 'body' => $body]);
|
||||
|
||||
if (!empty($languages)) {
|
||||
if (in_array($languages[0], $config->get('system', 'relay_deny_languages'))) {
|
||||
Logger::info('Unwanted language found - rejected', ['language' => $languages[0], 'network' => $network, 'url' => $url]);
|
||||
return false;
|
||||
}
|
||||
} elseif ($config->get('system', 'relay_deny_undetected_language')) {
|
||||
Logger::info('Undetected language found - rejected', ['body' => $body, 'network' => $network, 'url' => $url]);
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($scope == self::SCOPE_ALL) {
|
||||
Logger::info('Server accept all posts - accepted', ['network' => $network, 'url' => $url]);
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -152,13 +152,13 @@ class Profiler implements ContainerInterface
|
|||
* Saves a timestamp for a value - f.e. a call
|
||||
* Necessary for profiling Friendica
|
||||
*
|
||||
* @param int $timestamp the Timestamp
|
||||
* @param float $timestamp the Timestamp
|
||||
* @param string $value A value to profile
|
||||
* @param string $callstack A callstack string, generated if absent
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function saveTimestamp(int $timestamp, string $value, string $callstack = '')
|
||||
public function saveTimestamp(float $timestamp, string $value, string $callstack = '')
|
||||
{
|
||||
if (!$this->enabled) {
|
||||
return;
|
||||
|
|
@ -358,9 +358,9 @@ class Profiler implements ContainerInterface
|
|||
* @throws NotFoundExceptionInterface No entry was found for **this** identifier.
|
||||
* @throws ContainerExceptionInterface Error while retrieving the entry.
|
||||
*
|
||||
* @return int Entry.
|
||||
* @return float Entry.
|
||||
*/
|
||||
public function get(string $id): int
|
||||
public function get(string $id): float
|
||||
{
|
||||
if (!$this->has($id)) {
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -697,6 +697,7 @@ return [
|
|||
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],
|
||||
"alias" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""],
|
||||
"pubkey" => ["type" => "text", "comment" => ""],
|
||||
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
|
||||
"updated" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
|
||||
"interacting_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts this contact interactes with"],
|
||||
"interacted_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts that interacted with this contact"],
|
||||
|
|
|
|||
|
|
@ -525,6 +525,14 @@ return [
|
|||
// The authentication password for the redis database
|
||||
'redis_password' => null,
|
||||
|
||||
// relay_deny_languages (Array)
|
||||
// Array of languages (two digit format) that are rejected.
|
||||
'relay_deny_languages' => [],
|
||||
|
||||
// relay_deny_undetected_language (Boolean)
|
||||
// Deny undetected languages
|
||||
'relay_deny_undetected_language' => false,
|
||||
|
||||
// session_handler (database|cache|native)
|
||||
// Whether to use Cache to store session data or to use PHP native session storage.
|
||||
'session_handler' => 'database',
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user