From 184dcf75a75f50f02dd9efa1267ee446b4708a80 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Tue, 20 Jan 2015 22:54:25 +0100
Subject: [PATCH] Caching for scrape, keywords for remote_self, notifications
for addresses that aren't in your contact list.
---
include/Scrape.php | 11 +++++
include/follow.php | 23 +++++----
include/html2bbcode.php | 3 +-
include/items.php | 43 ++++++++++++++--
include/notifier.php | 107 ++++++++++++++++++++--------------------
mod/contacts.php | 14 +++---
mod/salmon.php | 14 ++----
7 files changed, 130 insertions(+), 85 deletions(-)
diff --git a/include/Scrape.php b/include/Scrape.php
index 629f7a063b..fa2d7944a3 100644
--- a/include/Scrape.php
+++ b/include/Scrape.php
@@ -343,6 +343,12 @@ function probe_url($url, $mode = PROBE_NORMAL) {
if(! $url)
return $result;
+ $result = Cache::get("probe_url:".$mode.":".$url);
+ if (!is_null($result)) {
+ $result = unserialize($result);
+ return $result;
+ }
+
$network = null;
$diaspora = false;
$diaspora_base = '';
@@ -401,6 +407,9 @@ function probe_url($url, $mode = PROBE_NORMAL) {
$pubkey = $diaspora_key;
$diaspora = true;
}
+ if($link['@attributes']['rel'] === 'http://ostatus.org/schema/1.0/subscribe') {
+ $diaspora = false;
+ }
}
// Status.Net can have more than one profile URL. We need to match the profile URL
@@ -759,5 +768,7 @@ function probe_url($url, $mode = PROBE_NORMAL) {
$result = $result2;
}
+ Cache::set("probe_url:".$mode.":".$url,serialize($result));
+
return $result;
}
diff --git a/include/follow.php b/include/follow.php
index 285f864b94..ba036cd48d 100644
--- a/include/follow.php
+++ b/include/follow.php
@@ -37,7 +37,7 @@ function new_contact($uid,$url,$interactive = false) {
call_hooks('follow', $arr);
- if(x($arr['contact'],'name'))
+ if(x($arr['contact'],'name'))
$ret = $arr['contact'];
else
$ret = probe_url($url);
@@ -73,7 +73,7 @@ function new_contact($uid,$url,$interactive = false) {
// do we have enough information?
-
+
if(! ((x($ret,'name')) && (x($ret,'poll')) && ((x($ret,'url')) || (x($ret,'addr'))))) {
$result['message'] .= t('The profile address specified does not provide adequate information.') . EOL;
if(! x($ret,'poll'))
@@ -120,12 +120,12 @@ function new_contact($uid,$url,$interactive = false) {
// the poll url is more reliable than the profile url, as we may have
// indirect links or webfinger links
- $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `poll` = '%s' LIMIT 1",
+ $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `poll` = '%s' AND `network` = '%s' LIMIT 1",
intval($uid),
- dbesc($ret['poll'])
+ dbesc($ret['poll']),
+ dbesc($ret['network'])
);
-
if(count($r)) {
// update contact
if($r[0]['rel'] == CONTACT_IS_FOLLOWER || ($network === NETWORK_DIASPORA && $r[0]['rel'] == CONTACT_IS_SHARING)) {
@@ -169,10 +169,10 @@ function new_contact($uid,$url,$interactive = false) {
if($ret['network'] === NETWORK_DIASPORA)
$new_relation = CONTACT_IS_FOLLOWER;
- // create contact record
- $r = q("INSERT INTO `contact` ( `uid`, `created`, `url`, `nurl`, `addr`, `alias`, `batch`, `notify`, `poll`, `poco`, `name`, `nick`, `photo`, `network`, `pubkey`, `rel`, `priority`,
+ // create contact record
+ $r = q("INSERT INTO `contact` ( `uid`, `created`, `url`, `nurl`, `addr`, `alias`, `batch`, `notify`, `poll`, `poco`, `name`, `nick`, `network`, `pubkey`, `rel`, `priority`,
`writable`, `hidden`, `blocked`, `readonly`, `pending`, `subhub` )
- VALUES ( %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, %d, %d, 0, 0, 0, %d ) ",
+ VALUES ( %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, %d, %d, 0, 0, 0, %d ) ",
intval($uid),
dbesc(datetime_convert()),
dbesc($ret['url']),
@@ -185,7 +185,6 @@ function new_contact($uid,$url,$interactive = false) {
dbesc($ret['poco']),
dbesc($ret['name']),
dbesc($ret['nick']),
- dbesc($ret['photo']),
dbesc($ret['network']),
dbesc($ret['pubkey']),
intval($new_relation),
@@ -196,8 +195,9 @@ function new_contact($uid,$url,$interactive = false) {
);
}
- $r = q("SELECT * FROM `contact` WHERE `url` = '%s' AND `uid` = %d LIMIT 1",
+ $r = q("SELECT * FROM `contact` WHERE `url` = '%s' AND `network` = '%s' AND `uid` = %d LIMIT 1",
dbesc($ret['url']),
+ dbesc($ret['network']),
intval($uid)
);
@@ -228,8 +228,7 @@ function new_contact($uid,$url,$interactive = false) {
`name-date` = '%s',
`uri-date` = '%s',
`avatar-date` = '%s'
- WHERE `id` = %d
- ",
+ WHERE `id` = %d",
dbesc($photos[0]),
dbesc($photos[1]),
dbesc($photos[2]),
diff --git a/include/html2bbcode.php b/include/html2bbcode.php
index 1b940d9497..650bbdcae8 100644
--- a/include/html2bbcode.php
+++ b/include/html2bbcode.php
@@ -76,13 +76,14 @@ function node2bbcodesub(&$doc, $oldnode, $attributes, $startbb, $endbb)
return($replace);
}
+if(!function_exists('deletenode')) {
function deletenode(&$doc, $node)
{
$xpath = new DomXPath($doc);
$list = $xpath->query("//".$node);
foreach ($list as $child)
$child->parentNode->removeChild($child);
-}
+}}
function html2bbcode($message)
{
diff --git a/include/items.php b/include/items.php
index dc4b231f28..77337fe0d9 100644
--- a/include/items.php
+++ b/include/items.php
@@ -882,6 +882,7 @@ function get_atom_elements($feed, $item, $contact = array()) {
$preview = $attachment->link;
$res["body"] = $res["title"].add_page_info($res['plink'], false, $preview, ($contact['fetch_further_information'] == 2), $contact['ffi_keyword_blacklist']);
+ $res["tag"] = add_page_keywords($res['plink'], false, $preview, ($contact['fetch_further_information'] == 2), $contact['ffi_keyword_blacklist']);
$res["title"] = "";
$res["object-type"] = ACTIVITY_OBJ_BOOKMARK;
unset($res["attach"]);
@@ -946,7 +947,7 @@ function add_page_info_data($data) {
return("\n[class=type-".$data["type"]."]".$text."[/class]".$hashtags);
}
-function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") {
+function query_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") {
require_once("mod/parse_url.php");
$data = Cache::get("parse_url:".$url);
@@ -959,7 +960,7 @@ function add_page_info($url, $no_photos = false, $photo = "", $keywords = false,
if ($photo != "")
$data["images"][0]["src"] = $photo;
- logger('add_page_info: fetch page info for '.$url.' '.print_r($data, true), LOGGER_DEBUG);
+ logger('fetch page info for '.$url.' '.print_r($data, true), LOGGER_DEBUG);
if (!$keywords AND isset($data["keywords"]))
unset($data["keywords"]);
@@ -974,6 +975,32 @@ function add_page_info($url, $no_photos = false, $photo = "", $keywords = false,
}
}
+ return($data);
+}
+
+function add_page_keywords($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") {
+ $data = query_page_info($url, $no_photos, $photo, $keywords, $keyword_blacklist);
+
+ $tags = "";
+ if (isset($data["keywords"]) AND count($data["keywords"])) {
+ $a = get_app();
+ foreach ($data["keywords"] AS $keyword) {
+ $hashtag = str_replace(array(" ", "+", "/", ".", "#", "'"),
+ array("","", "", "", "", ""), $keyword);
+
+ if ($tags != "")
+ $tags .= ",";
+
+ $tags .= "#[url=".$a->get_baseurl()."/search?tag=".rawurlencode($hashtag)."]".$hashtag."[/url]";
+ }
+ }
+
+ return($tags);
+}
+
+function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") {
+ $data = query_page_info($url, $no_photos, $photo, $keywords, $keyword_blacklist);
+
$text = add_page_info_data($data);
return($text);
@@ -4071,6 +4098,7 @@ function atom_entry($item,$type,$author,$owner,$comment = false,$cid = 0) {
else
$body = $item['body'];
+
$o = "\r\n\r\n<entry>\r\n";
if(is_array($author))
@@ -4085,13 +4113,22 @@ function atom_entry($item,$type,$author,$owner,$comment = false,$cid = 0) {
$o .= '<thr:in-reply-to ref="' . xmlify($parent_item) . '" type="text/html" href="' . xmlify($a->get_baseurl() . '/display/' . $owner['nickname'] . '/' . $item['parent']) . '" />' . "\r\n";
}
+ $htmlbody = $body;
+
+ if ($item['title'] != "")
+ $htmlbody = "[b]".$item['title']."[/b]\n\n".$htmlbody;
+
+ $htmlbody = bbcode(bb_remove_share_information($htmlbody), false, false, 7);
+
$o .= '<id>' . xmlify($item['uri']) . '</id>' . "\r\n";
$o .= '<title>' . xmlify($item['title']) . '</title>' . "\r\n";
$o .= '<published>' . xmlify(datetime_convert('UTC','UTC',$item['created'] . '+00:00',ATOM_TIME)) . '</published>' . "\r\n";
$o .= '<updated>' . xmlify(datetime_convert('UTC','UTC',$item['edited'] . '+00:00',ATOM_TIME)) . '</updated>' . "\r\n";
$o .= '<dfrn:env>' . base64url_encode($body, true) . '</dfrn:env>' . "\r\n";
- $o .= '<content type="' . $type . '" >' . xmlify((($type === 'html') ? bbcode($body) : $body)) . '</content>' . "\r\n";
+ $o .= '<content type="' . $type . '" >' . xmlify((($type === 'html') ? $htmlbody : $body)) . '</content>' . "\r\n";
$o .= '<link rel="alternate" type="text/html" href="' . xmlify($a->get_baseurl() . '/display/' . $owner['nickname'] . '/' . $item['id']) . '" />' . "\r\n";
+
+
if($comment)
$o .= '<dfrn:comment-allow>' . intval($item['last-child']) . '</dfrn:comment-allow>' . "\r\n";
diff --git a/include/notifier.php b/include/notifier.php
index cfd85593fb..067251b429 100644
--- a/include/notifier.php
+++ b/include/notifier.php
@@ -55,7 +55,7 @@ function notifier_run(&$argv, &$argc){
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
- unset($db_host, $db_user, $db_pass, $db_data);
+ unset($db_host, $db_user, $db_pass, $db_data);
}
require_once("include/session.php");
@@ -110,10 +110,11 @@ function notifier_run(&$argv, &$argc){
$recipients[] = $message[0]['contact-id'];
$item = $message[0];
- } elseif($cmd === 'expire') {
+ }
+ elseif($cmd === 'expire') {
$normal_mode = false;
$expire = true;
- $items = q("SELECT * FROM `item` WHERE `uid` = %d AND `wall` = 1
+ $items = q("SELECT * FROM `item` WHERE `uid` = %d AND `wall` = 1
AND `deleted` = 1 AND `changed` > UTC_TIMESTAMP() - INTERVAL 10 MINUTE",
intval($item_id)
);
@@ -121,7 +122,8 @@ function notifier_run(&$argv, &$argc){
$item_id = 0;
if(! count($items))
return;
- } elseif($cmd === 'suggest') {
+ }
+ elseif($cmd === 'suggest') {
$normal_mode = false;
$fsuggest = true;
@@ -176,7 +178,7 @@ function notifier_run(&$argv, &$argc){
if(! $parent_id)
return;
- $items = q("SELECT `item`.*, `sign`.`signed_text`,`sign`.`signature`,`sign`.`signer`
+ $items = q("SELECT `item`.*, `sign`.`signed_text`,`sign`.`signature`,`sign`.`signer`
FROM `item` LEFT JOIN `sign` ON `sign`.`iid` = `item`.`id` WHERE `parent` = %d and visible = 1 and moderated = 0 ORDER BY `id` ASC",
intval($parent_id)
);
@@ -290,8 +292,28 @@ function notifier_run(&$argv, &$argc){
$followup = true;
$public_message = false; // not public
$conversant_str = dbesc($parent['contact-id']);
- }
- else {
+ $recipients = array($parent['contact-id']);
+
+ if ($parent['network'] == NETWORK_OSTATUS) {
+
+ // Check if the recipient isn't in your contact list
+ $r = q("SELECT `url` FROM `contact` WHERE `id` = %d", $parent['contact-id']);
+ if (count($r)) {
+ $url_recipients = array();
+
+ $thrparent = q("SELECT `author-link` FROM `item` WHERE `uri` = '%s'", dbesc($target_item["thr-parent"]));
+ if (count($thrparent) AND (normalise_link($r[0]["url"]) != normalise_link($thrparent[0]["author-link"]))) {
+ require_once("include/Scrape.php");
+ $probed_contact = probe_url($thrparent[0]["author-link"]);
+ if ($probed_contact["notify"] != "") {
+ logger('scrape data for slapper: '.print_r($probed_contact, true));
+ $url_recipients[$probed_contact["notify"]] = $probed_contact["notify"];
+ }
+ }
+ }
+ logger("url_recipients".print_r($url_recipients,true));
+ }
+ } else {
$followup = false;
// don't send deletions onward for other people's stuff
@@ -437,8 +459,8 @@ function notifier_run(&$argv, &$argc){
set_config('system','site_pubkey', $res['pubkey']);
}
- $rp = q("SELECT `resource-id` , `scale`, type FROM `photo`
- WHERE `profile` = 1 AND `uid` = %d ORDER BY scale;", $uid);
+ $rp = q("SELECT `resource-id` , `scale`, type FROM `photo`
+ WHERE `profile` = 1 AND `uid` = %d ORDER BY scale;", $uid);
$photos = array();
$ext = Photo::supportedTypes();
foreach($rp as $p){
@@ -446,19 +468,19 @@ function notifier_run(&$argv, &$argc){
}
unset($rp, $ext);
- $atom .= replace_macros($sugg_template, array(
- '$name' => xmlify($owner['name']),
- '$photo' => xmlify($photos[4]),
- '$thumb' => xmlify($photos[5]),
- '$micro' => xmlify($photos[6]),
- '$url' => xmlify($owner['url']),
- '$request' => xmlify($owner['request']),
- '$confirm' => xmlify($owner['confirm']),
- '$notify' => xmlify($owner['notify']),
- '$poll' => xmlify($owner['poll']),
- '$sitepubkey' => xmlify(get_config('system','site_pubkey')),
- //'$pubkey' => xmlify($owner['pubkey']),
- //'$prvkey' => xmlify($owner['prvkey']),
+ $atom .= replace_macros($sugg_template, array(
+ '$name' => xmlify($owner['name']),
+ '$photo' => xmlify($photos[4]),
+ '$thumb' => xmlify($photos[5]),
+ '$micro' => xmlify($photos[6]),
+ '$url' => xmlify($owner['url']),
+ '$request' => xmlify($owner['request']),
+ '$confirm' => xmlify($owner['confirm']),
+ '$notify' => xmlify($owner['notify']),
+ '$poll' => xmlify($owner['poll']),
+ '$sitepubkey' => xmlify(get_config('system','site_pubkey')),
+ //'$pubkey' => xmlify($owner['pubkey']),
+ //'$prvkey' => xmlify($owner['prvkey']),
));
$recipients_relocate = q("SELECT * FROM contact WHERE uid = %d AND self = 0 AND network = '%s'" , intval($uid), NETWORK_DFRN);
unset($photos);
@@ -497,8 +519,7 @@ function notifier_run(&$argv, &$argc){
if($item_id == $item['id'] || $item['id'] == $item['parent'])
$atom .= atom_entry($item,'text',null,$owner,true);
- }
- else
+ } else
$atom .= atom_entry($item,'text',null,$owner,true);
if(($top_level) && ($public_message) && ($item['author-link'] === $item['owner-link']) && (! $expire))
@@ -517,8 +538,8 @@ function notifier_run(&$argv, &$argc){
$mail_disabled = ((function_exists('imap_open') && (! get_config('system','imap_disabled'))) ? 0 : 1);
if(! $mail_disabled) {
- if((! strlen($target_item['allow_cid'])) && (! strlen($target_item['allow_gid']))
- && (! strlen($target_item['deny_cid'])) && (! strlen($target_item['deny_gid']))
+ if((! strlen($target_item['allow_cid'])) && (! strlen($target_item['allow_gid']))
+ && (! strlen($target_item['deny_cid'])) && (! strlen($target_item['deny_gid']))
&& (intval($target_item['pubmail']))) {
$r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `network` = '%s'",
intval($uid),
@@ -541,7 +562,7 @@ function notifier_run(&$argv, &$argc){
else
$r = q("SELECT * FROM `contact` WHERE `id` IN ( %s ) AND `blocked` = 0 AND `pending` = 0 ",
dbesc($recip_str)
- );
+ );
require_once('include/salmon.php');
@@ -691,33 +712,13 @@ function notifier_run(&$argv, &$argc){
if(get_config('system','ostatus_disabled') || get_config('system','dfrn_only'))
break;
- $ostatus_contact = $contact;
-
- // If the contact doesn't fit with the contact, then fetch the correct contact
- // This is more a test than a good solution. The whole OStatus stuff needs a rework.
- if ($followup) {
- $thrparent = q("SELECT `author-link` FROM `item` WHERE `uri` = '%s'", dbesc($target_item["thr-parent"]));
- if (count($thrparent) AND (normalise_link($contact["url"]) != normalise_link($thrparent[0]["author-link"]))) {
- require_once("include/Scrape.php");
- $probed_contact = probe_url($thrparent[0]["author-link"]);
- if ($probed_contact["network"] != NETWORK_FEED) {
- $ostatus_contact = $probed_contact;
- $ostatus_contact["nurl"] = normalise_link($probed_contact["url"]);
- $ostatus_contact["thumb"] = $probed_contact["photo"];
- $ostatus_contact["micro"] = $probed_contact["photo"];
- }
- logger('scrape data for slapper: '.print_r($ostatus_contact, true));
- }
- }
-
- if($followup && $ostatus_contact['notify']) {
-
- logger('notifier: slapdelivery: ' . $ostatus_contact['name']);
- $deliver_status = slapper($owner,$ostatus_contact['notify'],$slap);
+ if($followup && $contact['notify']) {
+ logger('notifier: slapdelivery: ' . $contact['name']);
+ $deliver_status = slapper($owner,$contact['notify'],$slap);
if($deliver_status == (-1)) {
// queue message for redelivery
- add_to_queue($ostatus_contact['id'],NETWORK_OSTATUS,$slap);
+ add_to_queue($contact['id'],NETWORK_OSTATUS,$slap);
}
} else {
@@ -1012,6 +1013,6 @@ function notifier_run(&$argv, &$argc){
if (array_search(__file__,get_included_files())===0){
- notifier_run($_SERVER["argv"],$_SERVER["argc"]);
- killme();
+ notifier_run($_SERVER["argv"],$_SERVER["argc"]);
+ killme();
}
diff --git a/mod/contacts.php b/mod/contacts.php
index fbab84f562..047dc1ea90 100644
--- a/mod/contacts.php
+++ b/mod/contacts.php
@@ -407,19 +407,19 @@ function contacts_content(&$a) {
$url = "redir/{$contact['id']}";
$sparkle = ' class="sparkle" ';
}
- else {
+ else {
$url = $contact['url'];
$sparkle = '';
}
$insecure = t('Private communications are not available for this contact.');
- $last_update = (($contact['last-update'] == '0000-00-00 00:00:00')
- ? t('Never')
+ $last_update = (($contact['last-update'] == '0000-00-00 00:00:00')
+ ? t('Never')
: datetime_convert('UTC',date_default_timezone_get(),$contact['last-update'],'D, j M Y, g:i A'));
if($contact['last-update'] !== '0000-00-00 00:00:00')
- $last_update .= ' ' . (($contact['last-update'] == $contact['success_update']) ? t("\x28Update was successful\x29") : t("\x28Update was not successful\x29"));
+ $last_update .= ' ' . (($contact['last-update'] <= $contact['success_update']) ? t("\x28Update was successful\x29") : t("\x28Update was not successful\x29"));
$lblsuggest = (($contact['network'] === NETWORK_DFRN) ? t('Suggest friends') : '');
@@ -430,7 +430,7 @@ function contacts_content(&$a) {
$common = count_common_friends(local_user(),$contact['id']);
$common_text = (($common) ? sprintf( tt('%d contact in common','%d contacts in common', $common),$common) : '');
- $polling = (($contact['network'] === NETWORK_MAIL | $contact['network'] === NETWORK_FEED) ? 'polling' : '');
+ $polling = (($contact['network'] === NETWORK_MAIL | $contact['network'] === NETWORK_FEED) ? 'polling' : '');
$x = count_all_friends(local_user(), $contact['id']);
$all_friends = (($x) ? t('View all contacts') : '');
@@ -668,7 +668,7 @@ function contacts_content(&$a) {
$url = "redir/{$rr['id']}";
$sparkle = ' class="sparkle" ';
}
- else {
+ else {
$url = $rr['url'];
$sparkle = '';
}
@@ -681,7 +681,7 @@ function contacts_content(&$a) {
'id' => $rr['id'],
'alt_text' => $alt_text,
'dir_icon' => $dir_icon,
- 'thumb' => $rr['thumb'],
+ 'thumb' => $rr['thumb'],
'name' => $rr['name'],
'username' => $rr['name'],
'sparkle' => $sparkle,
diff --git a/mod/salmon.php b/mod/salmon.php
index 5df1e1c9e5..11e42d9436 100644
--- a/mod/salmon.php
+++ b/mod/salmon.php
@@ -15,7 +15,7 @@ function salmon_return($val) {
if($val >= 200 && $val < 300)
$err = 'OK';
- logger('mod-salmon returns ' . $val);
+ logger('mod-salmon returns ' . $val);
header($_SERVER["SERVER_PROTOCOL"] . ' ' . $val . ' ' . $err);
killme();
@@ -50,7 +50,7 @@ function salmon_post(&$a) {
$base = $dom->env;
elseif($dom->data)
$base = $dom;
-
+
if(! $base) {
logger('mod-salmon: unable to locate salmon data in xml ');
http_status_exit(400);
@@ -92,7 +92,7 @@ function salmon_post(&$a) {
// Create a fake feed wrapper so simplepie doesn't choke
$tpl = get_markup_template('fake_feed.tpl');
-
+
$base = substr($data,strpos($data,'<entry'));
$feedxml = $tpl . $base . '</feed>';
@@ -100,7 +100,7 @@ function salmon_post(&$a) {
logger('mod-salmon: Processed feed: ' . $feedxml);
// Now parse it like a normal atom feed to scrape out the author URI
-
+
$feed = new SimplePie();
$feed->set_raw_data($feedxml);
$feed->enable_order_by_date(false);
@@ -192,7 +192,7 @@ function salmon_post(&$a) {
);
}
}
- }
+ }
// is this a follower? Or have we ignored the person?
// If so we can not accept this post.
@@ -223,7 +223,3 @@ function salmon_post(&$a) {
http_status_exit(200);
}
-
-
-
-