Merge pull request #3613 from annando/some-ostatus

Speed up of OStatus transmission/Ignoring "created" with remote self

include/items.php

@@ -623,11 +623,11 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
 	$arr['owner-name']    = ((x($arr, 'owner-name'))    ? trim($arr['owner-name'])    : '');
 	$arr['owner-link']    = ((x($arr, 'owner-link'))    ? notags(trim($arr['owner-link']))    : '');
 	$arr['owner-avatar']  = ((x($arr, 'owner-avatar'))  ? notags(trim($arr['owner-avatar']))  : '');
-	$arr['created']       = ((x($arr, 'created') !== false) ? datetime_convert('UTC','UTC', $arr['created']) : datetime_convert());
+	$arr['received']      = ((x($arr, 'received') !== false) ? datetime_convert('UTC','UTC', $arr['received']) : datetime_convert());
-	$arr['edited']        = ((x($arr, 'edited')  !== false) ? datetime_convert('UTC','UTC', $arr['edited'])  : datetime_convert());
+	$arr['created']       = ((x($arr, 'created') !== false) ? datetime_convert('UTC','UTC', $arr['created']) : $arr['received']);
-	$arr['commented']     = ((x($arr, 'commented')  !== false) ? datetime_convert('UTC','UTC', $arr['commented'])  : datetime_convert());
+	$arr['edited']        = ((x($arr, 'edited') !== false) ? datetime_convert('UTC','UTC', $arr['edited']) : $arr['created']);
-	$arr['received']      = ((x($arr, 'received')  !== false) ? datetime_convert('UTC','UTC', $arr['received'])  : datetime_convert());
+	$arr['changed']       = ((x($arr, 'changed') !== false) ? datetime_convert('UTC','UTC', $arr['changed']) : $arr['created']);
-	$arr['changed']       = ((x($arr, 'changed')  !== false) ? datetime_convert('UTC','UTC', $arr['changed'])  : datetime_convert());
+	$arr['commented']     = ((x($arr, 'commented') !== false) ? datetime_convert('UTC','UTC', $arr['commented']) : $arr['created']);
 	$arr['title']         = ((x($arr, 'title'))         ? trim($arr['title'])         : '');
 	$arr['location']      = ((x($arr, 'location'))      ? trim($arr['location'])      : '');
 	$arr['coord']         = ((x($arr, 'coord'))         ? notags(trim($arr['coord']))         : '');
@@ -1640,6 +1640,9 @@ function item_is_remote_self($contact, &$datarray) {
 			$datarray['author-name']   = $datarray['owner-name'];
 			$datarray['author-link']   = $datarray['owner-link'];
 			$datarray['author-avatar'] = $datarray['owner-avatar'];
+
+			unset($datarray['created']);
+			unset($datarray['edited']);
 		}
 
 		if ($contact['network'] != NETWORK_FEED) {

include/ostatus.php

@@ -21,6 +21,7 @@ require_once 'include/follow.php';
 require_once 'include/api.php';
 require_once 'mod/proxy.php';
 require_once 'include/xml.php';
+require_once 'include/cache.php';
 
 /**
  * @brief This class contain functions for the OStatus protocol
@@ -195,7 +196,7 @@ class ostatus {
 			link_gcontact($gcid, $contact["uid"], $contact["id"]);
 		}
 
-		return($author);
+		return $author;
 	}
 
 	/**
@@ -677,14 +678,14 @@ class ostatus {
 		if ((count($elements) == 4) && ($elements[2] == "post"))
 			return "http://".$server[0]."/notice/".$elements[3];
 
-		if ((count($conversation) != 2) || ($conversation[1] ==""))
+		if ((count($conversation) != 2) || ($conversation[1] =="")) {
 			return $href;
-
+		}
-		if ($elements[3] == "objectType=thread")
+		if ($elements[3] == "objectType=thread") {
 			return "http://".$server[0]."/conversation/".$conversation[1];
-		else
+		} else {
 			return "http://".$server[0]."/notice/".$conversation[1];
-
+		}
 		return $href;
 	}
 
@@ -946,9 +947,9 @@ class ostatus {
 				STRAIGHT_JOIN `item` ON `item`.`parent` = `thritem`.`parent`
 				WHERE `term`.`uid` = %d AND `term`.`otype` = %d AND `term`.`type` = %d AND `term`.`url` = '%s'",
 				intval($uid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($conversation_url));
-		if ($parents)
+		if ($parents) {
 			$parent = $parents[0];
-		elseif (count($item) > 0) {
+		} elseif (count($item) > 0) {
 			$parent = $item;
 			$parent["type"] = "remote";
 			$parent["verb"] = ACTIVITY_POST;
@@ -956,9 +957,9 @@ class ostatus {
 		} else {
 			// Preset the parent
 			$r = q("SELECT `id` FROM `contact` WHERE `self` AND `uid`=%d", $uid);
-			if (!$r)
+			if (!dbm::is_result($r)) {
-				return(-2);
+				return -2;
-
+			}
 			$parent = array();
 			$parent["id"] = 0;
 			$parent["parent"] = 0;
@@ -985,9 +986,9 @@ class ostatus {
 			} elseif (!$conv_arr["success"] && (substr($conv, 0, 8) == "https://")) {
 				$conv = str_replace("https://", "http://", $conv);
 				$conv_as = fetch_url($conv."?page=".$pageno);
-			} else
+			} else {
 				$conv_as = $conv_arr["body"];
-
+			}
 			$conv_as = str_replace(',"statusnet:notice_info":', ',"statusnet_notice_info":', $conv_as);
 			$conv_as = json_decode($conv_as);
 
@@ -1015,9 +1016,10 @@ class ostatus {
 					self::store_conversation($item_id, $conversation_url);
 				}
 
-				return($item_stored);
+				return $item_stored;
-			} else
+			} else {
-				return(-3);
+				return -3;
+			}
 		}
 
 		$items = array_reverse($items);
@@ -1153,7 +1155,7 @@ class ostatus {
 			}
 
 			if (is_array($single_conv->to))
-				foreach($single_conv->to AS $to)
+				foreach ($single_conv->to AS $to)
 					if ($importer["nurl"] == normalise_link($to->id))
 						$mention = true;
 
@@ -1197,17 +1199,17 @@ class ostatus {
 				$arr['conversation-uri'] = $single_conv->status_net->conversation;
 			}
 
-			if (isset($single_conv->status_net->notice_info->source))
+			if (isset($single_conv->status_net->notice_info->source)) {
 				$arr["app"] = strip_tags($single_conv->status_net->notice_info->source);
-			elseif (isset($single_conv->statusnet->notice_info->source))
+			} elseif (isset($single_conv->statusnet->notice_info->source)) {
 				$arr["app"] = strip_tags($single_conv->statusnet->notice_info->source);
-			elseif (isset($single_conv->statusnet_notice_info->source))
+			} elseif (isset($single_conv->statusnet_notice_info->source)) {
 				$arr["app"] = strip_tags($single_conv->statusnet_notice_info->source);
-			elseif (isset($single_conv->provider->displayName))
+			} elseif (isset($single_conv->provider->displayName)) {
 				$arr["app"] = $single_conv->provider->displayName;
-			else
+			} else {
 				$arr["app"] = "OStatus";
-
+			}
 
 			$arr["source"] = json_encode($single_conv);
 			$arr["protocol"] = PROTOCOL_GS_CONVERSATION;
@@ -1324,7 +1326,7 @@ class ostatus {
 			}
 		}
 
-		return($item_stored);
+		return $item_stored;
 	}
 
 	/**
@@ -1367,16 +1369,16 @@ class ostatus {
 		// Skip if it isn't a pure repeated messages
 		// Does it start with a share?
 		if (strpos($body, "[share") > 0)
-			return("");
+			return "";
 
 		// Does it end with a share?
 		if (strlen($body) > (strrpos($body, "[/share]") + 8))
-			return("");
+			return "";
 
 		$attributes = preg_replace("/\[share(.*?)\]\s?(.*?)\s?\[\/share\]\s?/ism","$1",$body);
 		// Skip if there is no shared message in there
 		if ($body == $attributes)
-			return(false);
+			return false;
 
 		$guid = "";
 		preg_match("/guid='(.*?)'/ism", $attributes, $matches);
@@ -1496,12 +1498,12 @@ class ostatus {
 		$hub = get_config('system','huburl');
 
 		$hubxml = '';
-		if(strlen($hub)) {
+		if (strlen($hub)) {
 			$hubs = explode(',', $hub);
-			if(count($hubs)) {
+			if (count($hubs)) {
-				foreach($hubs as $h) {
+				foreach ($hubs as $h) {
 					$h = trim($h);
-					if(! strlen($h))
+					if (! strlen($h))
 						continue;
 					if ($h === '[internal]')
 						$h = App::get_baseurl() . '/pubsubhubbub';
@@ -1543,7 +1545,7 @@ class ostatus {
 				break;
 		}
 
-		if (($siteinfo["type"] != "photo") && isset($siteinfo["image"])) {
+		if (!Config::get('system', 'ostatus_not_attach_preview') && ($siteinfo["type"] != "photo") && isset($siteinfo["image"])) {
 			$imgdata = get_photo_info($siteinfo["image"]);
 			$attributes = array("rel" => "enclosure",
 					"href" => $siteinfo["image"],
@@ -1586,9 +1588,9 @@ class ostatus {
 	private function add_author($doc, $owner) {
 
 		$r = q("SELECT `homepage`, `publish` FROM `profile` WHERE `uid` = %d AND `is-default` LIMIT 1", intval($owner["uid"]));
-		if ($r)
+		if (dbm::is_result($r)) {
 			$profile = $r[0];
-
+		}
 		$author = $doc->createElement("author");
 		xml::add_element($doc, $author, "id", $owner["url"]);
 		xml::add_element($doc, $author, "activity:object-type", ACTIVITY_OBJ_PERSON);
@@ -1743,22 +1745,22 @@ class ostatus {
 
 		$r = q("SELECT * FROM `contact` WHERE `nurl` = '%s' AND `uid` IN (0, %d) ORDER BY `uid` DESC LIMIT 1",
 			dbesc(normalise_link($url)), intval($owner["uid"]));
-		if ($r) {
+		if (dbm::is_result($r)) {
 			$contact = $r[0];
 			$contact["uid"] = -1;
 		}
 
-		if (!$r) {
+		if (!dbm::is_result($r)) {
 			$r = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
 				dbesc(normalise_link($url)));
-			if ($r) {
+			if (dbm::is_result($r)) {
 				$contact = $r[0];
 				$contact["uid"] = -1;
 				$contact["success_update"] = $contact["updated"];
 			}
 		}
 
-		if (!$r)
+		if (!dbm::is_result($r))
 			$contact = owner;
 
 		if (!isset($contact["poll"])) {
@@ -1797,11 +1799,11 @@ class ostatus {
 		$r = q("SELECT * FROM `item` WHERE `uid` = %d AND `guid` = '%s' AND NOT `private` AND `network` IN ('%s', '%s', '%s') LIMIT 1",
 			intval($owner["uid"]), dbesc($repeated_guid),
 			dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
-		if ($r)
+		if (dbm::is_result($r)) {
 			$repeated_item = $r[0];
-		else
+		} else {
 			return false;
-
+		}
 		$contact = self::contact_entry($repeated_item['author-link'], $owner);
 
 		$parent_item = (($item['thr-parent']) ? $item['thr-parent'] : $item['parent-uri']);
@@ -2158,8 +2160,8 @@ class ostatus {
 
 		$tags = item_getfeedtags($item);
 
-		if(count($tags))
+		if (count($tags))
-			foreach($tags as $t)
+			foreach ($tags as $t)
 				if ($t[0] == "@")
 					$mentioned[$t[1]] = $t[1];
 
@@ -2194,8 +2196,8 @@ class ostatus {
 			xml::add_element($doc, $entry, "mastodon:scope", "public");
 		}
 
-		if(count($tags))
+		if (count($tags))
-			foreach($tags as $t)
+			foreach ($tags as $t)
 				if ($t[0] != "@")
 					xml::add_element($doc, $entry, "category", "", array("term" => $t[2]));
 
@@ -2227,19 +2229,33 @@ class ostatus {
 	 *
 	 * @return string XML feed
 	 */
-	public static function feed(App $a, $owner_nick, $last_update) {
+	public static function feed(App $a, $owner_nick, &$last_update) {
+		$stamp = microtime(true);
+
+		$cachekey = "ostatus:feed:".$owner_nick.":".$last_update;
+
+		$previous_created = $last_update;
+
+		$result = Cache::get($cachekey);
+		if (!is_null($result)) {
+			logger('Feed duration: '.number_format(microtime(true) - $stamp, 3).' - '.$owner_nick.' - '.$previous_created.' (cached)', LOGGER_DEBUG);
+			$last_update = $result['last_update'];
+			return $result['feed'];
+		}
 
 		$r = q("SELECT `contact`.*, `user`.`nickname`, `user`.`timezone`, `user`.`page-flags`
 				FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid`
 				WHERE `contact`.`self` AND `user`.`nickname` = '%s' LIMIT 1",
 				dbesc($owner_nick));
-		if (!$r)
+		if (!dbm::is_result($r)) {
 			return;
+		}
 
 		$owner = $r[0];
 
-		if (!strlen($last_update))
+		if (!strlen($last_update)) {
 			$last_update = 'now -30 days';
+		}
 
 		$check_date = datetime_convert('UTC','UTC',$last_update,'Y-m-d H:i:s');
 		$authorid = get_contact($owner["url"], 0);
@@ -2255,28 +2271,6 @@ class ostatus {
 				intval($authorid), dbesc($check_date),
 				dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN));
 
-/*		2016-10-23: The old query will be kept until we are sure that the query above is a good and fast replacement
-
-		$items = q("SELECT `item`.*, `item`.`id` AS `item_id` FROM `item`
-				STRAIGHT_JOIN `thread` ON `thread`.`iid` = `item`.`parent`
-				LEFT JOIN `item` AS `thritem` ON `thritem`.`uri`=`item`.`thr-parent` AND `thritem`.`uid`=`item`.`uid`
-				WHERE `item`.`uid` = %d AND `item`.`received` > '%s' AND NOT `item`.`private` AND NOT `item`.`deleted`
-					AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = ''
-					AND ((`item`.`wall` AND (`item`.`parent` = `item`.`id`))
-						OR (`item`.`network` = '%s' AND ((`thread`.`network` IN ('%s', '%s')) OR (`thritem`.`network` IN ('%s', '%s')))) AND `thread`.`mention`)
-					AND ((`item`.`owner-link` IN ('%s', '%s') AND (`item`.`parent` = `item`.`id`))
-						OR (`item`.`author-link` IN ('%s', '%s')))
-				ORDER BY `item`.`id` DESC
-				LIMIT 0, 300",
-				intval($owner["uid"]), dbesc($check_date), dbesc(NETWORK_DFRN),
-				//dbesc(NETWORK_OSTATUS), dbesc(NETWORK_OSTATUS),
-				//dbesc(NETWORK_OSTATUS), dbesc(NETWORK_OSTATUS),
-				dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN),
-				dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN),
-				dbesc($owner["nurl"]), dbesc(str_replace("http://", "https://", $owner["nurl"])),
-				dbesc($owner["nurl"]), dbesc(str_replace("http://", "https://", $owner["nurl"]))
-			);
-*/
 		$doc = new DOMDocument('1.0', 'utf-8');
 		$doc->formatOutput = true;
 
@@ -2288,9 +2282,20 @@ class ostatus {
 			}
 			$entry = self::entry($doc, $item, $owner);
 			$root->appendChild($entry);
+
+			if ($last_update < $item['created']) {
+				$last_update = $item['created'];
+			}
 		}
 
-		return(trim($doc->saveXML()));
+		$feeddata = trim($doc->saveXML());
+
+		$msg = array('feed' => $feeddata, 'last_update' => $last_update);
+		Cache::set($cachekey, $msg, CACHE_QUARTER_HOUR);
+
+		logger('Feed duration: '.number_format(microtime(true) - $stamp, 3).' - '.$owner_nick.' - '.$previous_created, LOGGER_DEBUG);
+
+		return $feeddata;
 	}
 
 	/**
@@ -2314,6 +2319,6 @@ class ostatus {
 
 		$doc->appendChild($entry);
 
-		return(trim($doc->saveXML()));
+		return trim($doc->saveXML());
 	}
 }

include/poller.php

@@ -273,7 +273,9 @@ function poller_exec_function($queue, $funcname, $argv) {
 
 	$argc = count($argv);
 
-	logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]);
+	$new_process_id = uniqid("wrk", true);
+
+	logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]." - Process PID: ".$new_process_id);
 
 	$stamp = (float)microtime(true);
 
@@ -295,7 +297,7 @@ function poller_exec_function($queue, $funcname, $argv) {
 	// For better logging create a new process id for every worker call
 	// But preserve the old one for the worker
 	$old_process_id = $a->process_id;
-	$a->process_id = uniqid("wrk", true);
+	$a->process_id = $new_process_id;
 	$a->queue = $queue;
 
 	$up_duration = number_format(microtime(true) - $poller_up_start, 3);
@@ -330,7 +332,7 @@ function poller_exec_function($queue, $funcname, $argv) {
 		logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 2 minutes (".round($duration/60, 3).")", LOGGER_DEBUG);
 	}
 
-	logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds.");
+	logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds. Process PID: ".$new_process_id);
 
 	// Write down the performance values into the log
 	if (Config::get("system", "profiler")) {

include/pubsubpublish.php

@@ -14,7 +14,7 @@ function pubsubpublish_run(&$argv, &$argc){
 	} else {
 		// We'll push to each subscriber that has push > 0,
 		// i.e. there has been an update (set in notifier.php).
-		$r = q("SELECT `id`, `callback_url` FROM `push_subscriber` WHERE `push` > 0");
+		$r = q("SELECT `id`, `callback_url` FROM `push_subscriber` WHERE `push` > 0 ORDER BY `last_update` DESC");
 
 		foreach ($r as $rr) {
 			logger("Publish feed to ".$rr["callback_url"], LOGGER_DEBUG);
@@ -43,7 +43,13 @@ function handle_pubsubhubbub($id) {
 
 	logger("Generate feed of user ".$rr['nickname']." to ".$rr['callback_url']." - last updated ".$rr['last_update'], LOGGER_DEBUG);
 
-	$params = ostatus::feed($a, $rr['nickname'], $rr['last_update']);
+	$last_update = $rr['last_update'];
+	$params = ostatus::feed($a, $rr['nickname'], $last_update);
+
+	if (!$params) {
+		return;
+	}
+
 	$hmac_sig = hash_hmac("sha1", $params, $rr['secret']);
 
 	$headers = array("Content-type: application/atom+xml",
@@ -60,10 +66,9 @@ function handle_pubsubhubbub($id) {
 	if ($ret >= 200 && $ret <= 299) {
 		logger('successfully pushed to '.$rr['callback_url']);
 
-		// set last_update to "now", and reset push=0
+		// set last_update to the "created" date of the last item, and reset push=0
-		$date_now = datetime_convert('UTC','UTC','now','Y-m-d H:i:s');
 		q("UPDATE `push_subscriber` SET `push` = 0, last_update = '%s' WHERE id = %d",
-			dbesc($date_now),
+			dbesc($last_update),
 			intval($rr['id']));
 
 	} else {