Merge pull request #2851 from annando/1610-performance-nodeinfo

Performance improvements to item storage and nodeinfo

include/items.php

@@ -669,7 +669,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 
 			// If its a post from myself then tag the thread as "mention"
 			logger("item_store: Checking if parent ".$parent_id." has to be tagged as mention for user ".$arr['uid'], LOGGER_DEBUG);
-			$u = q("select * from user where uid = %d limit 1", intval($arr['uid']));
+			$u = q("SELECT `nickname` FROM `user` WHERE `uid` = %d", intval($arr['uid']));
 			if(count($u)) {
 				$a = get_app();
 				$self = normalise_link($a->get_baseurl() . '/profile/' . $u[0]['nickname']);
@@ -679,8 +679,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 					logger("item_store: tagged thread ".$parent_id." as mention for user ".$self, LOGGER_DEBUG);
 				}
 			}
-		}
+		} else {
-		else {
 
 			// Allow one to see reply tweets from status.net even when
 			// we don't have or can't see the original post.
@@ -735,6 +734,19 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 		$arr["global"] = (count($isglobal) > 0);
 	}
 
+	// ACL settings
+	if(strlen($allow_cid) || strlen($allow_gid) || strlen($deny_cid) || strlen($deny_gid))
+		$private = 1;
+	else
+		$private = $arr['private'];
+
+	$arr["allow_cid"] = $allow_cid;
+	$arr["allow_gid"] = $allow_gid;
+	$arr["deny_cid"] = $deny_cid;
+	$arr["deny_gid"] = $deny_gid;
+	$arr["private"] = $private;
+	$arr["deleted"] = $parent_deleted;
+
 	// Fill the cache field
 	put_item_in_cache($arr);
 
@@ -807,41 +819,38 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 				dbesc($arr['received']),
 				intval($arr['contact-id'])
 			);
+
+		// Now do the same for the system wide contacts with uid=0
+		if (!$arr['private']) {
+			q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d",
+				dbesc($arr['received']),
+				dbesc($arr['received']),
+				intval($arr['owner-id'])
+			);
+
+			if ($arr['owner-id'] != $arr['author-id'])
+				q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d",
+					dbesc($arr['received']),
+					dbesc($arr['received']),
+					intval($arr['author-id'])
+				);
+		}
 	} else {
 		logger('item_store: could not locate created item');
 		return 0;
 	}
 
-	if((! $parent_id) || ($arr['parent-uri'] === $arr['uri']))
+	if(!$parent_id || ($arr['parent-uri'] === $arr['uri']))
 		$parent_id = $current_post;
 
-	if(strlen($allow_cid) || strlen($allow_gid) || strlen($deny_cid) || strlen($deny_gid))
+	// Set parent id
-		$private = 1;
+	$r = q("UPDATE `item` SET `parent` = %d WHERE `id` = %d",
-	else
-		$private = $arr['private'];
-
-	// Set parent id - and also make sure to inherit the parent's ACLs.
-
-	$r = q("UPDATE `item` SET `parent` = %d, `allow_cid` = '%s', `allow_gid` = '%s',
-		`deny_cid` = '%s', `deny_gid` = '%s', `private` = %d, `deleted` = %d WHERE `id` = %d",
 		intval($parent_id),
-		dbesc($allow_cid),
-		dbesc($allow_gid),
-		dbesc($deny_cid),
-		dbesc($deny_gid),
-		intval($private),
-		intval($parent_deleted),
 		intval($current_post)
 	);
 
 	$arr['id'] = $current_post;
 	$arr['parent'] = $parent_id;
-	$arr['allow_cid'] = $allow_cid;
-	$arr['allow_gid'] = $allow_gid;
-	$arr['deny_cid'] = $deny_cid;
-	$arr['deny_gid'] = $deny_gid;
-	$arr['private'] = $private;
-	$arr['deleted'] = $parent_deleted;
 
 	// update the commented timestamp on the parent
 	// Only update "commented" if it is really a comment

include/poller.php

@@ -128,8 +128,16 @@ function poller_run(&$argv, &$argc){
 
 		if (function_exists($funcname)) {
 			logger("Process ".$mypid." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." ".$r[0]["parameter"]);
+
+			// For better logging create a new process id for every worker call
+			// But preserve the old one for the worker
+			$old_process_id = $a->process_id;
+			$a->process_id = uniqid("wrk", true);
+
 			$funcname($argv, $argc);
 
+			$a->process_id = $old_process_id;
+
 			if ($cooldown > 0) {
 				logger("Process ".$mypid." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds");
 				sleep($cooldown);

mod/admin.php

@@ -1283,7 +1283,7 @@ function admin_page_users(&$a){
 		'user.email',
 		'user.register_date',
 		'user.login_date',
-		'lastitem.lastitem_date',
+		'lastitem_date',
 		'user.page-flags'
 	);
 
@@ -1301,26 +1301,17 @@ function admin_page_users(&$a){
 		}
 		if (x($_GET,'d')){
 			$new_direction = $_GET['d'];
-			
 		}
 	}
 	$sql_order = "`".str_replace('.','`.`',$order)."`";
 	$sql_order_direction = ($order_direction==="+")?"ASC":"DESC";
 
-	$users = q("SELECT `user`.* , `contact`.`name` , `contact`.`url` , `contact`.`micro`, `lastitem`.`lastitem_date`, `user`.`account_expired`
+	$users = q("SELECT `user`.*, `contact`.`name`, `contact`.`url`, `contact`.`micro`, `user`.`account_expired`,
-				FROM
+				(SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date`
-					(SELECT MAX(`item`.`changed`) as `lastitem_date`, `item`.`uid`
+				FROM `user`
-					FROM `item`
+				INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`
-					WHERE `item`.`type` = 'wall'
+				WHERE `user`.`verified`
-					GROUP BY `item`.`uid`) AS `lastitem`
+				ORDER BY $sql_order $sql_order_direction LIMIT %d, %d",
-						 RIGHT OUTER JOIN `user` ON `user`.`uid` = `lastitem`.`uid`,
-					   `contact`
-				WHERE
-					   `user`.`uid` = `contact`.`uid`
-						AND `user`.`verified` =1
-					AND `contact`.`self` =1
-				ORDER BY $sql_order $sql_order_direction LIMIT %d, %d
-				",
 				intval($a->pager['start']),
 				intval($a->pager['itemspage'])
 				);

mod/item.php

@@ -292,7 +292,6 @@ function item_post(&$a) {
 		// If this is a comment, set the permissions from the parent.
 
 		if($parent_item) {
-			$private = 0;
 
 			// for non native networks use the network of the original post as network of the item
 			if (($parent_item['network'] != NETWORK_DIASPORA)
@@ -300,19 +299,13 @@ function item_post(&$a) {
 				AND ($network == ""))
 				$network = $parent_item['network'];
 
-			if(($parent_item['private'])
-				|| strlen($parent_item['allow_cid'])
-				|| strlen($parent_item['allow_gid'])
-				|| strlen($parent_item['deny_cid'])
-				|| strlen($parent_item['deny_gid'])) {
-				$private = (($parent_item['private']) ? $parent_item['private'] : 1);
-			}
-
 			$str_contact_allow = $parent_item['allow_cid'];
 			$str_group_allow   = $parent_item['allow_gid'];
 			$str_contact_deny  = $parent_item['deny_cid'];
 			$str_group_deny    = $parent_item['deny_gid'];
+			$private           = $parent_item['private'];
 		}
+
 		$pubmail_enable    = ((x($_REQUEST,'pubmail_enable') && intval($_REQUEST['pubmail_enable']) && (! $private)) ? 1 : 0);
 
 		// if using the API, we won't see pubmail_enable - figure out if it should be set
@@ -460,7 +453,6 @@ function item_post(&$a) {
 				if(! count($r))
 					continue;
 
-
 				$r = q("UPDATE `photo` SET `allow_cid` = '%s', `allow_gid` = '%s', `deny_cid` = '%s', `deny_gid` = '%s'
 					WHERE `resource-id` = '%s' AND `uid` = %d AND `album` = '%s' ",
 					dbesc($str_contact_allow),
@@ -471,7 +463,6 @@ function item_post(&$a) {
 					intval($profile_uid),
 					dbesc( t('Wall Photos'))
 				);
-
 			}
 		}
 	}
@@ -725,6 +716,11 @@ function item_post(&$a) {
 	$datarray['self']          = $self;
 //	$datarray['prvnets']       = $user['prvnets'];
 
+	$datarray['parent-uri'] = ($parent == 0) ? $uri : $parent_item['uri'];
+	$datarray['plink'] = $a->get_baseurl().'/display/'.urlencode($datarray['guid']);
+	$datarray['last-child'] = 1;
+	$datarray['visible'] = 1;
+
 	if($orig_post)
 		$datarray['edit']      = true;
 
@@ -789,11 +785,9 @@ function item_post(&$a) {
 			goaway($a->get_baseurl() . "/" . $return_path );
 		}
 		killme();
-	}
+	} else
-	else
 		$post_id = 0;
 
-
 	$r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`,
 					`owner-name`,`owner-link`,`owner-avatar`, `owner-id`,
 					`author-name`, `author-link`, `author-avatar`, `author-id`,
@@ -802,7 +796,8 @@ function item_post(&$a) {
 					`tag`, `inform`, `verb`, `object-type`, `postopts`,
 					`allow_cid`, `allow_gid`, `deny_cid`, `deny_gid`, `private`,
 					`pubmail`, `attach`, `bookmark`,`origin`, `moderated`, `file`,
-					`rendered-html`, `rendered-hash`)
+					`rendered-html`, `rendered-hash`,
+					`parent`, `parent-uri`, `plink`, `last-child`, `visible`)
 		VALUES('%s', '%s', %d, '%s', %d, %d, '%s', %d,
 			'%s', '%s', '%s', %d,
 			'%s', '%s', '%s', %d,
@@ -811,7 +806,8 @@ function item_post(&$a) {
 			'%s', '%s', '%s', '%s', '%s',
 			'%s', '%s', '%s', '%s', %d,
 			%d, '%s', %d, %d, %d, '%s',
-			'%s', '%s')",
+			'%s', '%s',
+			%d, '%s', '%s', %d, %d)",
 		dbesc($datarray['guid']),
 		dbesc($datarray['extid']),
 		intval($datarray['uid']),
@@ -857,7 +853,12 @@ function item_post(&$a) {
 		intval($datarray['moderated']),
 		dbesc($datarray['file']),
 		dbesc($datarray['rendered-html']),
-		dbesc($datarray['rendered-hash'])
+		dbesc($datarray['rendered-hash']),
+		intval($datarray['parent']),
+		dbesc($datarray['parent-uri']),
+		dbesc($datarray['plink']),
+		intval($datarray['last-child']),
+		intval($datarray['visible'])
 	       );
 
 	$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' LIMIT 1",
@@ -873,7 +874,6 @@ function item_post(&$a) {
 	logger('mod_item: saved item ' . $post_id);
 
 	$datarray["id"] = $post_id;
-	$datarray["plink"] = $a->get_baseurl().'/display/'.urlencode($datarray["guid"]);
 
 	// update filetags in pconfig
 	file_tag_update_pconfig($uid,$categories_old,$categories_new,'category');
@@ -881,23 +881,18 @@ function item_post(&$a) {
 	if($parent) {
 
 		// This item is the last leaf and gets the comment box, clear any ancestors
-		$r = q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent` = %d ",
+		$r = q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent` = %d AND `last-child` AND `id` != %d",
+			dbesc(datetime_convert()),
+			intval($parent),
+			intval($post_id)
+		);
+
+		// update the commented timestamp on the parent
+		q("UPDATE `item` SET `visible` = 1, `commented` = '%s', `changed` = '%s' WHERE `id` = %d",
+			dbesc(datetime_convert()),
 			dbesc(datetime_convert()),
 			intval($parent)
 		);
-		update_thread($parent, true);
-
-		// Inherit ACLs from the parent item.
-
-		$r = q("UPDATE `item` SET `allow_cid` = '%s', `allow_gid` = '%s', `deny_cid` = '%s', `deny_gid` = '%s', `private` = %d
-			WHERE `id` = %d",
-			dbesc($parent_item['allow_cid']),
-			dbesc($parent_item['allow_gid']),
-			dbesc($parent_item['deny_cid']),
-			dbesc($parent_item['deny_gid']),
-			intval($parent_item['private']),
-			intval($post_id)
-		);
 
 		if($contact_record != $author) {
 			notification(array(
@@ -927,6 +922,10 @@ function item_post(&$a) {
 	} else {
 		$parent = $post_id;
 
+		$r = q("UPDATE `item` SET `parent` = %d WHERE `id` = %d",
+			intval($parent),
+			intval($post_id));
+
 		if($contact_record != $author) {
 			notification(array(
 				'type'         => NOTIFY_WALL,
@@ -946,41 +945,6 @@ function item_post(&$a) {
 		}
 	}
 
-	// fallback so that parent always gets set to non-zero.
-
-	if(! $parent)
-		$parent = $post_id;
-
-	$r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s', `plink` = '%s', `changed` = '%s', `last-child` = 1, `visible` = 1
-		WHERE `id` = %d",
-		intval($parent),
-		dbesc(($parent == $post_id) ? $uri : $parent_item['uri']),
-		dbesc($a->get_baseurl().'/display/'.urlencode($datarray['guid'])),
-		dbesc(datetime_convert()),
-		intval($post_id)
-	);
-
-	// photo comments turn the corresponding item visible to the profile wall
-	// This way we don't see every picture in your new photo album posted to your wall at once.
-	// They will show up as people comment on them.
-
-	if(! $parent_item['visible']) {
-		$r = q("UPDATE `item` SET `visible` = 1 WHERE `id` = %d",
-			intval($parent_item['id'])
-		);
-		update_thread($parent_item['id']);
-	}
-
-	// update the commented timestamp on the parent
-
-	q("UPDATE `item` set `commented` = '%s', `changed` = '%s' WHERE `id` = %d",
-		dbesc(datetime_convert()),
-		dbesc(datetime_convert()),
-		intval($parent)
-	);
-	if ($post_id != $parent)
-		update_thread($parent);
-
 	call_hooks('post_local_end', $datarray);
 
 	if(strlen($emailcc) && $profile_uid == local_user()) {
@@ -1022,6 +986,23 @@ function item_post(&$a) {
 
 	if ($post_id == $parent)
 		add_thread($post_id);
+	else {
+		update_thread($parent, true);
+
+		// Insert an item entry for UID=0 for global entries
+		// We have to remove or change some data before that,
+		// so that the post appear like a regular received post.
+		unset($datarray['self']);
+		unset($datarray['wall']);
+		unset($datarray['origin']);
+
+		if (in_array($datarray['type'], array("net-comment", "wall-comment")))
+			$datarray['type'] = 'remote-comment';
+		elseif ($datarray['type'] == 'wall')
+			$datarray['type'] = 'remote';
+
+		add_shadow_entry($datarray);
+	}
 
 	// This is a real juggling act on shared hosting services which kill your processes
 	// e.g. dreamhost. We used to start delivery to our native delivery agents in the background

mod/nodeinfo.php

@@ -185,20 +185,13 @@ function nodeinfo_cron() {
 	}
         logger("cron_start");
 
-	$users = q("SELECT profile.*, `user`.`login_date`, `lastitem`.`lastitem_date`
+	$users = q("SELECT `user`.`uid`, `user`.`login_date`,
-			FROM (SELECT MAX(`item`.`changed`) as `lastitem_date`, `item`.`uid`
+			(SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date`
-				FROM `item`
+			FROM `user`
-					WHERE `item`.`type` = 'wall'
+			INNER JOIN `profile` ON `profile`.`uid` = `user`.`uid` AND `profile`.`is-default`
-						GROUP BY `item`.`uid`) AS `lastitem`
+			WHERE (`profile`.`publish` OR `profile`.`net-publish`) AND `user`.`verified`
-						RIGHT OUTER JOIN `user` ON `user`.`uid` = `lastitem`.`uid`, `contact`, `profile`
+				AND NOT `user`.`blocked` AND NOT `user`.`account_removed`
-                                WHERE
-					`user`.`uid` = `contact`.`uid` AND `profile`.`uid` = `user`.`uid`
-					AND `profile`.`is-default` AND (`profile`.`publish` OR `profile`.`net-publish`)
-					AND `user`.`verified` AND `contact`.`self`
-					AND NOT `user`.`blocked`
-					AND NOT `user`.`account_removed`
 				AND NOT `user`.`account_expired`");
-
 	if (is_array($users)) {
 			$total_users = count($users);
 			$active_users_halfyear = 0;