clones/friendica
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2851 from annando/1610-performance-nodeinfo 

Performance improvements to item storage and nodeinfo
This commit is contained in:
Tobias Diekershoff 2016-10-08 08:42:50 +02:00 committed by GitHub
parent 43df08f860 b70676d231
commit a1c63994d2
5 changed files with 110 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
include/items.php
View File

@ -669,7 +669,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
// If its a post from myself then tag the thread as "mention"
logger("item_store: Checking if parent ".$parent_id." has to be tagged as mention for user ".$arr['uid'], LOGGER_DEBUG);
$u = q("select * from user where uid = %d limit 1", intval($arr['uid']));
$u = q("SELECT `nickname` FROM `user` WHERE `uid` = %d", intval($arr['uid']));
if(count($u)) {
$a = get_app();
$self = normalise_link($a->get_baseurl() . '/profile/' . $u[0]['nickname']);
@ -679,8 +679,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
logger("item_store: tagged thread ".$parent_id." as mention for user ".$self, LOGGER_DEBUG);
}
}
}
else {
} else {
// Allow one to see reply tweets from status.net even when
// we don't have or can't see the original post.
@ -735,6 +734,19 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
$arr["global"] = (count($isglobal) > 0);
}
// ACL settings
if(strlen($allow_cid) || strlen($allow_gid) || strlen($deny_cid) || strlen($deny_gid))
$private = 1;
else
$private = $arr['private'];
$arr["allow_cid"] = $allow_cid;
$arr["allow_gid"] = $allow_gid;
$arr["deny_cid"] = $deny_cid;
$arr["deny_gid"] = $deny_gid;
$arr["private"] = $private;
$arr["deleted"] = $parent_deleted;
// Fill the cache field
put_item_in_cache($arr);
@ -807,41 +819,38 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
dbesc($arr['received']),
intval($arr['contact-id'])
);
// Now do the same for the system wide contacts with uid=0
if (!$arr['private']) {
q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d",
dbesc($arr['received']),
dbesc($arr['received']),
intval($arr['owner-id'])
);
if ($arr['owner-id'] != $arr['author-id'])
q("UPDATE `contact` SET `success_update` = '%s', `last-item` = '%s' WHERE `id` = %d",
dbesc($arr['received']),
dbesc($arr['received']),
intval($arr['author-id'])
);
}
} else {
logger('item_store: could not locate created item');
return 0;
}
if((! $parent_id) || ($arr['parent-uri'] === $arr['uri']))
if(!$parent_id || ($arr['parent-uri'] === $arr['uri']))
$parent_id = $current_post;
if(strlen($allow_cid) || strlen($allow_gid) || strlen($deny_cid) || strlen($deny_gid))
$private = 1;
else
$private = $arr['private'];
// Set parent id - and also make sure to inherit the parent's ACLs.
$r = q("UPDATE `item` SET `parent` = %d, `allow_cid` = '%s', `allow_gid` = '%s',
`deny_cid` = '%s', `deny_gid` = '%s', `private` = %d, `deleted` = %d WHERE `id` = %d",
// Set parent id
$r = q("UPDATE `item` SET `parent` = %d WHERE `id` = %d",
intval($parent_id),
dbesc($allow_cid),
dbesc($allow_gid),
dbesc($deny_cid),
dbesc($deny_gid),
intval($private),
intval($parent_deleted),
intval($current_post)
);
$arr['id'] = $current_post;
$arr['parent'] = $parent_id;
$arr['allow_cid'] = $allow_cid;
$arr['allow_gid'] = $allow_gid;
$arr['deny_cid'] = $deny_cid;
$arr['deny_gid'] = $deny_gid;
$arr['private'] = $private;
$arr['deleted'] = $parent_deleted;
// update the commented timestamp on the parent
// Only update "commented" if it is really a comment

8
include/poller.php
View File

@ -128,8 +128,16 @@ function poller_run(&$argv, &$argc){
if (function_exists($funcname)) {
logger("Process ".$mypid." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." ".$r[0]["parameter"]);
// For better logging create a new process id for every worker call
// But preserve the old one for the worker
$old_process_id = $a->process_id;
$a->process_id = uniqid("wrk", true);
$funcname($argv, $argc);
$a->process_id = $old_process_id;
if ($cooldown > 0) {
logger("Process ".$mypid." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds");
sleep($cooldown);

35
mod/admin.php
View File

@ -1279,14 +1279,14 @@ function admin_page_users(&$a){
/* ordering */
$valid_orders = array(
'contact.name',
'contact.name',
'user.email',
'user.register_date',
'user.login_date',
'lastitem.lastitem_date',
'lastitem_date',
'user.page-flags'
);
$order = "contact.name";
$order_direction = "+";
if (x($_GET,'o')){
@ -1295,38 +1295,29 @@ function admin_page_users(&$a){
$order_direction = "-";
$new_order = substr($new_order,1);
}
if (in_array($new_order, $valid_orders)){
$order = $new_order;
}
if (x($_GET,'d')){
$new_direction = $_GET['d'];
}
}
$sql_order = "`".str_replace('.','`.`',$order)."`";
$sql_order_direction = ($order_direction==="+")?"ASC":"DESC";
$users = q("SELECT `user`.* , `contact`.`name` , `contact`.`url` , `contact`.`micro`, `lastitem`.`lastitem_date`, `user`.`account_expired`
FROM
(SELECT MAX(`item`.`changed`) as `lastitem_date`, `item`.`uid`
FROM `item`
WHERE `item`.`type` = 'wall'
GROUP BY `item`.`uid`) AS `lastitem`
RIGHT OUTER JOIN `user` ON `user`.`uid` = `lastitem`.`uid`,
`contact`
WHERE
`user`.`uid` = `contact`.`uid`
AND `user`.`verified` =1
AND `contact`.`self` =1
ORDER BY $sql_order $sql_order_direction LIMIT %d, %d
",
$users = q("SELECT `user`.*, `contact`.`name`, `contact`.`url`, `contact`.`micro`, `user`.`account_expired`,
(SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date`
FROM `user`
INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`
WHERE `user`.`verified`
ORDER BY $sql_order $sql_order_direction LIMIT %d, %d",
intval($a->pager['start']),
intval($a->pager['itemspage'])
);
//echo "<pre>$users"; killme();
$adminlist = explode(",", str_replace(" ", "", $a->config['admin_email']));
$_setup_users = function ($e) use ($adminlist){
$accounts = array(

115
mod/item.php
View File

@ -292,7 +292,6 @@ function item_post(&$a) {
// If this is a comment, set the permissions from the parent.
if($parent_item) {
$private = 0;
// for non native networks use the network of the original post as network of the item
if (($parent_item['network'] != NETWORK_DIASPORA)
@ -300,19 +299,13 @@ function item_post(&$a) {
AND ($network == ""))
$network = $parent_item['network'];
if(($parent_item['private'])
|| strlen($parent_item['allow_cid'])
|| strlen($parent_item['allow_gid'])
|| strlen($parent_item['deny_cid'])
|| strlen($parent_item['deny_gid'])) {
$private = (($parent_item['private']) ? $parent_item['private'] : 1);
}
$str_contact_allow = $parent_item['allow_cid'];
$str_group_allow = $parent_item['allow_gid'];
$str_contact_deny = $parent_item['deny_cid'];
$str_group_deny = $parent_item['deny_gid'];
$private = $parent_item['private'];
}
$pubmail_enable = ((x($_REQUEST,'pubmail_enable') && intval($_REQUEST['pubmail_enable']) && (! $private)) ? 1 : 0);
// if using the API, we won't see pubmail_enable - figure out if it should be set
@ -460,7 +453,6 @@ function item_post(&$a) {
if(! count($r))
continue;
$r = q("UPDATE `photo` SET `allow_cid` = '%s', `allow_gid` = '%s', `deny_cid` = '%s', `deny_gid` = '%s'
WHERE `resource-id` = '%s' AND `uid` = %d AND `album` = '%s' ",
dbesc($str_contact_allow),
@ -471,7 +463,6 @@ function item_post(&$a) {
intval($profile_uid),
dbesc( t('Wall Photos'))
);
}
}
}
@ -725,6 +716,11 @@ function item_post(&$a) {
$datarray['self'] = $self;
// $datarray['prvnets'] = $user['prvnets'];
$datarray['parent-uri'] = ($parent == 0) ? $uri : $parent_item['uri'];
$datarray['plink'] = $a->get_baseurl().'/display/'.urlencode($datarray['guid']);
$datarray['last-child'] = 1;
$datarray['visible'] = 1;
if($orig_post)
$datarray['edit'] = true;
@ -789,11 +785,9 @@ function item_post(&$a) {
goaway($a->get_baseurl() . "/" . $return_path );
}
killme();
}
else
} else
$post_id = 0;
$r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`,
`owner-name`,`owner-link`,`owner-avatar`, `owner-id`,
`author-name`, `author-link`, `author-avatar`, `author-id`,
@ -802,7 +796,8 @@ function item_post(&$a) {
`tag`, `inform`, `verb`, `object-type`, `postopts`,
`allow_cid`, `allow_gid`, `deny_cid`, `deny_gid`, `private`,
`pubmail`, `attach`, `bookmark`,`origin`, `moderated`, `file`,
`rendered-html`, `rendered-hash`)
`rendered-html`, `rendered-hash`,
`parent`, `parent-uri`, `plink`, `last-child`, `visible`)
VALUES('%s', '%s', %d, '%s', %d, %d, '%s', %d,
'%s', '%s', '%s', %d,
'%s', '%s', '%s', %d,
@ -811,7 +806,8 @@ function item_post(&$a) {
'%s', '%s', '%s', '%s', '%s',
'%s', '%s', '%s', '%s', %d,
%d, '%s', %d, %d, %d, '%s',
'%s', '%s')",
'%s', '%s',
%d, '%s', '%s', %d, %d)",
dbesc($datarray['guid']),
dbesc($datarray['extid']),
intval($datarray['uid']),
@ -857,7 +853,12 @@ function item_post(&$a) {
intval($datarray['moderated']),
dbesc($datarray['file']),
dbesc($datarray['rendered-html']),
dbesc($datarray['rendered-hash'])
dbesc($datarray['rendered-hash']),
intval($datarray['parent']),
dbesc($datarray['parent-uri']),
dbesc($datarray['plink']),
intval($datarray['last-child']),
intval($datarray['visible'])
);
$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' LIMIT 1",
@ -873,7 +874,6 @@ function item_post(&$a) {
logger('mod_item: saved item ' . $post_id);
$datarray["id"] = $post_id;
$datarray["plink"] = $a->get_baseurl().'/display/'.urlencode($datarray["guid"]);
// update filetags in pconfig
file_tag_update_pconfig($uid,$categories_old,$categories_new,'category');
@ -881,23 +881,18 @@ function item_post(&$a) {
if($parent) {
// This item is the last leaf and gets the comment box, clear any ancestors
$r = q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent` = %d ",
$r = q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent` = %d AND `last-child` AND `id` != %d",
dbesc(datetime_convert()),
intval($parent),
intval($post_id)
);
// update the commented timestamp on the parent
q("UPDATE `item` SET `visible` = 1, `commented` = '%s', `changed` = '%s' WHERE `id` = %d",
dbesc(datetime_convert()),
dbesc(datetime_convert()),
intval($parent)
);
update_thread($parent, true);
// Inherit ACLs from the parent item.
$r = q("UPDATE `item` SET `allow_cid` = '%s', `allow_gid` = '%s', `deny_cid` = '%s', `deny_gid` = '%s', `private` = %d
WHERE `id` = %d",
dbesc($parent_item['allow_cid']),
dbesc($parent_item['allow_gid']),
dbesc($parent_item['deny_cid']),
dbesc($parent_item['deny_gid']),
intval($parent_item['private']),
intval($post_id)
);
if($contact_record != $author) {
notification(array(
@ -927,6 +922,10 @@ function item_post(&$a) {
} else {
$parent = $post_id;
$r = q("UPDATE `item` SET `parent` = %d WHERE `id` = %d",
intval($parent),
intval($post_id));
if($contact_record != $author) {
notification(array(
'type' => NOTIFY_WALL,
@ -946,41 +945,6 @@ function item_post(&$a) {
}
}
// fallback so that parent always gets set to non-zero.
if(! $parent)
$parent = $post_id;
$r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s', `plink` = '%s', `changed` = '%s', `last-child` = 1, `visible` = 1
WHERE `id` = %d",
intval($parent),
dbesc(($parent == $post_id) ? $uri : $parent_item['uri']),
dbesc($a->get_baseurl().'/display/'.urlencode($datarray['guid'])),
dbesc(datetime_convert()),
intval($post_id)
);
// photo comments turn the corresponding item visible to the profile wall
// This way we don't see every picture in your new photo album posted to your wall at once.
// They will show up as people comment on them.
if(! $parent_item['visible']) {
$r = q("UPDATE `item` SET `visible` = 1 WHERE `id` = %d",
intval($parent_item['id'])
);
update_thread($parent_item['id']);
}
// update the commented timestamp on the parent
q("UPDATE `item` set `commented` = '%s', `changed` = '%s' WHERE `id` = %d",
dbesc(datetime_convert()),
dbesc(datetime_convert()),
intval($parent)
);
if ($post_id != $parent)
update_thread($parent);
call_hooks('post_local_end', $datarray);
if(strlen($emailcc) && $profile_uid == local_user()) {
@ -1022,6 +986,23 @@ function item_post(&$a) {
if ($post_id == $parent)
add_thread($post_id);
else {
update_thread($parent, true);
// Insert an item entry for UID=0 for global entries
// We have to remove or change some data before that,
// so that the post appear like a regular received post.
unset($datarray['self']);
unset($datarray['wall']);
unset($datarray['origin']);
if (in_array($datarray['type'], array("net-comment", "wall-comment")))
$datarray['type'] = 'remote-comment';
elseif ($datarray['type'] == 'wall')
$datarray['type'] = 'remote';
add_shadow_entry($datarray);
}
// This is a real juggling act on shared hosting services which kill your processes
// e.g. dreamhost. We used to start delivery to our native delivery agents in the background

21
mod/nodeinfo.php
View File

@ -185,20 +185,13 @@ function nodeinfo_cron() {
}
logger("cron_start");
$users = q("SELECT profile.*, `user`.`login_date`, `lastitem`.`lastitem_date`
FROM (SELECT MAX(`item`.`changed`) as `lastitem_date`, `item`.`uid`
FROM `item`
WHERE `item`.`type` = 'wall'
GROUP BY `item`.`uid`) AS `lastitem`
RIGHT OUTER JOIN `user` ON `user`.`uid` = `lastitem`.`uid`, `contact`, `profile`
WHERE
`user`.`uid` = `contact`.`uid` AND `profile`.`uid` = `user`.`uid`
AND `profile`.`is-default` AND (`profile`.`publish` OR `profile`.`net-publish`)
AND `user`.`verified` AND `contact`.`self`
AND NOT `user`.`blocked`
AND NOT `user`.`account_removed`
AND NOT `user`.`account_expired`");
$users = q("SELECT `user`.`uid`, `user`.`login_date`,
(SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date`
FROM `user`
INNER JOIN `profile` ON `profile`.`uid` = `user`.`uid` AND `profile`.`is-default`
WHERE (`profile`.`publish` OR `profile`.`net-publish`) AND `user`.`verified`
AND NOT `user`.`blocked` AND NOT `user`.`account_removed`
AND NOT `user`.`account_expired`");
if (is_array($users)) {
$total_users = count($users);
$active_users_halfyear = 0;