clones/friendica
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into 1701-performance

Browse Source
This commit is contained in:
Michael
2017-02-23 06:55:35 +00:00
parent f5a634abbc 55288e8e6e
commit f9f27c2f5f
29 changed files with 398 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/Contact.php
+4
View File
@@ -730,6 +730,10 @@ function posts_from_contact_url(App $a, $contact_url) {
$sql = "`item`.`uid` = %d";
}
if (!dbm::is_result($r)) {
return '';
}
$author_id = intval($r[0]["author-id"]);
if (get_config('system', 'old_pager')) {
include/ParseUrl.php
+17 -39
View File
@@ -21,13 +21,13 @@ class ParseUrl {
/**
* @brief Search for chached embeddable data of an url otherwise fetch it
*
*
* @param type $url The url of the page which should be scraped
* @param type $no_guessing If true the parse doens't search for
* preview pictures
* @param type $do_oembed The false option is used by the function fetch_oembed()
* to avoid endless loops
*
*
* @return array which contains needed data for embedding
* string 'url' => The url of the parsed page
* string 'type' => Content type
@@ -37,9 +37,9 @@ class ParseUrl {
* if $no_geuessing = false
* array'images' = Array of preview pictures
* string 'keywords' => The tags which belong to the content
*
*
* @see ParseUrl::getSiteinfo() for more information about scraping
* embeddable content
* embeddable content
*/
public static function getSiteinfoCached($url, $no_guessing = false, $do_oembed = true) {
@@ -71,21 +71,21 @@ class ParseUrl {
}
/**
* @brief Parse a page for embeddable content information
*
*
* This method parses to url for meta data which can be used to embed
* the content. If available it prioritizes Open Graph meta tags.
* If this is not available it uses the twitter cards meta tags.
* As fallback it uses standard html elements with meta informations
* like \<title\>Awesome Title\</title\> or
* \<meta name="description" content="An awesome description"\>
*
*
* @param type $url The url of the page which should be scraped
* @param type $no_guessing If true the parse doens't search for
* preview pictures
* @param type $do_oembed The false option is used by the function fetch_oembed()
* to avoid endless loops
* @param type $count Internal counter to avoid endless loops
*
*
* @return array which contains needed data for embedding
* string 'url' => The url of the parsed page
* string 'type' => Content type
@@ -95,13 +95,13 @@ class ParseUrl {
* if $no_geuessing = false
* array'images' = Array of preview pictures
* string 'keywords' => The tags which belong to the content
*
*
* @todo https://developers.google.com/+/plugins/snippet/
* @verbatim
* <meta itemprop="name" content="Awesome title">
* <meta itemprop="description" content="An awesome description">
* <meta itemprop="image" content="http://maple.libertreeproject.org/images/tree-icon.png">
*
*
* <body itemscope itemtype="http://schema.org/Product">
* <h1 itemprop="name">Shiny Trinket</h1>
* <img itemprop="image" src="{image-url}" />
@@ -130,7 +130,7 @@ class ParseUrl {
$url = trim($url, "'");
$url = trim($url, '"');
$url = original_url($url);
$url = strip_tracking_query_params($url);
$siteinfo["url"] = $url;
$siteinfo["type"] = "link";
@@ -142,8 +142,7 @@ class ParseUrl {
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_NOBODY, 1);
curl_setopt($ch, CURLOPT_TIMEOUT, 3);
curl_setopt($ch, CURLOPT_TIMEOUT, 10);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
@@ -151,7 +150,6 @@ class ParseUrl {
$header = curl_exec($ch);
$curl_info = @curl_getinfo($ch);
$http_code = $curl_info["http_code"];
curl_close($ch);
$a->save_timestamp($stamp1, "network");
@@ -197,26 +195,6 @@ class ParseUrl {
}
}
$stamp1 = microtime(true);
// Now fetch the body as well
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, 1);
curl_setopt($ch, CURLOPT_NOBODY, 0);
curl_setopt($ch, CURLOPT_TIMEOUT, 10);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, (($check_cert) ? 2 : false));
$header = curl_exec($ch);
$curl_info = @curl_getinfo($ch);
$http_code = $curl_info["http_code"];
curl_close($ch);
$a->save_timestamp($stamp1, "network");
// Fetch the first mentioned charset. Can be in body or header
$charset = "";
if (preg_match('/charset=(.*?)['."'".'"\s\n]/', $header, $matches)) {
@@ -476,7 +454,7 @@ class ParseUrl {
/**
* @brief Convert tags from CSV to an array
*
*
* @param string $string Tags
* @return array with formatted Hashtags
*/
@@ -492,9 +470,9 @@ class ParseUrl {
/**
* @brief Add a hasht sign to a string
*
*
* This method is used as callback function
*
*
* @param string $tag The pure tag name
* @param int $k Counter for internal use
*/
@@ -504,16 +482,16 @@ class ParseUrl {
/**
* @brief Add a scheme to an url
*
*
* The src attribute of some html elements (e.g. images)
* can miss the scheme so we need to add the correct
* scheme
*
*
* @param string $url The url which possibly does have
* a missing scheme (a link to an image)
* @param string $scheme The url with a correct scheme
* (e.g. the url from the webpage which does contain the image)
*
*
* @return string The url with a scheme
*/
private static function completeUrl($url, $scheme) {
include/api.php
+2 -2
View File
@@ -626,7 +626,7 @@ use \Friendica\Core\Config;
// count friends
$r = q("SELECT count(*) as `count` FROM `contact`
WHERE `uid` = %d AND `rel` IN ( %d, %d )
AND `self`=0 AND NOT `blocked` AND `hidden`=0",
AND `self`=0 AND NOT `blocked` AND NOT `pending` AND `hidden`=0",
intval($uinfo[0]['uid']),
intval(CONTACT_IS_SHARING),
intval(CONTACT_IS_FRIEND)
@@ -635,7 +635,7 @@ use \Friendica\Core\Config;
$r = q("SELECT count(*) as `count` FROM `contact`
WHERE `uid` = %d AND `rel` IN ( %d, %d )
AND `self`=0 AND NOT `blocked` AND `hidden`=0",
AND `self`=0 AND NOT `blocked` AND NOT `pending` AND `hidden`=0",
intval($uinfo[0]['uid']),
intval(CONTACT_IS_FOLLOWER),
intval(CONTACT_IS_FRIEND)
include/diaspora.php
+13 -6
View File
@@ -8,6 +8,8 @@
* This will change in the future.
*/
use \Friendica\Core\Config;
require_once("include/items.php");
require_once("include/bb2diaspora.php");
require_once("include/Scrape.php");
@@ -309,10 +311,6 @@ class Diaspora {
return false;
}
// Use a dummy importer to import the data for the public copy
$importer = array("uid" => 0, "page-flags" => PAGE_FREELOVE);
$message_id = self::dispatch($importer,$msg);
// Now distribute it to the followers
$r = q("SELECT `user`.* FROM `user` WHERE `user`.`uid` IN
(SELECT `contact`.`uid` FROM `contact` WHERE `contact`.`network` = '%s' AND `contact`.`addr` = '%s')
@@ -320,13 +318,22 @@ class Diaspora {
dbesc(NETWORK_DIASPORA),
dbesc($msg["author"])
);
if ($r) {
if (dbm::is_result($r)) {
foreach ($r as $rr) {
logger("delivering to: ".$rr["username"]);
self::dispatch($rr,$msg);
}
} else {
logger("No subscribers for ".$msg["author"]." ".print_r($msg, true), LOGGER_DEBUG);
$social_relay = (bool)Config::get('system', 'relay_subscribe', false);
// Use a dummy importer to import the data for the public copy
if ($social_relay) {
$importer = array("uid" => 0, "page-flags" => PAGE_FREELOVE);
$message_id = self::dispatch($importer,$msg);
} else {
logger("Unwanted message from ".$msg["author"]." send by ".$_SERVER["REMOTE_ADDR"]." with ".$_SERVER["HTTP_USER_AGENT"].": ".print_r($msg, true), LOGGER_DEBUG);
}
}
return $message_id;
include/identity.php
+4 -1
View File
@@ -374,7 +374,10 @@ function profile_sidebar($profile, $block = 0) {
if (dbm::is_result($r))
$updated = date("c", strtotime($r[0]['updated']));
$r = q("SELECT COUNT(*) AS `total` FROM `contact` WHERE `uid` = %d AND NOT `self` AND NOT `blocked` AND NOT `hidden` AND NOT `archive`
$r = q("SELECT COUNT(*) AS `total` FROM `contact`
WHERE `uid` = %d
AND NOT `self` AND NOT `blocked` AND NOT `pending`
AND NOT `hidden` AND NOT `archive`
AND `network` IN ('%s', '%s', '%s', '')",
intval($profile['uid']),
dbesc(NETWORK_DFRN),
include/items.php
+9 -4
View File
@@ -857,10 +857,15 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
}
// Now we store the data in the spool directory
$file = 'item-'.round(microtime(true) * 10000).".msg";
$spool = get_spoolpath().'/'.$file;
file_put_contents($spool, json_encode($arr));
logger("Item wasn't stored - Item was spooled into file ".$file, LOGGER_DEBUG);
// We use "microtime" to keep the arrival order and "mt_rand" to avoid duplicates
$file = 'item-'.round(microtime(true) * 10000).'-'.mt_rand().'.msg';
$spoolpath = get_spoolpath();
if ($spoolpath != "") {
$spool = $spoolpath.'/'.$file;
file_put_contents($spool, json_encode($arr));
logger("Item wasn't stored - Item was spooled into file ".$file, LOGGER_DEBUG);
}
return 0;
}
include/network.php
+53 -24
View File
@@ -11,11 +11,11 @@ require_once('include/Probe.php');
/**
* @brief Curl wrapper
*
*
* If binary flag is true, return binary results.
* Set the cookiejar argument to a string (e.g. "/tmp/friendica-cookies.txt")
* to preserve cookies from one request to the next.
*
*
* @param string $url URL to fetch
* @param boolean $binary default false
* TRUE if asked to return binary results (file download)
@@ -23,7 +23,7 @@ require_once('include/Probe.php');
* @param integer $timeout Timeout in seconds, default system config value or 60 seconds
* @param string $accept_content supply Accept: header with 'accept_content' as the value
* @param string $cookiejar Path to cookie jar file
*
*
* @return string The fetched content
*/
function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_content=Null, $cookiejar = 0) {
@@ -220,13 +220,13 @@ function z_fetch_url($url,$binary = false, &$redirects = 0, $opts=array()) {
/**
* @brief Post request to $url
*
*
* @param string $url URL to post
* @param mixed $params
* @param string $headers HTTP headers
* @param integer $redirects Recursion counter for internal use - default = 0
* @param integer $timeout The timeout in seconds, default system config value or 60 seconds
*
*
* @return string The content
*/
function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0) {
@@ -387,10 +387,10 @@ function http_status_exit($val, $description = array()) {
/**
* @brief Check URL to se if ts's real
*
*
* Take a URL from the wild, prepend http:// if necessary
* and check DNS to see if it's real (or check if is a valid IP address)
*
*
* @param string $url The URL to be validated
* @return boolean True if it's a valid URL, fals if something wrong with it
*/
@@ -417,7 +417,7 @@ function validate_url(&$url) {
/**
* @brief Checks that email is an actual resolvable internet address
*
*
* @param string $addr The email address
* @return boolean True if it's a valid email address, false if it's not
*/
@@ -438,10 +438,10 @@ function validate_email($addr) {
/**
* @brief Check if URL is allowed
*
*
* Check $url against our list of allowed sites,
* wildcards allowed. If allowed_sites is unset return true;
*
*
* @param string $url URL which get tested
* @return boolean True if url is allowed otherwise return false
*/
@@ -483,9 +483,9 @@ function allowed_url($url) {
/**
* @brief Check if email address is allowed to register here.
*
*
* Compare against our list (wildcards allowed).
*
*
* @param type $email
* @return boolean False if not allowed, true if allowed
* or if allowed list is not configured
@@ -672,42 +672,71 @@ function fix_contact_ssl_policy(&$contact,$new_policy) {
}
}
function original_url($url, $depth=1, $fetchbody = false) {
$a = get_app();
// Remove Analytics Data from Google and other tracking platforms
/**
* @brief Remove Google Analytics and other tracking platforms params from URL
*
* @param string $url Any user-submitted URL that may contain tracking params
* @return string The same URL stripped of tracking parameters
*/
function strip_tracking_query_params($url)
{
$urldata = parse_url($url);
if (is_string($urldata["query"])) {
$query = $urldata["query"];
parse_str($query, $querydata);
if (is_array($querydata))
foreach ($querydata AS $param=>$value)
if (is_array($querydata)) {
foreach ($querydata AS $param => $value) {
if (in_array($param, array("utm_source", "utm_medium", "utm_term", "utm_content", "utm_campaign",
"wt_mc", "pk_campaign", "pk_kwd", "mc_cid", "mc_eid",
"fb_action_ids", "fb_action_types", "fb_ref",
"awesm", "wtrid",
"woo_campaign", "woo_source", "woo_medium", "woo_content", "woo_term"))) {
$pair = $param."=".urlencode($value);
$pair = $param . "=" . urlencode($value);
$url = str_replace($pair, "", $url);
// Second try: if the url isn't encoded completely
$pair = $param."=".str_replace(" ", "+", $value);
$pair = $param . "=" . str_replace(" ", "+", $value);
$url = str_replace($pair, "", $url);
// Third try: Maybey the url isn't encoded at all
$pair = $param."=".$value;
$pair = $param . "=" . $value;
$url = str_replace($pair, "", $url);
$url = str_replace(array("?&", "&&"), array("?", ""), $url);
}
}
}
if (substr($url, -1, 1) == "?")
if (substr($url, -1, 1) == "?") {
$url = substr($url, 0, -1);
}
}
return $url;
}
/**
* @brief Returns the original URL of the provided URL
*
* This function strips tracking query params and follows redirections, either
* through HTTP code or meta refresh tags. Stops after 10 redirections.
*
* @todo Remove the $fetchbody parameter that generates an extraneous HEAD request
*
* @see ParseUrl::getSiteinfo
*
* @param string $url A user-submitted URL
* @param int $depth The current redirection recursion level (internal)
* @param bool $fetchbody Wether to fetch the body or not after the HEAD requests
* @return string A canonical URL
*/
function original_url($url, $depth = 1, $fetchbody = false) {
$a = get_app();
$url = strip_tracking_query_params($url);
if ($depth > 10)
return($url);
@@ -823,7 +852,7 @@ function short_link($url) {
/**
* @brief Encodes content to json
*
*
* This function encodes an array to json format
* and adds an application/json HTTP header to the output.
* After finishing the process is getting killed.
include/spool_post.php
+27 -1
View File
@@ -27,15 +27,41 @@ function spool_post_run($argv, $argc) {
$path = get_spoolpath();
if (is_writable($path)){
if (($path != '') AND is_writable($path)){
if ($dh = opendir($path)) {
while (($file = readdir($dh)) !== false) {
// It is not named like a spool file, so we don't care.
if (substr($file, 0, 5) != "item-") {
continue;
}
$fullfile = $path."/".$file;
// We don't care about directories either
if (filetype($fullfile) != "file") {
continue;
}
// We can't read or write the file? So we don't care about it.
if (!is_writable($fullfile) OR !is_readable($fullfile)) {
continue;
}
$arr = json_decode(file_get_contents($fullfile), true);
// If it isn't an array then it is no spool file
if (!is_array($arr)) {
continue;
}
// Skip if it doesn't seem to be an item array
if (!isset($arr['uid']) AND !isset($arr['uri']) AND !isset($arr['network'])) {
continue;
}
$result = item_store($arr);
logger("Spool file ".$file." stored: ".$result, LOGGER_DEBUG);
unlink($fullfile);
}
include/text.php
+4 -3
View File
@@ -875,7 +875,7 @@ function contact_block() {
return $o;
$r = q("SELECT COUNT(*) AS `total` FROM `contact`
WHERE `uid` = %d AND NOT `self` AND NOT `blocked`
AND NOT `hidden` AND NOT `archive`
AND NOT `pending` AND NOT `hidden` AND NOT `archive`
AND `network` IN ('%s', '%s', '%s')",
intval($a->profile['uid']),
dbesc(NETWORK_DFRN),
@@ -893,8 +893,9 @@ function contact_block() {
// Splitting the query in two parts makes it much faster
$r = q("SELECT `id` FROM `contact`
WHERE `uid` = %d AND NOT `self` AND NOT `blocked`
AND NOT `hidden` AND NOT `archive`
AND `network` IN ('%s', '%s', '%s') ORDER BY RAND() LIMIT %d",
AND NOT `pending` AND NOT `hidden` AND NOT `archive`
AND `network` IN ('%s', '%s', '%s')
ORDER BY RAND() LIMIT %d",
intval($a->profile['uid']),
dbesc(NETWORK_DFRN),
dbesc(NETWORK_OSTATUS),