blallo/tt-rss
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

implement hard limits on downloaded data size for general fetching and cache plugins: MAX_DOWNLOAD_FILE_SIZE & MAX_CACHE_FILE_SIZE

Browse source
This commit is contained in:
Andrew Dolgov 2018-05-20 11:08:33 +03:00
parent f851cdf362
commit b14f6d58b4
3 changed files with 22 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
include/functions.php
View file

@ -56,6 +56,10 @@
// default sleep interval between feed updates (sec) // default sleep interval between feed updates (sec)
define_default('MIN_CACHE_FILE_SIZE', 1024); define_default('MIN_CACHE_FILE_SIZE', 1024);
// do not cache files smaller than that (bytes) // do not cache files smaller than that (bytes)
define_default('MAX_CACHE_FILE_SIZE', 64*1024*1024);
// do not cache files larger than that (bytes)
define_default('MAX_DOWNLOAD_FILE_SIZE', 16*1024*1024);
// do not download general files larger than that (bytes)
define_default('CACHE_MAX_DAYS', 7); define_default('CACHE_MAX_DAYS', 7);
// max age in days for various automatically cached (temporary) files // max age in days for various automatically cached (temporary) files
define_default('MAX_CONDITIONAL_INTERVAL', 3600*12); define_default('MAX_CONDITIONAL_INTERVAL', 3600*12);
@ -317,6 +321,7 @@
} }
} }
// TODO: max_size currently only works for CURL transfers
// TODO: multiple-argument way is deprecated, first parameter is a hash now // TODO: multiple-argument way is deprecated, first parameter is a hash now
function fetch_file_contents($options /* previously: 0: $url , 1: $type = false, 2: $login = false, 3: $pass = false, function fetch_file_contents($options /* previously: 0: $url , 1: $type = false, 2: $login = false, 3: $pass = false,
4: $post_query = false, 5: $timeout = false, 6: $timestamp = 0, 7: $useragent = false*/) { 4: $post_query = false, 5: $timeout = false, 6: $timestamp = 0, 7: $useragent = false*/) {
@ -370,6 +375,7 @@
$last_modified = isset($options["last_modified"]) ? $options["last_modified"] : ""; $last_modified = isset($options["last_modified"]) ? $options["last_modified"] : "";
$useragent = isset($options["useragent"]) ? $options["useragent"] : false; $useragent = isset($options["useragent"]) ? $options["useragent"] : false;
$followlocation = isset($options["followlocation"]) ? $options["followlocation"] : true; $followlocation = isset($options["followlocation"]) ? $options["followlocation"] : true;
$max_size = isset($options["max_size"]) ? $options["max_size"] : MAX_DOWNLOAD_FILE_SIZE; // in bytes
$url = ltrim($url, ' '); $url = ltrim($url, ' ');
$url = str_replace(' ', '%20', $url); $url = str_replace(' ', '%20', $url);
@ -401,6 +407,20 @@
curl_setopt($ch, CURLOPT_ENCODING, ""); curl_setopt($ch, CURLOPT_ENCODING, "");
//curl_setopt($ch, CURLOPT_REFERER, $url); //curl_setopt($ch, CURLOPT_REFERER, $url);
if ($max_size) {
curl_setopt($ch, CURLOPT_NOPROGRESS, false);
curl_setopt($ch, CURLOPT_BUFFERSIZE, 128); // needed to get 5 arguments in progress function?
// holy shit closures in php
// download & upload are *expected* sizes respectively, could be zero
curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, function($curl_handle, $download_size, $downloaded, $upload_size, $uploaded) use( &$max_size) {
//_debug("[curl progressfunction] $downloaded $max_size");
return ($downloaded > $max_size) ? 1 : 0; // if max size is set, abort when exceeding it
});
}
if (!ini_get("open_basedir")) { if (!ini_get("open_basedir")) {
curl_setopt($ch, CURLOPT_COOKIEJAR, "/dev/null"); curl_setopt($ch, CURLOPT_COOKIEJAR, "/dev/null");
} }

2
plugins/af_zz_imgproxy/init.php Normal file → Executable file
View file

@ -61,7 +61,7 @@ class Af_Zz_ImgProxy extends Plugin {
send_local_file($local_filename); send_local_file($local_filename);
} else { } else {
$data = fetch_file_contents(array("url" => $url)); $data = fetch_file_contents(["url" => $url, "max_size" => MAX_CACHE_FILE_SIZE]);
if ($data) { if ($data) {

2
plugins/cache_starred_images/init.php Normal file → Executable file
View file

@ -194,7 +194,7 @@ class Cache_Starred_Images extends Plugin implements IHandler {
//_debug("cache_images: downloading: $src to $local_filename"); //_debug("cache_images: downloading: $src to $local_filename");
if (!file_exists($local_filename)) { if (!file_exists($local_filename)) {
$file_content = fetch_file_contents($src); $file_content = fetch_file_contents(["url" => $src, "max_size" => MAX_CACHE_FILE_SIZE]);
if ($file_content && strlen($file_content) > MIN_CACHE_FILE_SIZE) { if ($file_content && strlen($file_content) > MIN_CACHE_FILE_SIZE) {
file_put_contents($local_filename, $file_content); file_put_contents($local_filename, $file_content);