From 153cb6d30510f9804fe596ac6777bc2f35111ae2 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Thu, 17 Aug 2017 14:40:21 +0300 Subject: [PATCH] add support for http 304 not modified (no timestamp calculation bullshit like last time) --- classes/rssutils.php | 31 ++++++++++++++++-- include/functions.php | 61 +++++++++++++++++++++++++---------- schema/ttrss_schema_mysql.sql | 3 +- schema/ttrss_schema_pgsql.sql | 3 +- schema/versions/mysql/132.sql | 10 ++++++ schema/versions/pgsql/132.sql | 10 ++++++ 6 files changed, 96 insertions(+), 22 deletions(-) create mode 100644 schema/versions/mysql/132.sql create mode 100644 schema/versions/pgsql/132.sql diff --git a/classes/rssutils.php b/classes/rssutils.php index b6693b8c..460709f0 100644 --- a/classes/rssutils.php +++ b/classes/rssutils.php @@ -292,7 +292,8 @@ class RSSUtils { $result = db_query("SELECT id,update_interval,auth_login, feed_url,auth_pass,cache_images, mark_unread_on_update, owner_uid, - auth_pass_encrypted, feed_language + auth_pass_encrypted, feed_language, + last_modified FROM ttrss_feeds WHERE id = '$feed'"); $owner_uid = db_fetch_result($result, 0, "owner_uid"); @@ -312,6 +313,7 @@ class RSSUtils { $auth_pass = decrypt_string($auth_pass); } + $stored_last_modified = db_fetch_result($result, 0, "last_modified"); $cache_images = sql_bool_to_bool(db_fetch_result($result, 0, "cache_images")); $fetch_url = db_fetch_result($result, 0, "feed_url"); $feed_language = db_escape_string(mb_strtolower(db_fetch_result($result, 0, "feed_language"))); @@ -359,18 +361,33 @@ class RSSUtils { _debug("local cache will not be used for this feed", $debug_enabled); } + global $fetch_last_modified; + // fetch feed from source if (!$feed_data) { + _debug("stored last modified: $stored_last_modified", $debug_enabled); _debug("fetching [$fetch_url]...", $debug_enabled); if (ini_get("open_basedir") && function_exists("curl_init")) { _debug("not using CURL due to open_basedir restrictions"); } - $feed_data = fetch_file_contents($fetch_url, false, + /*$feed_data = fetch_file_contents($fetch_url, false, $auth_login, $auth_pass, false, $no_cache ? FEED_FETCH_NO_CACHE_TIMEOUT : FEED_FETCH_TIMEOUT, - 0); + 0);*/ + + // TODO: last_modified should be limited, if the feed has not been updated for a while + // we probably should force one update without the header + // unfortunately last_updated gets bumped on http 304 so that daemon would work properly + + $feed_data = fetch_file_contents([ + "url" => $fetch_url, + "login" => $auth_login, + "pass" => $auth_pass, + "timeout" => $no_cache ? FEED_FETCH_NO_CACHE_TIMEOUT : FEED_FETCH_TIMEOUT, + "last_modified" => $force_refetch ? "" : $stored_last_modified + ]); global $fetch_curl_used; @@ -383,6 +400,14 @@ class RSSUtils { $feed_data = trim($feed_data); _debug("fetch done.", $debug_enabled); + _debug("source last modified: " . $fetch_last_modified); + + if ($feed_data && $fetch_last_modified != $stored_last_modified) { + $last_modified_escaped = db_escape_string(substr($fetch_last_modified, 0, 245)); + + db_query("UPDATE ttrss_feeds SET last_modified = '$last_modified_escaped' WHERE id = '$feed'"); + + } // cache vanilla feed data for re-use if ($feed_data && !$auth_pass && !$auth_login && is_writable(CACHE_DIR . "/simplepie")) { diff --git a/include/functions.php b/include/functions.php index dc2e9f64..8ea8719c 100644 --- a/include/functions.php +++ b/include/functions.php @@ -1,6 +1,6 @@ array( 'method' => 'GET', 'ignore_errors' => true, 'timeout' => $timeout ? $timeout : FILE_FETCH_TIMEOUT, 'protocol_version'=> 1.1, - 'header' => "If-Modified-Since: ".gmdate("D, d M Y H:i:s \\G\\M\\T\r\n", $timestamp) - ))); + 'header' => "If-Modified-Since: $last_modified\r\n") + )); } else { $context = stream_context_create(array( 'http' => array( @@ -497,10 +518,16 @@ if (isset($http_response_header) && is_array($http_response_header)) { foreach ($http_response_header as $h) { - if (substr(strtolower($h), 0, 13) == 'content-type:') { - $fetch_last_content_type = substr($h, 14); + list ($key, $value) = explode(": ", $h); + + $key = strtolower($key); + + if ($key == 'content-type') { + $fetch_last_content_type = $value; // don't abort here b/c there might be more than one // e.g. if we were being redirected -- last one is the right one + } else if ($key == 'last-modified') { + $fetch_last_modified = $value; } if (substr(strtolower($h), 0, 7) == 'http/1.') { diff --git a/schema/ttrss_schema_mysql.sql b/schema/ttrss_schema_mysql.sql index 1ec40966..67a22e6d 100644 --- a/schema/ttrss_schema_mysql.sql +++ b/schema/ttrss_schema_mysql.sql @@ -103,6 +103,7 @@ create table ttrss_feeds (id integer not null auto_increment primary key, purge_interval integer not null default 0, last_updated datetime default null, last_error varchar(250) not null default '', + last_modified varchar(250) not null default '', favicon_avg_color varchar(11) default null, site_url varchar(250) not null default '', auth_login varchar(250) not null default '', @@ -282,7 +283,7 @@ create table ttrss_tags (id integer primary key auto_increment, create table ttrss_version (schema_version int not null) ENGINE=InnoDB DEFAULT CHARSET=UTF8; -insert into ttrss_version values (131); +insert into ttrss_version values (132); create table ttrss_enclosures (id integer primary key auto_increment, content_url text not null, diff --git a/schema/ttrss_schema_pgsql.sql b/schema/ttrss_schema_pgsql.sql index 6db73cd5..7870b4b3 100644 --- a/schema/ttrss_schema_pgsql.sql +++ b/schema/ttrss_schema_pgsql.sql @@ -73,6 +73,7 @@ create table ttrss_feeds (id serial not null primary key, purge_interval integer not null default 0, last_updated timestamp default null, last_error text not null default '', + last_modified text not null default '', favicon_avg_color varchar(11) default null, site_url varchar(250) not null default '', auth_login varchar(250) not null default '', @@ -264,7 +265,7 @@ create index ttrss_tags_post_int_id_idx on ttrss_tags(post_int_id); create table ttrss_version (schema_version int not null); -insert into ttrss_version values (131); +insert into ttrss_version values (132); create table ttrss_enclosures (id serial not null primary key, content_url text not null, diff --git a/schema/versions/mysql/132.sql b/schema/versions/mysql/132.sql new file mode 100644 index 00000000..1f1fb590 --- /dev/null +++ b/schema/versions/mysql/132.sql @@ -0,0 +1,10 @@ +begin; + +alter table ttrss_feeds add column last_modified varchar(250); +update ttrss_feeds set last_modified = ''; +alter table ttrss_feeds change last_modified last_modified varchar(250) not null; +alter table ttrss_feeds alter column last_modified set default ''; + +UPDATE ttrss_version SET schema_version = 132; + +commit; diff --git a/schema/versions/pgsql/132.sql b/schema/versions/pgsql/132.sql new file mode 100644 index 00000000..e6b4bd13 --- /dev/null +++ b/schema/versions/pgsql/132.sql @@ -0,0 +1,10 @@ +begin; + +alter table ttrss_feeds add column last_modified text; +update ttrss_feeds set last_modified = ''; +alter table ttrss_feeds alter column last_modified set not null; +alter table ttrss_feeds alter column last_modified set default ''; + +UPDATE ttrss_version SET schema_version = 132; + +commit;