From 02008cb19ac99727889cca05c7eddfbeef30d684 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Wed, 23 Jan 2008 10:19:36 +0100 Subject: [PATCH] add multiprocess update daemon --- functions.php | 13 ++- update_daemon2.php | 55 +++++++++++++ update_daemon2_client.php | 161 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 226 insertions(+), 3 deletions(-) create mode 100644 update_daemon2.php create mode 100644 update_daemon2_client.php diff --git a/functions.php b/functions.php index 919cd926..9bd98795 100644 --- a/functions.php +++ b/functions.php @@ -468,12 +468,19 @@ _debug("update_rss_feed: start"); } - $result = db_query($link, "SELECT id,update_interval,auth_login,auth_pass,cache_images - FROM ttrss_feeds WHERE id = '$feed'"); + if (DB_TYPE == "pgsql") { + $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started >= NOW() - INTERVAL '120 seconds')"; + } else { + $updstart_thresh_qpart = "(ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started >= DATE_SUB(NOW(), INTERVAL 120 SECOND))"; + } + + $result = db_query($link, "SELECT id,update_interval,auth_login, + auth_pass,cache_images + FROM ttrss_feeds WHERE id = '$feed' AND $updstart_thresh_qpart"); if (db_num_rows($result) == 0) { if (defined('DAEMON_EXTENDED_DEBUG') || $_GET['xdebug']) { - _debug("update_rss_feed: feed $feed [$feed_url] NOT FOUND"); + _debug("update_rss_feed: feed $feed [$feed_url] NOT FOUND/SKIPPED"); } return; } diff --git a/update_daemon2.php b/update_daemon2.php new file mode 100644 index 00000000..3721a097 --- /dev/null +++ b/update_daemon2.php @@ -0,0 +1,55 @@ +#!/usr/bin/php + 0) $running_jobs--; + print posix_getpid() . ": SIGCHLD received, jobs left: $running_jobs\n"; + pcntl_waitpid(-1, $status, WNOHANG); + } + + pcntl_signal(SIGCHLD, 'sigchld_handler'); + + while (true) { + + $next_spawn = $last_checkpoint + SPAWN_INTERVAL - time(); + + print "[MASTER] active jobs: $running_jobs, next spawn at $next_spawn sec\n"; + + if ($last_checkpoint + SPAWN_INTERVAL < time()) { + + for ($j = $running_jobs; $j < MAX_JOBS; $j++) { + print "[MASTER] spawning client $j..."; + $pid = pcntl_fork(); + if ($pid == -1) { + die("fork failed!\n"); + } else if ($pid) { + $running_jobs++; + print "OK [$running_jobs]\n"; + } else { + pcntl_signal(SIGCHLD, SIG_IGN); + passthru(CLIENT_PROCESS); + exit(0); + } + } + $last_checkpoint = time(); + } + sleep(1); + } + +?> diff --git a/update_daemon2_client.php b/update_daemon2_client.php new file mode 100644 index 00000000..2d069c37 --- /dev/null +++ b/update_daemon2_client.php @@ -0,0 +1,161 @@ +#!/usr/bin/php + PURGE_INTERVAL) { + _debug("Purging old posts (random 30 feeds)..."); + global_purge_old_posts($link, true, 30); + $last_purge = time(); + } + + // Process all other feeds using last_updated and interval parameters + + $random_qpart = sql_random_function(); + + if (DAEMON_UPDATE_LOGIN_LIMIT > 0) { + if (DB_TYPE == "pgsql") { + $login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".DAEMON_UPDATE_LOGIN_LIMIT." days'"; + } else { + $login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".DAEMON_UPDATE_LOGIN_LIMIT." DAY)"; + } + } else { + $login_thresh_qpart = ""; + } + + if (DB_TYPE == "pgsql") { + $update_limit_qpart = "AND ttrss_feeds.last_updated < NOW() - INTERVAL '".(DAEMON_SLEEP_INTERVAL*2)." seconds'"; + } else { + $update_limit_qpart = "AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ".(DAEMON_SLEEP_INTERVAL*2)." SECOND)"; + } + + if (DB_TYPE == "pgsql") { + $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started >= NOW() - INTERVAL '120 seconds')"; + } else { + $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started >= DATE_SUB(NOW(), INTERVAL 120 SECOND))"; + } + + $result = db_query($link, "SELECT feed_url,ttrss_feeds.id,owner_uid, + SUBSTRING(last_updated,1,19) AS last_updated, + update_interval + FROM + ttrss_feeds,ttrss_users + WHERE + ttrss_users.id = owner_uid $login_thresh_qpart $update_limit_qpart + $updstart_thresh_qpart + ORDER BY $random_qpart DESC LIMIT " . DAEMON_FEED_LIMIT); + + $user_prefs_cache = array(); + + _debug(sprintf("Scheduled %d feeds to update...\n", db_num_rows($result))); + + while ($line = db_fetch_assoc($result)) { + + $upd_intl = $line["update_interval"]; + $user_id = $line["owner_uid"]; + + if (!$upd_intl || $upd_intl == 0) { + if (!$user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL']) { + $upd_intl = get_pref($link, 'DEFAULT_UPDATE_INTERVAL', $user_id); + $user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL'] = $upd_intl; + } else { + $upd_intl = $user_prefs_cache[$user_id]['DEFAULT_UPDATE_INTERVAL']; + } + } + + if ($upd_intl < 0) { +# print "Updates disabled.\n"; + continue; + } + + _debug("Feed: " . $line["feed_url"] . ", " . $line["last_updated"]); + +// _debug(sprintf("\tLU: %d, INTL: %d, UID: %d) ", +// time() - strtotime($line["last_updated"]), $upd_intl*60, $user_id)); + + if (!$line["last_updated"] || + time() - strtotime($line["last_updated"]) > ($upd_intl * 60)) { + + _debug("Updating..."); + + pcntl_alarm(300); + + update_rss_feed($link, $line["feed_url"], $line["id"], true); + + pcntl_alarm(0); + + sleep(1); // prevent flood (FIXME make this an option?) + } else { + _debug("Update not needed."); + } + } + + if (DAEMON_SENDS_DIGESTS) send_headlines_digests($link); + + db_close($link); +?>