2012-06-20 07:35:34 +02:00
#!/usr/bin/env php
2007-02-24 17:46:39 +01:00
< ? php
2012-12-09 10:41:22 +01:00
set_include_path ( dirname ( __FILE__ ) . " /include " . PATH_SEPARATOR .
get_include_path ());
2011-12-11 20:59:25 +01:00
2010-11-09 14:33:08 +01:00
define ( 'DISABLE_SESSIONS' , true );
2011-04-15 10:38:11 +02:00
chdir ( dirname ( __FILE__ ));
2013-04-17 13:36:34 +02:00
require_once " autoload.php " ;
2010-11-10 11:14:44 +01:00
require_once " functions.php " ;
2011-12-13 13:06:25 +01:00
require_once " rssfuncs.php " ;
2007-03-01 09:54:55 +01:00
require_once " config.php " ;
2013-03-15 12:56:40 +01:00
require_once " sanity_check.php " ;
2007-03-01 09:54:55 +01:00
require_once " db.php " ;
2010-11-09 14:33:08 +01:00
require_once " db-prefs.php " ;
2012-10-23 22:25:56 +02:00
if ( ! defined ( 'PHP_EXECUTABLE' ))
define ( 'PHP_EXECUTABLE' , '/usr/bin/php' );
2013-04-17 14:23:15 +02:00
init_plugins ();
2012-12-24 06:52:15 +01:00
2013-03-21 11:48:47 +01:00
$longopts = array ( " feeds " ,
" feedbrowser " ,
" daemon " ,
" daemon-loop " ,
" task: " ,
" cleanup-tags " ,
" quiet " ,
2013-03-21 11:56:04 +01:00
" log: " ,
2013-03-21 11:48:47 +01:00
" indexes " ,
2013-04-17 16:19:26 +02:00
" pidlock: " ,
2013-04-04 17:15:14 +02:00
" update-schema " ,
2013-03-21 11:48:47 +01:00
" convert-filters " ,
" force-update " ,
2015-08-04 12:32:52 +02:00
" gen-search-idx " ,
2013-03-21 11:48:47 +01:00
" list-plugins " ,
2016-01-17 12:24:20 +01:00
" debug-feed: " ,
" force-refetch " ,
" force-rehash " ,
2017-01-07 12:25:46 +01:00
" decrypt-feeds " ,
2013-03-21 11:48:47 +01:00
" help " );
2013-04-18 10:27:34 +02:00
foreach ( PluginHost :: getInstance () -> get_commands () as $command => $data ) {
2013-03-28 15:37:36 +01:00
array_push ( $longopts , $command . $data [ " suffix " ]);
2013-03-21 11:48:47 +01:00
}
$options = getopt ( " " , $longopts );
2012-10-23 22:25:56 +02:00
2014-02-06 20:03:05 +01:00
if ( ! is_array ( $options )) {
die ( " error: getopt() failed. " .
" Most probably you are using PHP CGI to run this script " .
" instead of required PHP CLI. Check tt-rss wiki page on updating feeds for " .
" additional information. \n " );
}
2013-03-21 11:48:47 +01:00
if ( count ( $options ) == 0 && ! defined ( 'STDIN' )) {
2012-08-23 11:17:22 +02:00
?> <html>
< head >
2012-10-23 22:25:56 +02:00
< title > Tiny Tiny RSS data update script .</ title >
2012-08-23 11:17:22 +02:00
< meta http - equiv = " Content-Type " content = " text/html; charset=utf-8 " >
2013-05-19 19:22:01 +02:00
< link rel = " stylesheet " type = " text/css " href = " css/utility.css " >
2012-08-23 11:17:22 +02:00
</ head >
< body >
2013-03-27 07:59:26 +01:00
< div class = " floatingLogo " >< img src = " images/logo_small.png " ></ div >
2012-10-23 22:25:56 +02:00
< h1 >< ? php echo __ ( " Tiny Tiny RSS data update script. " ) ?> </h1>
2012-08-23 11:17:22 +02:00
2017-02-23 11:06:39 +01:00
< ? php print_error ( " Please run this script from the command line. Use option \" --help \" to display command help if this error is displayed erroneously. " ); ?>
2012-08-23 11:17:22 +02:00
</ body ></ html >
< ? php
exit ;
}
2013-03-21 11:48:47 +01:00
if ( count ( $options ) == 0 || isset ( $options [ " help " ]) ) {
2010-11-09 14:33:08 +01:00
print " Tiny Tiny RSS data update script. \n \n " ;
print " Options: \n " ;
2013-03-21 11:48:47 +01:00
print " --feeds - update feeds \n " ;
print " --feedbrowser - update feedbrowser \n " ;
print " --daemon - start single-process update daemon \n " ;
print " --task N - create lockfile using this task id \n " ;
print " --cleanup-tags - perform tags table maintenance \n " ;
2013-03-21 12:05:57 +01:00
print " --quiet - don't output messages to stdout \n " ;
2013-03-21 11:56:04 +01:00
print " --log FILE - log messages to FILE \n " ;
2013-03-21 11:48:47 +01:00
print " --indexes - recreate missing schema indexes \n " ;
2013-04-04 17:15:14 +02:00
print " --update-schema - update database schema \n " ;
2015-08-04 12:32:52 +02:00
print " --gen-search-idx - generate basic PostgreSQL fulltext search index \n " ;
2013-03-21 11:48:47 +01:00
print " --convert-filters - convert type1 filters to type2 \n " ;
print " --force-update - force update of all feeds \n " ;
print " --list-plugins - list all available plugins \n " ;
2016-01-17 12:24:20 +01:00
print " --debug-feed N - perform debug update of feed N \n " ;
print " --force-refetch - debug update: force refetch feed data \n " ;
print " --force-rehash - debug update: force rehash articles \n " ;
2017-01-07 12:25:46 +01:00
print " --decrypt-feeds - decrypt feed passwords \n " ;
2013-03-21 11:48:47 +01:00
print " --help - show this help \n " ;
2012-12-24 06:52:15 +01:00
print " Plugin options: \n " ;
2013-04-18 10:27:34 +02:00
foreach ( PluginHost :: getInstance () -> get_commands () as $command => $data ) {
2013-03-28 15:37:36 +01:00
$args = $data [ 'arghelp' ];
printf ( " --%-19s - %s \n " , " $command $args " , $data [ " description " ]);
2012-12-24 06:52:15 +01:00
}
2010-11-09 14:33:08 +01:00
return ;
2007-03-01 09:54:55 +01:00
}
2007-05-04 05:15:58 +02:00
2013-04-17 10:36:27 +02:00
if ( ! isset ( $options [ 'daemon' ])) {
require_once " errorhandler.php " ;
}
2013-04-17 10:10:35 +02:00
if ( ! isset ( $options [ 'update-schema' ])) {
2013-04-17 14:23:15 +02:00
$schema_version = get_schema_version ();
2013-04-17 10:10:35 +02:00
if ( $schema_version != SCHEMA_VERSION ) {
die ( " Schema version is wrong, please upgrade the database. \n " );
}
}
2013-03-21 12:05:57 +01:00
define ( 'QUIET' , isset ( $options [ 'quiet' ]));
2013-03-21 11:56:04 +01:00
if ( isset ( $options [ " log " ])) {
_debug ( " Logging to " . $options [ " log " ]);
define ( 'LOGFILE' , $options [ " log " ]);
}
2013-03-21 11:48:47 +01:00
if ( ! isset ( $options [ " daemon " ])) {
2010-11-09 14:33:08 +01:00
$lock_filename = " update.lock " ;
} else {
$lock_filename = " update_daemon.lock " ;
}
2007-02-24 17:46:39 +01:00
2013-03-21 11:48:47 +01:00
if ( isset ( $options [ " task " ])) {
_debug ( " Using task id " . $options [ " task " ]);
$lock_filename = $lock_filename . " -task_ " . $options [ " task " ];
}
2013-04-17 16:19:26 +02:00
if ( isset ( $options [ " pidlock " ])) {
$my_pid = $options [ " pidlock " ];
$lock_filename = " update_daemon- $my_pid .lock " ;
}
_debug ( " Lock: $lock_filename " );
2010-11-09 14:33:08 +01:00
$lock_handle = make_lockfile ( $lock_filename );
$must_exit = false ;
2007-02-24 17:46:39 +01:00
2013-04-17 20:09:11 +02:00
if ( isset ( $options [ " task " ]) && isset ( $options [ " pidlock " ])) {
2013-04-17 16:19:26 +02:00
$waits = $options [ " task " ] * 5 ;
_debug ( " Waiting before update ( $waits ) " );
sleep ( $waits );
}
2010-11-09 14:33:08 +01:00
// Try to lock a file in order to avoid concurrent update.
if ( ! $lock_handle ) {
die ( " error: Can't create lockfile ( $lock_filename ). " .
" Maybe another update process is already running. \n " );
}
2007-02-24 17:46:39 +01:00
2013-03-21 11:48:47 +01:00
if ( isset ( $options [ " force-update " ])) {
_debug ( " marking all feeds as needing update... " );
2013-04-17 14:23:15 +02:00
db_query ( " UPDATE ttrss_feeds SET last_update_started = '1970-01-01',
2013-03-21 11:48:47 +01:00
last_updated = '1970-01-01' " );
}
if ( isset ( $options [ " feeds " ])) {
2013-04-17 14:23:15 +02:00
update_daemon_common ();
2013-04-24 12:54:59 +02:00
housekeeping_common ( true );
2011-04-21 06:51:59 +02:00
2013-04-18 10:27:34 +02:00
PluginHost :: getInstance () -> run_hooks ( PluginHost :: HOOK_UPDATE_TASK , " hook_update_task " , $op );
2010-11-09 14:33:08 +01:00
}
2007-02-24 17:46:39 +01:00
2013-03-21 11:48:47 +01:00
if ( isset ( $options [ " feedbrowser " ])) {
2013-04-17 14:23:15 +02:00
$count = update_feedbrowser_cache ();
2010-11-09 14:33:08 +01:00
print " Finished, $count feeds processed. \n " ;
2007-02-24 17:46:39 +01:00
}
2010-11-09 14:33:08 +01:00
2013-03-21 11:48:47 +01:00
if ( isset ( $options [ " daemon " ])) {
2010-11-09 14:33:08 +01:00
while ( true ) {
2013-04-01 17:51:43 +02:00
$quiet = ( isset ( $options [ " quiet " ])) ? " --quiet " : " " ;
2013-09-02 08:54:20 +02:00
$log = isset ( $options [ 'log' ]) ? '--log ' . $options [ 'log' ] : '' ;
2013-04-01 17:51:43 +02:00
2013-09-02 08:54:20 +02:00
passthru ( PHP_EXECUTABLE . " " . $argv [ 0 ] . " --daemon-loop $quiet $log " );
2010-11-09 14:33:08 +01:00
_debug ( " Sleeping for " . DAEMON_SLEEP_INTERVAL . " seconds... " );
sleep ( DAEMON_SLEEP_INTERVAL );
2007-02-24 17:46:39 +01:00
}
2007-03-01 09:54:55 +01:00
}
2007-05-05 13:07:38 +02:00
2013-03-21 11:48:47 +01:00
if ( isset ( $options [ " daemon-loop " ])) {
2010-11-09 14:33:08 +01:00
if ( ! make_stampfile ( 'update_daemon.stamp' )) {
2013-03-30 13:28:21 +01:00
_debug ( " warning: unable to create stampfile \n " );
2010-11-09 14:33:08 +01:00
}
2007-05-05 13:07:38 +02:00
2013-04-24 12:15:46 +02:00
update_daemon_common ( isset ( $options [ " pidlock " ]) ? 50 : DAEMON_FEED_LIMIT );
2013-05-20 12:39:14 +02:00
if ( ! isset ( $options [ " pidlock " ]) || $options [ " task " ] == 0 )
housekeeping_common ( true );
2011-04-12 17:33:12 +02:00
2013-04-24 11:43:40 +02:00
PluginHost :: getInstance () -> run_hooks ( PluginHost :: HOOK_UPDATE_TASK , " hook_update_task " , $op );
2007-02-24 17:46:39 +01:00
}
2013-03-21 11:48:47 +01:00
if ( isset ( $options [ " cleanup-tags " ])) {
2013-04-17 14:23:15 +02:00
$rc = cleanup_tags ( 14 , 50000 );
2012-08-04 20:15:40 +02:00
_debug ( " $rc tags deleted. \n " );
2010-11-13 17:59:42 +01:00
}
2013-03-21 11:48:47 +01:00
if ( isset ( $options [ " indexes " ])) {
2012-08-09 12:20:27 +02:00
_debug ( " PLEASE BACKUP YOUR DATABASE BEFORE PROCEEDING! " );
_debug ( " Type 'yes' to continue. " );
if ( read_stdin () != 'yes' )
exit ;
_debug ( " clearing existing indexes... " );
if ( DB_TYPE == " pgsql " ) {
2013-04-17 14:23:15 +02:00
$result = db_query ( " SELECT relname FROM
2012-08-09 12:20:27 +02:00
pg_catalog . pg_class WHERE relname LIKE 'ttrss_%'
AND relname NOT LIKE '%_pkey'
AND relkind = 'i' " );
} else {
2013-04-17 14:23:15 +02:00
$result = db_query ( " SELECT index_name,table_name FROM
2012-08-09 12:20:27 +02:00
information_schema . statistics WHERE index_name LIKE 'ttrss_%' " );
}
while ( $line = db_fetch_assoc ( $result )) {
if ( DB_TYPE == " pgsql " ) {
$statement = " DROP INDEX " . $line [ " relname " ];
_debug ( $statement );
} else {
$statement = " ALTER TABLE " .
$line [ 'table_name' ] . " DROP INDEX " . $line [ 'index_name' ];
_debug ( $statement );
}
2013-04-17 14:23:15 +02:00
db_query ( $statement , false );
2012-08-09 12:20:27 +02:00
}
_debug ( " reading indexes from schema for: " . DB_TYPE );
$fp = fopen ( " schema/ttrss_schema_ " . DB_TYPE . " .sql " , " r " );
if ( $fp ) {
while ( $line = fgets ( $fp )) {
$matches = array ();
if ( preg_match ( " /^create index ([^ ]+) on ([^ ]+) $ /i " , $line , $matches )) {
$index = $matches [ 1 ];
$table = $matches [ 2 ];
$statement = " CREATE INDEX $index ON $table " ;
_debug ( $statement );
2013-04-17 14:23:15 +02:00
db_query ( $statement );
2012-08-09 12:20:27 +02:00
}
}
fclose ( $fp );
} else {
_debug ( " unable to open schema file. " );
}
_debug ( " all done. " );
}
2013-03-21 11:48:47 +01:00
if ( isset ( $options [ " convert-filters " ])) {
2012-08-30 16:50:56 +02:00
_debug ( " WARNING: this will remove all existing type2 filters. " );
_debug ( " Type 'yes' to continue. " );
if ( read_stdin () != 'yes' )
exit ;
_debug ( " converting filters... " );
2013-04-17 14:23:15 +02:00
db_query ( " DELETE FROM ttrss_filters2 " );
2012-08-30 16:50:56 +02:00
2013-04-17 14:23:15 +02:00
$result = db_query ( " SELECT * FROM ttrss_filters ORDER BY id " );
2012-08-30 16:50:56 +02:00
while ( $line = db_fetch_assoc ( $result )) {
$owner_uid = $line [ " owner_uid " ];
2012-08-31 10:54:37 +02:00
// date filters are removed
if ( $line [ " filter_type " ] != 5 ) {
$filter = array ();
if ( sql_bool_to_bool ( $line [ " cat_filter " ])) {
$feed_id = " CAT: " . ( int ) $line [ " cat_id " ];
} else {
$feed_id = ( int ) $line [ " feed_id " ];
}
2012-08-30 16:50:56 +02:00
2012-08-31 10:54:37 +02:00
$filter [ " enabled " ] = $line [ " enabled " ] ? " on " : " off " ;
$filter [ " rule " ] = array (
json_encode ( array (
" reg_exp " => $line [ " reg_exp " ],
" feed_id " => $feed_id ,
" filter_type " => $line [ " filter_type " ])));
2012-08-30 16:50:56 +02:00
2012-08-31 10:54:37 +02:00
$filter [ " action " ] = array (
json_encode ( array (
" action_id " => $line [ " action_id " ],
" action_param_label " => $line [ " action_param " ],
" action_param " => $line [ " action_param " ])));
2012-08-30 16:50:56 +02:00
2012-08-31 10:54:37 +02:00
// Oh god it's full of hacks
2012-08-31 10:24:13 +02:00
2012-08-31 10:54:37 +02:00
$_REQUEST = $filter ;
$_SESSION [ " uid " ] = $owner_uid ;
2012-08-30 16:50:56 +02:00
2013-04-18 21:19:14 +02:00
$filters = new Pref_Filters ( $_REQUEST );
2012-08-31 10:54:37 +02:00
$filters -> add ();
}
2012-08-30 16:50:56 +02:00
}
}
2013-04-04 17:15:14 +02:00
if ( isset ( $options [ " update-schema " ])) {
_debug ( " checking for updates ( " . DB_TYPE . " )... " );
2013-04-17 19:40:04 +02:00
$updater = new DbUpdater ( Db :: get (), DB_TYPE , SCHEMA_VERSION );
2013-04-04 17:15:14 +02:00
if ( $updater -> isUpdateRequired ()) {
_debug ( " schema update required, version " . $updater -> getSchemaVersion () . " to " . SCHEMA_VERSION );
_debug ( " WARNING: please backup your database before continuing. " );
_debug ( " Type 'yes' to continue. " );
if ( read_stdin () != 'yes' )
exit ;
for ( $i = $updater -> getSchemaVersion () + 1 ; $i <= SCHEMA_VERSION ; $i ++ ) {
_debug ( " performing update up to version $i ... " );
2016-04-26 19:04:24 +02:00
$result = $updater -> performUpdateTo ( $i , false );
2013-04-04 17:15:14 +02:00
_debug ( $result ? " OK! " : " FAILED! " );
if ( ! $result ) return ;
}
} else {
_debug ( " update not required. " );
}
}
2015-08-04 12:32:52 +02:00
if ( isset ( $options [ " gen-search-idx " ])) {
echo " Generating search index (stemming set to English)... \n " ;
2015-08-04 11:52:49 +02:00
2015-08-04 13:31:42 +02:00
$result = db_query ( " SELECT COUNT(id) AS count FROM ttrss_entries WHERE tsvector_combined IS NULL " );
2015-08-04 11:52:49 +02:00
$count = db_fetch_result ( $result , 0 , " count " );
2015-08-04 13:36:39 +02:00
print " Articles to process: $count . \n " ;
2015-08-04 11:52:49 +02:00
2015-08-04 13:36:39 +02:00
$limit = 500 ;
$processed = 0 ;
2015-08-04 11:52:49 +02:00
while ( true ) {
2015-08-04 13:36:39 +02:00
$result = db_query ( " SELECT id, title, content FROM ttrss_entries WHERE tsvector_combined IS NULL ORDER BY id LIMIT $limit " );
2015-08-04 11:52:49 +02:00
2015-08-05 15:58:07 +02:00
while ( $line = db_fetch_assoc ( $result )) {
2015-08-18 11:04:19 +02:00
$tsvector_combined = db_escape_string ( mb_substr ( $line [ 'title' ] . ' ' . strip_tags ( str_replace ( '<' , ' <' , $line [ 'content' ])),
2015-08-05 15:58:07 +02:00
0 , 1000000 ));
2015-08-04 11:52:49 +02:00
2015-08-05 15:58:07 +02:00
db_query ( " UPDATE ttrss_entries SET tsvector_combined = to_tsvector('english', ' $tsvector_combined ') WHERE id = " . $line [ " id " ]);
}
2015-08-04 11:52:49 +02:00
2015-08-05 15:58:07 +02:00
$processed += db_num_rows ( $result );
print " Processed $processed articles... \n " ;
2015-08-04 13:36:39 +02:00
2015-08-05 15:58:07 +02:00
if ( db_num_rows ( $result ) != $limit ) {
2015-08-04 11:52:49 +02:00
echo " All done. \n " ;
break ;
}
}
}
2013-03-21 11:48:47 +01:00
if ( isset ( $options [ " list-plugins " ])) {
2013-05-07 09:35:10 +02:00
$tmppluginhost = new PluginHost ();
2015-10-08 16:02:32 +02:00
$tmppluginhost -> load_all ( $tmppluginhost :: KIND_ALL , false );
2012-12-25 15:57:02 +01:00
$enabled = array_map ( " trim " , explode ( " , " , PLUGINS ));
echo " List of all available plugins: \n " ;
2012-12-24 12:39:42 +01:00
foreach ( $tmppluginhost -> get_plugins () as $name => $plugin ) {
2012-12-25 07:02:08 +01:00
$about = $plugin -> about ();
2012-12-24 12:39:42 +01:00
2012-12-25 15:57:02 +01:00
$status = $about [ 3 ] ? " system " : " user " ;
if ( in_array ( $name , $enabled )) $name .= " * " ;
printf ( " %-50s %-10s v%.2f (by %s) \n %s \n \n " ,
$name , $status , $about [ 0 ], $about [ 2 ], $about [ 1 ]);
2012-12-24 12:39:42 +01:00
}
2012-12-25 15:57:02 +01:00
echo " Plugins marked by * are currently enabled for all users. \n " ;
2012-12-24 12:39:42 +01:00
}
2016-01-17 12:24:20 +01:00
if ( isset ( $options [ " debug-feed " ])) {
$feed = $options [ " debug-feed " ];
if ( isset ( $options [ " force-refetch " ])) $_REQUEST [ " force_refetch " ] = true ;
if ( isset ( $options [ " force-rehash " ])) $_REQUEST [ " force_rehash " ] = true ;
$_REQUEST [ 'xdebug' ] = 1 ;
update_rss_feed ( $feed );
}
2017-01-07 12:25:46 +01:00
if ( isset ( $options [ " decrypt-feeds " ])) {
$result = db_query ( " SELECT id, auth_pass FROM ttrss_feeds WHERE auth_pass_encrypted = true " );
if ( ! function_exists ( " mcrypt_decrypt " )) {
_debug ( " mcrypt functions not available. " );
return ;
}
require_once " crypt.php " ;
$total = 0 ;
db_query ( " BEGIN " );
while ( $line = db_fetch_assoc ( $result )) {
_debug ( " processing feed id " . $line [ " id " ]);
$auth_pass = db_escape_string ( decrypt_string ( $line [ " auth_pass " ]));
2017-02-23 11:06:39 +01:00
db_query ( " UPDATE ttrss_feeds SET auth_pass_encrypted = false, auth_pass = ' $auth_pass '
2017-01-07 12:25:46 +01:00
WHERE id = " . $line["id"] );
++ $total ;
}
db_query ( " COMMIT " );
_debug ( " $total feeds processed. " );
}
2013-04-18 10:27:34 +02:00
PluginHost :: getInstance () -> run_commands ( $options );
2012-12-24 06:52:15 +01:00
2017-01-06 18:06:09 +01:00
if ( file_exists ( LOCK_DIRECTORY . " / $lock_filename " ))
if ( strtoupper ( substr ( PHP_OS , 0 , 3 )) == 'WIN' )
fclose ( $lock_handle );
2013-06-07 07:27:52 +02:00
unlink ( LOCK_DIRECTORY . " / $lock_filename " );
2013-04-12 07:55:41 +02:00
?>
