MastodonHelp/web/clitools/peerscrawl.php

476 lines
18 KiB
PHP
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/php
<?php
/*
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
define('N',"\n");
define('SNAME',basename(__FILE__));
define('BNAME',preg_replace('/\.[^.]*$/','',SNAME));
require(__DIR__.'/../site/mustard/include/gurl.php');
require(__DIR__.'/../site/mustard/include/ghs.php');
require(__DIR__.'/../site/mustard/include/ght.php');
require(__DIR__.'/../site/mustard/include/gracetime.php');
require(__DIR__.'/../site/mustard/include/parsetime.php');
use function mysqli_real_escape_string as myesc;
$opts=[
'inifp'=>__DIR__.'/../conf/mustard.ini',
'startinst'=>'mastodon.social',
'gracetime'=>$gracetime,
'peersfp'=>__DIR__.'/peers',
'apeersfp'=>__DIR__.'/peers.all',
'cpeersfp'=>__DIR__.'/peers.checked',
'excludefp'=>null,
'timeout'=>8,
'curltimeout'=>15,
'loop'=>false,
'excludedead'=>false,
'ignorelock'=>false,
'minmsgimplev'=>1
];
$msgimplevs=['Debug', 'Info', 'Warning', 'Error'];
foreach ($msgimplevs as $val) $imsgimplevs[]=lcfirst($val);
$imsgimplevs[]='none';
$ghtsa=[[' day',' days'],[' hour',' hours'],[' minute',' minutes'],[' second',' seconds']];
$help='SYNOPSIS
'.SNAME.' [options]
DESCRIPTION
This program tries to build a fairly complete list of fediverse instances
exposing the [instance]/api/v1/instance/peers endpoint.
OPTIONS
-s, --startinst <domain>
Defines the first instance to crawl.
DEFAULT: «'.$opts['startinst'].
-e, --excludefp <file>
Defines a file containing exclusion rules: one regular expression per
line (empty lines are ignored). Any instance matching any defined regex
will be ignored by the program. Changes made to this file during program
execution will be taken into account.
-g, --gracetime <time>
If an instance has not been responding for longer than this time, consider
it dead. See section «TIME SPECIFICATION» below to see how to specify time.
DEFAULT: '.ght($opts['gracetime'],$ghtsa).'
-E, --excludedead
Exclude instances marked considered dead (see previous option).
-p, --peersfp <file>
Defines the file into which the ordered list of responding instances
will be saved.
DEFAULT: «'.$opts['peersfp'].
-a, --apeersfp <file>
Defines the file into which the ordered list of all instances will
be saved.
DEFAULT: «'.$opts['apeersfp'].
-c, --cpeersfp <file>
Defines the file into which the ordered list of all checked instances will
be saved.
DEFAULT: «'.$opts['cpeersfp'].
-I, --ignorelock
Normally, if its lockfile exists, the program exits with an error before
doing anything. With this option the lockfile is ignored. Please verify
that the program is not already running before using it.
-t, --timeout <time>
Defines the timeout in seconds for every connection attempt. See section
«TIME SPECIFICATION» below to see how to specify time.
DEFAULT: '.ght($opts['timeout'],$ghtsa).'
-T, --curltimeout <time>
Defines the timeout in seconds for every download. See section «TIME
SPECIFICATION» below to see how to specify time.
DEFAULT: '.ght($opts['curltimeout'],$ghtsa).'
-m, --minmsgimplev <«debug»|«info»|«warning»|«error»|«none»>
Defines the minimum “importance level” of messages to be written to the text
user interface. There are 4 “importance levels”, in this order of
importance: «debug», «info», «warning», «error». Setting this option to any
of these values will write to the text user interface all the messages with
the specified or a greater level; setting it to the special value «none»
will completely disable message writing.
DEFAULT: '.$imsgimplevs[$opts['minmsgimplev']].'
-h, --help
Show this help text and exit.
TIME SPECIFICATION
An example is better than ~5148 words :-)
To specify 1 year, 6 months (made of 31 days), 2 weeks, 3 days, 5 hours,
7 minutes and 12 seconds you can use «1y,6M,2w,3d,5h,7m,12s»; but you can
also use «12s,7m,5h,3d,2w,6M,1y», or even «18M,1w,1w,2d,1d,3h,2h,7m,12s».
This program comes with ABSOLUTELY NO WARRANTY; for details see the source.
This is free software, and you are welcome to redistribute it under certain
conditions; see <http://www.gnu.org/licenses/> for details.'.N;
for ($i=1; $i<$argc; $i++) {
if ($argv[$i]=='-s' || $argv[$i]=='--startinst') {
if ($i+1>=$argc)
mexit(3,'option «'.$argv[$i].'» has to be followed by a domain name (use «-h» for more info).'.N,1,false);
$i++;
$opts['startinst']=$argv[$i];
} elseif ($argv[$i]=='-g' || $argv[$i]=='--gracetime') {
if ($i+1>=$argc || ($time=parsetime($argv[$i+1]))===false)
mexit(3,'option «'.$argv[$i].'» requires a valid time specification as an argument (use «-h» to read help).'.N,1,false);
$i++;
$opts['gracetime']=$time;
} elseif ($argv[$i]=='-p' || $argv[$i]=='--peersfp') {
if ($i+1>=$argc)
mexit(3,'option «'.$argv[$i].'» has to be followed by a files path (use «-h» for more info).'.N,1,false);
$i++;
$opts['peersfp']=$argv[$i];
} elseif ($argv[$i]=='-a' || $argv[$i]=='--apeersfp') {
if ($i+1>=$argc)
mexit(3,'option «'.$argv[$i].'» has to be followed by a files path (use «-h» for more info).'.N,1,false);
$i++;
$opts['apeersfp']=$argv[$i];
} elseif ($argv[$i]=='-c' || $argv[$i]=='--cpeersfp') {
if ($i+1>=$argc)
mexit(3,'option «'.$argv[$i].'» has to be followed by a files path (use «-h» for more info).'.N,1,false);
$i++;
$opts['cpeersfp']=$argv[$i];
} elseif ($argv[$i]=='-I' || $argv[$i]=='--ignorelock') {
$opts['ignorelock']=true;
} elseif ($argv[$i]=='-e' || $argv[$i]=='--excludefp') {
if ($i+1>=$argc)
mexit(3,'option «'.$argv[$i].'» has to be followed by a files path (use «-h» for more info).'.N,1,false);
$i++;
$opts['excludefp']=$argv[$i];
} elseif ($argv[$i]=='-t' || $argv[$i]=='--timeout') {
if ($i+1>=$argc || ($time=parsetime($argv[$i+1]))===false)
mexit(3,'option «'.$argv[$i].'» requires a valid time specification as an argument (use «-h» to read help).'.N,1,false);
$i++;
$opts['timeout']=$time;
} elseif ($argv[$i]=='-T' || $argv[$i]=='--curltimeout') {
if ($i+1>=$argc || ($time=parsetime($argv[$i+1]))===false)
mexit(3,'option «'.$argv[$i].'» requires a valid time specification as an argument (use «-h» to read help).'.N,1,false);
$i++;
$opts['curltimeout']=$time;
} elseif ($argv[$i]=='-E' || $argv[$i]=='--excludedead') {
$opts['excludedead']=true;
} elseif ($argv[$i]=='-m' || $argv[$i]=='--minmsgimplev') {
if ($i+1>=$argc || !in_array(strtolower($argv[$i+1]),$imsgimplevs))
mexit(3,'option «'.$argv[$i].'» requires a valid “message importance level” value as an argument (use «-h» to read help).'.N,1,false);
$i++;
$opts['minmsgimplev']=array_search(strtolower($argv[$i]),$imsgimplevs);
} elseif ($argv[$i]=='-h' || $argv[$i]=='--help') {
echo($help);
exit(0);
} else {
mexit(3,'dont know how to interpret «'.$argv[$i].'» (use «-h» to read the help text).'.N,1,false);
}
}
$lockfp=__DIR__.'/'.BNAME.'.lock';
if (is_file($lockfp) && !$opts['ignorelock']) mexit(3,'lockfile exists: it seems the program is already running; if youre sure its not, you can use «-I» to force execution.'.N,1,false);
if (@touch($lockfp)===false) mexit(3,'could not create lockfile «'.$lockfp.'».'.N,1,false);
//declare(ticks=1);
pcntl_async_signals(true);
pcntl_signal(SIGTERM,'sighandler');// Termination ('kill' was called)
pcntl_signal(SIGHUP,'sighandler');// Terminal log-out
pcntl_signal(SIGINT,'sighandler');// Interrupted (Ctrl-C is pressed)
$iniarr=@parse_ini_file($opts['inifp']);
if ($iniarr===false) mexit(3,'couldnt open «'.$opts['inifp'].'».'.N,1,true);
try { $link=@mysqli_connect($iniarr['db_host'],$iniarr['db_admin_name'],$iniarr['db_admin_password'],$iniarr['db_name'],$iniarr['db_port'],$iniarr['db_socket']); }
catch (Exception $error) { mexit(3,'couldnt connect to MySQL server: '.mysqli_connect_error().'.'.N,1,true); }
// for php versions < 8
if ($link===false) mexit(3,'couldnt connect to MySQL server: '.mysqli_connect_error().'.'.N,1,true);
try { $res=mysqli_set_charset($link,'utf8mb4'); }
catch (Exception $error) { mexit(3,'couldnt set «utf8mb4» charset for MySQL: '.mysqli_error($link).' ('.mysqli_errno($link).'.'.N,1,true); }
// for php versions < 8
if ($res===false) mexit(3,'couldnt set «utf8mb4» charset for MySQL: '.mysqli_error($link).' ('.mysqli_errno($link).').'.N,1,true);
$deadinsts=[];
if ($opts['excludedead']) {
$res=myq($link,'SELECT URI FROM Instances WHERE LastOkCheckTS<'.$graceline.' OR (LastOkCheckTS IS NULL AND InsertTS<'.$graceline.')');
lecho(0,'got '.mysqli_num_rows($res).' dead instances from Instances table.'.N);
while ($row=mysqli_fetch_assoc($res))
if (!in_array($row['URI'],$deadinsts))
$deadinsts[]=$row['URI'];
$graceline=time()-$opts['gracetime'];
$res=myq($link,'SELECT Hostname FROM Peers WHERE LastOkCheckTS<'.$graceline.' OR (LastOkCheckTS IS NULL AND FirstCheckTS<'.$graceline.')');
lecho(0,'got '.mysqli_num_rows($res).' dead instances from Peers table.'.N);
while ($row=mysqli_fetch_assoc($res))
if (!in_array($row['Hostname'],$deadinsts))
$deadinsts[]=$row['URI'];
unset($res,$row);
lecho(1,'loaded list of dead instances ('.count($deadinsts).').'.N);
}
$insts=[];
$cinsts=[];
$ainsts=[];
$exarr=[];
$notifs=[];
$peersf=@fopen($opts['peersfp'],'w');
if ($peersf===false) mexit(3,'couldnt open «'.$opts['peersfp'].'» in write mode.'.N,1,true);
$apeersf=@fopen($opts['apeersfp'],'w');
if ($apeersf===false) mexit(3,'couldnt open «'.$opts['apeersfp'].'» in write mode.'.N,1,true);
$cpeersf=@fopen($opts['cpeersfp'],'w');
if ($cpeersf===false) mexit(3,'couldnt open «'.$opts['cpeersfp'].'» in write mode.'.N,1,true);
$maxround=1;
$newc=0;
$tini=time();
// go
crawl([$opts['startinst']],1);
lecho(1,'done crawling! :-)'.N);
$now=time();
lecho(1,'crawl started on '.date('Y-m-d H:i:s',$tini).' and ended on '.date('Y-m-d H:i:s',$now).'.'.N);
lecho(1,count($ainsts).' URIs checked in '.ght($now-$tini).', '.$maxround.' rounds; '.count($insts).' responded; found '.$newc.' new instances; max. memory usage: '.ghs(memory_get_peak_usage(true)).'.'.N);
fclose($peersf);
fclose($cpeersf);
fclose($apeersf);
sortcheckandsave($insts,'list of responding instances',$opts['peersfp']);
sortcheckandsave($cinsts,'list of checked instances',$opts['cpeersfp']);
sortcheckandsave($ainsts,'list of all instances',$opts['apeersfp']);
mysqli_close($link);
unlink($lockfp);
lecho(1,'done :-)'.N);
exit(0);
// functions
function crawl($list,$id) {
global $insts, $deadinsts, $cinsts, $ainsts, $tini, $opts, $peersf, $cpeersf, $apeersf, $maxround, $newc, $link;
lecho(1,'###### START OF ROUND '.$id.' ######'.N);
$nlist=[];
$c=count($list);
$i=0;
$rtini=time();
foreach ($list as $inst) {
if (!in_array($inst,$ainsts)) {
$ainsts[]=$inst;
fwrite($apeersf,$inst.N);
}
$i++;
$now=time();
$rtela=$now-$rtini;
lecho(1,'working on «'.$inst.'»: round '.$id.', '.$i.'/'.$c.'; TET: '.ght($now-$tini,null,0).'; ETR of this round: '.ght($rtela/$i*$c-$rtela,null,0).'; using '.ghs(memory_get_usage(true)).' mem. (peak: '.ghs(memory_get_peak_usage(true)).'); '.count($insts).' discovered instances; '.count($nlist).' instances in next round list; '.$newc.' new instances found.'.N);
waituntilonline();
updexarr();
lecho(0,'trying to load «'.$inst.s peers...'.N);
$peers=gurl('https://'.$inst.'/api/v1/instance/peers',$opts['timeout'],$opts['curltimeout']);
$cinsts[]=$inst;// don't need to check if in_array
fwrite($cpeersf,$inst.N);
$responded=0;
if ($peers['cont']===false) {
lecho(2,'could not load «'.$inst.s peers: '.$peers['emsg'].'.'.N);
} else {
$peers=@json_decode($peers['cont'],true);
if (!is_array($peers)) {
lecho(2,'loading «'.$inst.s peers, got not good JSON.'.N);
} else {
lecho(1,'successfully loaded «'.$inst.s peers :-)'.N);
$responded=1;
if (!in_array($inst,$insts)) {
lecho(1,'instance «'.$inst.'» responded :-)'.N);
$insts[]=$inst;
fwrite($peersf,$inst.N);
$res=myq($link,'SELECT ID FROM Instances WHERE URI=\''.myesc($link,$inst).'\'');
if (mysqli_num_rows($res)==0) {
lecho(1,'instance «'.$inst.'» is new :-)'.N);
myq($link,'INSERT INTO Instances SET URI=\''.myesc($link,$inst).'\', InsertTS='.time());
$newc++;
}
}
foreach ($peers as $peer) {
if (!in_array($peer,$ainsts)) {
$ainsts[]=$peer;
fwrite($apeersf,$peer.N);
}
$whynot=[];
if (in_array($peer,$cinsts)) $whynot[]='it has already been checked';
if (!is_string($peer)) $whynot[]='its name is not a string';
if (!validhostname($peer)) $whynot[]='its name is not a valid hostname';
if (ckexarr($peer)) $whynot[]='its name matches an exclusion regexp';
if (in_array($peer,$list)) $whynot[]='it is already present in current list';
if (in_array($peer,$nlist)) $whynot[]='it has already been added to next round list';
if ($opts['excludedead'] && in_array($peer,$deadinsts)) $whynot[]='its dead';
if (count($whynot)>0) {
lecho(0,'not adding peer «'.$peer.'» to next round list because '.implode(', ',$whynot).'.'.N);
} else {
lecho(1,'adding peer «'.$peer.'» to next round list :-)'.N);
$nlist[]=$peer;
}
}
}
}
$res=myq($link,'SELECT * FROM Peers WHERE Hostname=\''.myesc($link,$inst).'\'');
$nrows=mysqli_num_rows($res);
if ($nrows>0) {
if ($nrows>1) lecho(2,'«'.$inst.'» has '.$nrows.' records in “Peers” table! :-('.N);
$row=mysqli_fetch_assoc($res);
if ($responded) myq($link,'UPDATE Peers SET LastOkCheckTS='.$now.' WHERE ID='.$row['ID']);
} else {
$query='INSERT INTO Peers SET Hostname=\''.myesc($link,$inst).'\', FirstCheckTS='.$now;
if ($responded) $query.=', LastOkCheckTS='.$now;
myq($link,$query);
}
}
if (count($nlist)>0) {
unset($list);
crawl($nlist,$id+1);
$maxround=$id+1;
} else {
lecho(1,'next round list is empty.'.N);
}
lecho(1,'###### END OF ROUND '.$id.' ######'.N);
}
function mexit($lev,$msg,$code,$remlock) {
global $link, $peersf, $cpeersf, $apeersf, $lockfp;
if (isset($link) && $link!==false) mysqli_close($link);
if (isset($peersf) && $peersf!==false) @fclose($peersf);
if (isset($cpeersf) && $cpeersf!==false) @fclose($cpeersf);
if (isset($apeersf) && $apeersf!==false) @fclose($apeersf);
if ($remlock && isset($lockfp) && is_file($lockfp)) unlink($lockfp);
lecho($lev,$msg);
exit($code);
}
function lecho($lev,$msg) {
global $opts, $msgimplevs;
$time=microdate();
$msg=$time.' '.$msgimplevs[$lev].': '.$msg;
if ($lev>=$opts['minmsgimplev']) {
if ($lev<2)
echo($msg);
else
fwrite(STDERR,$msg);
}
}
function myq(&$link,$query) {
try { $res=mysqli_query($link,$query); }
catch (Exception $error) { mexit(3,'query «'.$query.'» failed: '.$error->getMessage().' ('.$error->getCode().').'.N,2,true); }
// for php versions < 8, which seem to not catch mysql exceptions
if ($res===false) mexit(3,'query «'.$query.'» failed: '.mysqli_error($link).' ('.mysqli_errno($link).').'.N,2,true);
return($res);
}
function microdate($time=null) {
if (is_null($time)) $time=microtime(false);
$time=explode(' ',$time);
return(date('Y-m-d H:i:s',$time[1]).'.'.substr($time[0],2));
}
function sortcheckandsave(&$arr,$arrdesc,&$fp) {
$buc=count($arr);
$arr=array_unique($arr);
$auc=count($arr);
if ($buc!=$auc) lecho(2,$arrdesc.' contained duplicates, better check the code ;-)'.N);
lecho(1,'saving ordered '.$arrdesc.' into «'.$fp.'».'.N);
sort($arr);
$f=@fopen($fp,'w');
if ($f!==false) {
foreach ($arr as $val)
fwrite($f,$val.N);
fclose($f);
} else {
lecho(2,'couldnt open «'.$fp.'» for writing.'.N);
}
}
function sighandler($signal) {
echo(N);
mexit(1,'interrupted (signal: '.$signal.').'.N,0,true);
}
function isempty($val) {
if (preg_match('/^\s*$/',$val)===1)
return(true);
else
return(false);
}
function waituntilonline() {
$url='www.google.com';
$gotoff=false;
while (false===($f=@fsockopen($url,80,$errno,$errstr,1))) {
$gotoff=true;
lecho(2,'it seems we are offline, waiting for 10 seconds before retrying...'.N);
sleep(10);
}
fclose($f);
if ($gotoff) lecho(1,'it seems we are back online! :-)'.N);
}
function updexarr() {
global $exarr, $opts;
if (!is_null($opts['excludefp'])) {
$f=@fopen($opts['excludefp'],'r');
if ($f!==false) {
$i=0;
$exarr=[];
while (!feof($f)) {
$i++;
$line=trim(fgets($f));
if (!isempty($line)) {
if (@preg_match($line,'foo')!==false)
$exarr[]=$line;
else
lecho(2,'exclude file «'.$opts['excludefp'].'» contains an invalid regular expression on line '.$i.': «'.$line.'».'.N);
}
}
} else {
lecho(2,'could not open exclude file «'.$opts['excludefp'].'» for reading.'.N);
}
}
}
function ckexarr($inst) {
global $exarr;
foreach ($exarr as $re)
if (preg_match($re,$inst)===1) return(true);
return(false);
}
function ismultibyte($s) {
preg_replace('/./u','.',$s,-1,$c);
(strlen($s)>$c) ? $r=true : $r=false;
return($r);
}
function validhostname($hostname) {
//$hostname=preg_replace('#/.*#','',$hostname);
//$hostname=preg_replace('#:[0-9]+$#','',$hostname);
if (ismultibyte($hostname)) $hostname=idn_to_ascii($hostname,IDNA_DEFAULT,INTL_IDNA_VARIANT_UTS46);
//echo($hostname.N);
if (strlen($hostname)>253) return(false);
$labels=explode('.',$hostname);
foreach($labels as $label) {
$len=strlen($label);
if ($len<1 || $len>63) return(false);
if (preg_match('#^-#',$label)==1) return(false);
if (preg_match('#-$#',$label)==1) return(false);
//if (preg_match('#--#',$label)==1) return(false);
if (preg_match('#^[a-zA-Z0-9-]+$#',$label)!==1) return(false);
}
return(true);
}
//$url='www.team.starschlep.com/'; if (validhostname($url)) echo('OK: '.$url.N); else echo('KO: '.$url.N); die();
?>