Jones/MastodonHelp
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Added truncs where needed; added code to check for “noindex” on user’s profile page when “noindex” is not set in accounts info

Browse source
This commit is contained in:
pezcurrel 2022-12-10 12:35:22 +01:00
parent eeb3ce6edf
commit 4b7f6a199c
1 changed files with 28 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
web/clitools/crawler.php
View file

@ -228,14 +228,15 @@ if (!$opts['dontrestore'] && file_exists($currinstjfp) && file_exists($instsjfp)
}
function truncs($str,$tab,$col,$ctx) {
global $tables, $tronconi, $iswin;
global $tables, $iswin;
if (is_null($str)) return(null);
if ($iswin)
$tab=strtolower($tab);
$size=$tables[$tab][$col];
$len=mb_strlen($str,'UTF-8');
if ($len>$size) {
notify($ctx.': had to truncate to '.$size.' characters the value to be inserted into «'.$col.'» column in «'.$tab.'» table because it was too long ('.$len.' characters).',3);
$str=mb_substr($str,0,$size-1,'UTF-8').'…';
notify($ctx.': had to truncate string to '.$size.' chars to be able to insert it into «'.$col.'» column in «'.$tab.'» table.',3);
}
return($str);
}
@ -721,9 +722,29 @@ while ($i<$cinsts) {
//foreach ($buf as $user) echo($user['username'].' '); echo(N.N);
foreach ($buf as $user) {
if (make(['id', 'username', 'display_name', 'locked', 'bot', 'discoverable', 'created_at', 'note', 'url', 'avatar', 'header', 'statuses_count', 'last_status_at', 'fields'], $user)) {
if (!isset($user['noindex'])) $user['noindex']=false;
if (!isset($user['noindex'])) {
$user['noindex']=true;
eecho(0,'«'.$host.'»: «'.$user['username'].'»: «noindex» is undefined, trying to define it fetching user’s profile page...'.N);
$page=getfc($user['url'],$opts['timeout']);
// here ckratelimit is not needed because it's a normal web page, not json from mastodon api
//var_dump($page);
if ($page['cont']!==false) {
//<meta content='noindex, noarchive' name='robots'>
if (preg_match('/<meta\s+content=[\'"]noindex/ui',$page['cont'])!==1) {
$user['noindex']=false;
eecho(0,'«'.$user['url'].'»: «noindex» is not set.'.N);
} else {
eecho(0,'«'.$user['url'].'»: «noindex» is set.'.N);
}
} else {
eecho(2,'«'.$host.'»: could not fetch «'.$user['url'].'»: '.$page['emsg'].N);
}
}
$snote=strip_tags($user['note']);
if (preg_match('/(^|\W)#nobots?($|\W)/iu',$snote)===1) $user['noindex']=true;
if (preg_match('/(^|\W)#globdir($|\W)/iu',$snote)===1) $user['noindex']=false;
$user['tags']=[];
if (!$user['noindex']) {
if (!$user['noindex'] && $info['version']>='3.3.0') {
eecho(0,'«'.$host.'»: trying to fetch tags for user «'.$user['username'].'»...'.N);
$tags=@getfc('https://'.$host.'/api/v1/accounts/'.$user['id'].'/featured_tags',$opts['timeout']);
if ($tags['cont']!==false) {
@ -758,7 +779,7 @@ while ($i<$cinsts) {
}
}
foreach ($users as $locid=>$user) {
$query='SET host='.myv($link,$host).', locid='.myv($link,$user['id']).', username='.myv($link,$user['username']).', display_name='.myv($link,$user['display_name']).', locked='.myv($link,$user['locked']).', bot='.myv($link,$user['bot']).', discoverable='.myv($link,$user['discoverable']).', created_at='.myv($link,$user['created_at']).', note='.myv($link,$user['note']).', url='.myv($link,$user['url']).', avatar='.myv($link,$user['avatar']).', header='.myv($link,$user['header']).', statuses_count='.myv($link,$user['statuses_count']).', last_status_at='.myv($link,$user['last_status_at']).', tags='.myv($link,$user['tags']);
$query='SET host='.myv($link,$host).', locid='.myv($link,$user['id']).', username='.myv($link,truncs($user['username'], 'Users', 'username', '«'.$host.'»')).', display_name='.myv($link,truncs($user['display_name'], 'Users', 'display_name', '«'.$host.'»')).', locked='.myv($link,$user['locked']).', bot='.myv($link,$user['bot']).', discoverable='.myv($link,$user['discoverable']).', created_at='.myv($link,$user['created_at']).', note='.myv($link,truncs($user['note'], 'Users', 'note', '«'.$host.'»')).', url='.myv($link,truncs($user['url'], 'Users', 'url', '«'.$host.'»')).', avatar='.myv($link,truncs($user['avatar'], 'Users', 'avatar', '«'.$host.'»')).', header='.myv($link,truncs($user['header'], 'Users', 'header', '«'.$host.'»')).', statuses_count='.myv($link,$user['statuses_count']).', last_status_at='.myv($link,$user['last_status_at']).', tags='.myv($link,truncs($user['tags'], 'Users', 'tags', '«'.$host.'»'));
$uid=0;
if (!array_key_exists($user['id'],$exusers)) {
if (!$user['noindex']) {
@ -789,6 +810,8 @@ while ($i<$cinsts) {
eecho(0,'«'.$host.'»: saving user fields for user «'.$user['username'].'» ('.$uid.')...'.N);
foreach ($user['fields'] as $field) {
(is_null($field['verified_at'])) ? $field['verified_at']=0 : $field['verified_at']=1;
$field['name']=truncs($field['name'],'UsersFields','name','«'.$host.'»');
$field['value']=truncs($field['value'],'UsersFields','value','«'.$host.'»');
if (!$opts['dryrun']) mysqli_query($link,'INSERT INTO UsersFields SET UserID='.$uid.', name='.myv($link,$field['name']).', value='.myv($link,$field['value']).', verified='.$field['verified_at']) or mexit(__LINE__.': '.mysqli_error($link).N,3);
}
}