boyska/rss-bridge
1
0
Fork 0
forked from blallo/rss-bridge
Code Releases Activity

Merge branch 'FeedExpander' of https://github.com/logmanoriginal/rss-bridge

Browse source
This commit is contained in:
logmanoriginal 2016-09-05 20:27:45 +02:00
commit 579d13462f
13 changed files with 288 additions and 442 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
bridges/CADBridge.php
View file

@ -1,12 +1,24 @@
<?php <?php
class CADBridge extends BridgeAbstract{ class CADBridge extends FeedExpander {
const MAINTAINER = "nyutag"; const MAINTAINER = "nyutag";
const NAME = "CAD Bridge"; const NAME = "CAD Bridge";
const URI = "http://www.cad-comic.com/"; const URI = "http://www.cad-comic.com/";
const DESCRIPTION = "Returns the newest articles."; const DESCRIPTION = "Returns the newest articles.";
public function collectData(){
$this->collectExpandableDatas('http://cdn2.cad-comic.com/rss.xml', 10);
}
protected function parseItem($newsItem){
$item = $this->parseRSS_2_0_Item($newsItem);
$item['content'] = $this->CADExtractContent($item['uri']);
return $item;
}
private function CADExtractContent($url) { private function CADExtractContent($url) {
$html3 = $this->getSimpleHTMLDOM($url); if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$html3 = $this->get_cached($url);
// The request might fail due to missing https support or wrong URL // The request might fail due to missing https support or wrong URL
if($html3 == false) if($html3 == false)
@ -32,33 +44,6 @@ class CADBridge extends BridgeAbstract{
return '<img src="'.$img.'"/>'; return '<img src="'.$img.'"/>';
} }
public function collectData(){
function CADUrl($string) {
$html2 = explode("\"", $string);
$string = $html2[1];
if (substr($string,0,4) != 'http')
return 'notanurl';
return $string;
}
$html = $this->getSimpleHTMLDOM('http://cdn2.cad-comic.com/rss.xml') or $this->returnServerError('Could not request CAD.');
$limit = 0;
foreach($html->find('item') as $element) {
if($limit < 5) {
$item = array();
$item['title'] = $element->find('title', 0)->innertext;
$item['uri'] = CADUrl($element->find('description', 0)->innertext);
if ($item['uri'] != 'notanurl') {
$item['timestamp'] = strtotime($element->find('pubDate', 0)->plaintext);
$item['content'] = $this->CADExtractContent($item['uri']);
$this->items[] = $item;
$limit++;
}
}
}
}
public function getCacheDuration(){ public function getCacheDuration(){
return 3600*2; // 2 hours return 3600*2; // 2 hours
} }

39
bridges/CommonDreamsBridge.php
View file

@ -1,39 +1,28 @@
<?php <?php
class CommonDreamsBridge extends BridgeAbstract{ class CommonDreamsBridge extends FeedExpander {
const MAINTAINER = "nyutag"; const MAINTAINER = "nyutag";
const NAME = "CommonDreams Bridge"; const NAME = "CommonDreams Bridge";
const URI = "http://www.commondreams.org/"; const URI = "http://www.commondreams.org/";
const DESCRIPTION = "Returns the newest articles."; const DESCRIPTION = "Returns the newest articles.";
public function collectData(){
$this->collectExpandableDatas('http://www.commondreams.org/rss.xml', 10);
}
protected function parseItem($newsItem){
$item = $this->parseRSS_2_0_Item($newsItem);
$item['content'] = $this->CommonDreamsExtractContent($item['uri']);
return $item;
}
private function CommonDreamsExtractContent($url) { private function CommonDreamsExtractContent($url) {
$html3 = $this->getSimpleHTMLDOM($url); if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$html3 = $this->get_cached($url);
$text = $html3->find('div[class=field--type-text-with-summary]', 0)->innertext; $text = $html3->find('div[class=field--type-text-with-summary]', 0)->innertext;
$html3->clear(); $html3->clear();
unset ($html3); unset ($html3);
return $text; return $text;
} }
public function collectData(){
function CommonDreamsUrl($string) {
$html2 = explode(" ", $string);
$string = $html2[2] . "/node/" . $html2[0];
return $string;
}
$html = $this->getSimpleHTMLDOM('http://www.commondreams.org/rss.xml') or $this->returnServerError('Could not request CommonDreams.');
$limit = 0;
foreach($html->find('item') as $element) {
if($limit < 4) {
$item = array();
$item['title'] = $element->find('title', 0)->innertext;
$item['uri'] = CommonDreamsUrl($element->find('guid', 0)->innertext);
$item['timestamp'] = strtotime($element->find('pubDate', 0)->plaintext);
$item['content'] = $this->CommonDreamsExtractContent($item['uri']);
$this->items[] = $item;
$limit++;
}
}
}
} }

64
bridges/DauphineLibereBridge.php
View file

@ -1,10 +1,10 @@
<?php <?php
class DauphineLibereBridge extends BridgeAbstract { class DauphineLibereBridge extends FeedExpander {
const MAINTAINER = "qwertygc"; const MAINTAINER = "qwertygc";
const NAME = "Dauphine Bridge"; const NAME = "Dauphine Bridge";
const URI = "http://www.ledauphine.com/"; const URI = "http://www.ledauphine.com/";
const DESCRIPTION = "Returns the newest articles."; const DESCRIPTION = "Returns the newest articles.";
const PARAMETERS = array( array( const PARAMETERS = array( array(
'u'=>array( 'u'=>array(
@ -30,41 +30,33 @@ class DauphineLibereBridge extends BridgeAbstract {
) )
)); ));
private function ExtractContent($url, $context) { public function collectData(){
$html2 = $this->getSimpleHTMLDOM($url); $url = self::URI . 'rss';
$text = $html2->find('div.column', 0)->innertext;
$text = preg_replace('@<script[^>]*?>.*?</script>@si', '', $text);
return $text;
}
public function collectData(){ if (empty($this->getInput('u'))) {
$url = self::URI . $this->getInput('u') . '/rss';
}
$context = stream_context_create($opts); $this->collectExpandableDatas($url, 10);
}
if (empty($this->getInput('u'))) { protected function parseItem($newsItem){
$html = $this->getSimpleHTMLDOM(self::URI.$this->getInput('u').'/rss') $item = $this->parseRSS_2_0_Item($newsItem);
or $this->returnServerError('Could not request DauphineLibere.'); $item['content'] = $this->ExtractContent($item['uri']);
} else { return $item;
$html = $this->getSimpleHTMLDOM(self::URI.'rss') }
or $this->returnServerError('Could not request DauphineLibere.');
}
$limit = 0;
foreach($html->find('item') as $element) { private function ExtractContent($url) {
if($limit < 10) { if($this->get_cached_time($url) <= strtotime('-24 hours'))
$item = array(); $this->remove_from_cache($url);
$item['title'] = $element->find('title', 0)->innertext; $html2 = $this->get_cached($url);
$item['uri'] = $element->find('guid', 0)->plaintext; $text = $html2->find('div.column', 0)->innertext;
$item['timestamp'] = strtotime($element->find('pubDate', 0)->plaintext); $text = preg_replace('@<script[^>]*?>.*?</script>@si', '', $text);
$item['content'] = $this->ExtractContent($item['uri'], $context); return $text;
$this->items[] = $item; }
$limit++;
}
}
}
public function getCacheDuration(){ public function getCacheDuration(){
return 3600*2; // 2 hours return 3600*2; // 2 hours
} }
} }
?> ?>

35
bridges/DeveloppezDotComBridge.php
View file

@ -1,11 +1,21 @@
<?php <?php
class DeveloppezDotComBridge extends BridgeAbstract{ class DeveloppezDotComBridge extends FeedExpander {
const MAINTAINER = "polopollo"; const MAINTAINER = "polopollo";
const NAME = "Developpez.com Actus (FR)"; const NAME = "Developpez.com Actus (FR)";
const URI = "http://www.developpez.com/"; const URI = "http://www.developpez.com/";
const DESCRIPTION = "Returns the 15 newest posts from DeveloppezDotCom (full text)."; const DESCRIPTION = "Returns the 15 newest posts from DeveloppezDotCom (full text).";
public function collectData(){
$this->collectExpandableDatas(self::URI . 'index/rss', 15);
}
protected function parseItem($newsItem){
$item = $this->parseRSS_2_0_Item($newsItem);
$item['content'] = $this->DeveloppezDotComExtractContent($item['uri']);
return $item;
}
private function DeveloppezDotComStripCDATA($string) { private function DeveloppezDotComStripCDATA($string) {
$string = str_replace('<![CDATA[', '', $string); $string = str_replace('<![CDATA[', '', $string);
$string = str_replace(']]>', '', $string); $string = str_replace(']]>', '', $string);
@ -32,31 +42,14 @@ class DeveloppezDotComBridge extends BridgeAbstract{
} }
private function DeveloppezDotComExtractContent($url) { private function DeveloppezDotComExtractContent($url) {
$articleHTMLContent = $this->getSimpleHTMLDOM($url); if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$articleHTMLContent = $this->get_cached($url);
$text = $this->convert_smart_quotes($articleHTMLContent->find('div.content', 0)->innertext); $text = $this->convert_smart_quotes($articleHTMLContent->find('div.content', 0)->innertext);
$text = utf8_encode($text); $text = utf8_encode($text);
return trim($text); return trim($text);
} }
public function collectData(){
$rssFeed = $this->getSimpleHTMLDOM(self::URI.'index/rss')
or $this->returnServerError('Could not request '.self::URI.'index/rss');
$limit = 0;
foreach($rssFeed->find('item') as $element) {
if($limit < 10) {
$item = array();
$item['title'] = $this->DeveloppezDotComStripCDATA($element->find('title', 0)->innertext);
$item['uri'] = $this->DeveloppezDotComStripCDATA($element->find('guid', 0)->plaintext);
$item['timestamp'] = strtotime($element->find('pubDate', 0)->plaintext);
$content = $this->DeveloppezDotComExtractContent($item['uri']);
$item['content'] = strlen($content) ? $content : $element->description; //In case of it is a tutorial, we just keep the original description
$this->items[] = $item;
$limit++;
}
}
}
public function getCacheDuration(){ public function getCacheDuration(){
return 1800; // 30min return 1800; // 30min
} }

178
bridges/FuturaSciencesBridge.php
View file

@ -1,5 +1,5 @@
<?php <?php
class FuturaSciencesBridge extends BridgeAbstract { class FuturaSciencesBridge extends FeedExpander {
const MAINTAINER = 'ORelio'; const MAINTAINER = 'ORelio';
const NAME = 'Futura-Sciences Bridge'; const NAME = 'Futura-Sciences Bridge';
@ -79,108 +79,90 @@ class FuturaSciencesBridge extends BridgeAbstract {
)); ));
public function collectData(){ public function collectData(){
$url = self::URI . 'rss/' . $this->getInput('feed') . '.xml';
$this->collectExpandableDatas($url, 10);
}
function StripCDATA($string) { protected function parseItem($newsItem){
$string = str_replace('<![CDATA[', '', $string); $item = $this->parseRSS_2_0_Item($newsItem);
$string = str_replace(']]>', '', $string); $item['uri'] = str_replace('#xtor=RSS-8', '', $item['uri']);
return $string; if($this->get_cached_time($item['uri']) <= strtotime('-24 hours'))
} $this->remove_from_cache($item['uri']);
$article = $this->get_cached($item['uri'])
or $this->returnServerError('Could not request Futura-Sciences: ' . $item['uri']);
$item['content'] = $this->ExtractArticleContent($article);
$item['author'] = empty($this->ExtractAuthor($article)) ? $item['author'] : $this->ExtractAuthor($article);
return $item;
}
function StripWithDelimiters($string, $start, $end) { function StripWithDelimiters($string, $start, $end) {
while (strpos($string, $start) !== false) { while (strpos($string, $start) !== false) {
$section_to_remove = substr($string, strpos($string, $start)); $section_to_remove = substr($string, strpos($string, $start));
$section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end));
$string = str_replace($section_to_remove, '', $string);
} return $string;
}
function StripRecursiveHTMLSection($string, $tag_name, $tag_start) {
$open_tag = '<'.$tag_name;
$close_tag = '</'.$tag_name.'>';
$close_tag_length = strlen($close_tag);
if (strpos($tag_start, $open_tag) === 0) {
while (strpos($string, $tag_start) !== false) {
$max_recursion = 100;
$section_to_remove = null;
$section_start = strpos($string, $tag_start);
$search_offset = $section_start;
do {
$max_recursion--;
$section_end = strpos($string, $close_tag, $search_offset);
$search_offset = $section_end + $close_tag_length;
$section_to_remove = substr($string, $section_start, $section_end - $section_start + $close_tag_length);
$open_tag_count = substr_count($section_to_remove, $open_tag);
$close_tag_count = substr_count($section_to_remove, $close_tag);
} while ($open_tag_count > $close_tag_count && $max_recursion > 0);
$string = str_replace($section_to_remove, '', $string); $string = str_replace($section_to_remove, '', $string);
} return $string;
}
function StripRecursiveHTMLSection($string, $tag_name, $tag_start) {
$open_tag = '<'.$tag_name;
$close_tag = '</'.$tag_name.'>';
$close_tag_length = strlen($close_tag);
if (strpos($tag_start, $open_tag) === 0) {
while (strpos($string, $tag_start) !== false) {
$max_recursion = 100;
$section_to_remove = null;
$section_start = strpos($string, $tag_start);
$search_offset = $section_start;
do {
$max_recursion--;
$section_end = strpos($string, $close_tag, $search_offset);
$search_offset = $section_end + $close_tag_length;
$section_to_remove = substr($string, $section_start, $section_end - $section_start + $close_tag_length);
$open_tag_count = substr_count($section_to_remove, $open_tag);
$close_tag_count = substr_count($section_to_remove, $close_tag);
} while ($open_tag_count > $close_tag_count && $max_recursion > 0);
$string = str_replace($section_to_remove, '', $string);
}
}
return $string;
}
// Extracts the author from an article or element
function ExtractAuthor($article, $element){
$article_author = $article->find('span.author', 0);
if($article_author){
$authorname = trim(str_replace(', Futura-Sciences', '', $article_author->plaintext));
if(empty($authorname)){
$element_author = $element->find('author', 0);
if($element_author)
$authorname = StripCDATA($element_author->plaintext);
else
return '';
}
return $authorname;
}
return '';
}
$url = $this->getURI().'rss/'.$this->getInput('feed').'.xml';
$html = $this->getSimpleHTMLDOM($url)
or $this->returnServerError('Could not request Futura-Sciences: '.$url);
$limit = 0;
foreach($html->find('item') as $element) {
if ($limit < 10) {
$article_url = str_replace('#xtor=RSS-8', '', StripCDATA($element->find('guid', 0)->plaintext));
$article = $this->getSimpleHTMLDOM($article_url) or $this->returnServerError('Could not request Futura-Sciences: '.$article_url);
$contents = $article->find('div.content', 0)->innertext;
foreach (array(
'<div class="clear',
'<div class="sharebar2',
'<div class="diaporamafullscreen"',
'<div style="margin-bottom:10px;" class="noprint"',
'<div class="ficheprevnext',
'<div class="bar noprint',
'<div class="toolbar noprint',
'<div class="addthis_toolbox',
'<div class="noprint',
'<div class="bg bglight border border-full noprint',
'<div class="httplogbar-wrapper noprint',
'<div id="forumcomments'
) as $div_start) {
$contents = StripRecursiveHTMLSection($contents , 'div', $div_start);
}
$contents = StripWithDelimiters($contents, '<hr ', '/>');
$contents = StripWithDelimiters($contents, '<p class="content-date', '</p>');
$contents = StripWithDelimiters($contents, '<h1 class="content-title', '</h1>');
$contents = StripWithDelimiters($contents, 'fs:definition="', '"');
$contents = StripWithDelimiters($contents, 'fs:xt:clicktype="', '"');
$contents = StripWithDelimiters($contents, 'fs:xt:clickname="', '"');
$item = array();
$item['author'] = ExtractAuthor($article, $element);
$item['uri'] = $article_url;
$item['title'] = StripCDATA($element->find('title', 0)->innertext);
$item['timestamp'] = strtotime(StripCDATA($element->find('pubDate', 0)->plaintext));
$item['content'] = trim($contents);
$this->items[] = $item;
$limit++;
} }
} }
return $string;
}
function ExtractArticleContent($article){
$contents = $article->find('div.content', 0)->innertext;
foreach (array(
'<div class="clear',
'<div class="sharebar2',
'<div class="diaporamafullscreen"',
'<div style="margin-bottom:10px;" class="noprint"',
'<div class="ficheprevnext',
'<div class="bar noprint',
'<div class="toolbar noprint',
'<div class="addthis_toolbox',
'<div class="noprint',
'<div class="bg bglight border border-full noprint',
'<div class="httplogbar-wrapper noprint',
'<div id="forumcomments'
) as $div_start) {
$contents = $this->StripRecursiveHTMLSection($contents , 'div', $div_start);
}
$contents = $this->StripWithDelimiters($contents, '<hr ', '/>');
$contents = $this->StripWithDelimiters($contents, '<p class="content-date', '</p>');
$contents = $this->StripWithDelimiters($contents, '<h1 class="content-title', '</h1>');
$contents = $this->StripWithDelimiters($contents, 'fs:definition="', '"');
$contents = $this->StripWithDelimiters($contents, 'fs:xt:clicktype="', '"');
$contents = $this->StripWithDelimiters($contents, 'fs:xt:clickname="', '"');
return $contents;
}
// Extracts the author from an article or element
function ExtractAuthor($article){
$article_author = $article->find('span.author', 0);
if($article_author){
return trim(str_replace(', Futura-Sciences', '', $article_author->plaintext));
}
return '';
} }
} }

36
bridges/LeJournalDuGeekBridge.php
View file

@ -1,19 +1,25 @@
<?php <?php
class LeJournalDuGeekBridge extends BridgeAbstract{ class LeJournalDuGeekBridge extends FeedExpander {
const MAINTAINER = "polopollo"; const MAINTAINER = "polopollo";
const NAME = "journaldugeek.com (FR)"; const NAME = "journaldugeek.com (FR)";
const URI = "http://www.journaldugeek.com/"; const URI = "http://www.journaldugeek.com/";
const DESCRIPTION = "Returns the 5 newest posts from LeJournalDuGeek (full text)."; const DESCRIPTION = "Returns the 5 newest posts from LeJournalDuGeek (full text).";
private function LeJournalDuGeekStripCDATA($string) { public function collectData(){
$string = str_replace('<![CDATA[', '', $string); $this->collectExpandableDatas(self::URI . 'rss', 5);
$string = str_replace(']]>', '', $string); }
return $string;
protected function parseItem($newsItem){
$item = $this->parseRSS_2_0_Item($newsItem);
$item['content'] = $this->LeJournalDuGeekExtractContent($item['uri']);
return $item;
} }
private function LeJournalDuGeekExtractContent($url) { private function LeJournalDuGeekExtractContent($url) {
$articleHTMLContent = $this->getSimpleHTMLDOM($url); if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$articleHTMLContent = $this->get_cached($url);
$text = $articleHTMLContent->find('div.post-content', 0)->innertext; $text = $articleHTMLContent->find('div.post-content', 0)->innertext;
foreach($articleHTMLContent->find('a.more') as $element) { foreach($articleHTMLContent->find('a.more') as $element) {
@ -34,24 +40,6 @@ class LeJournalDuGeekBridge extends BridgeAbstract{
return $text; return $text;
} }
public function collectData(){
$rssFeed = $this->getSimpleHTMLDOM(self::URI.'rss')
or $this->returnServerError('Could not request '.self::URI.'/rss');
$limit = 0;
foreach($rssFeed->find('item') as $element) {
if($limit < 5) {
$item = array();
$item['title'] = $this->LeJournalDuGeekStripCDATA($element->find('title', 0)->innertext);
$item['uri'] = $this->LeJournalDuGeekStripCDATA($element->find('guid', 0)->plaintext);
$item['timestamp'] = strtotime($element->find('pubDate', 0)->plaintext);
$item['content'] = $this->LeJournalDuGeekExtractContent($item['uri']);
$this->items[] = $item;
$limit++;
}
}
}
public function getCacheDuration(){ public function getCacheDuration(){
return 1800; // 30min return 1800; // 30min
} }

74
bridges/LeMondeInformatiqueBridge.php
View file

@ -1,5 +1,5 @@
<?php <?php
class LeMondeInformatiqueBridge extends BridgeAbstract { class LeMondeInformatiqueBridge extends FeedExpander {
const MAINTAINER = "ORelio"; const MAINTAINER = "ORelio";
const NAME = "Le Monde Informatique"; const NAME = "Le Monde Informatique";
@ -7,54 +7,38 @@ class LeMondeInformatiqueBridge extends BridgeAbstract {
const DESCRIPTION = "Returns the newest articles."; const DESCRIPTION = "Returns the newest articles.";
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'rss/rss.xml', 10);
}
function StripCDATA($string) { protected function parseItem($newsItem){
$string = str_replace('<![CDATA[', '', $string); $item = $this->parseRSS_1_0_Item($newsItem);
$string = str_replace(']]>', '', $string); if($this->get_cached_time($item['uri']) <= strtotime('-24 hours'))
return $string; $this->remove_from_cache($item['uri']);
} $article_html = $this->get_cached($item['uri'])
or $this->returnServerError('Could not request LeMondeInformatique: ' . $item['uri']);
$item['content'] = $this->CleanArticle($article_html->find('div#article', 0)->innertext);
$item['title'] = $article_html->find('h1.cleanprint-title', 0)->plaintext;
return $item;
}
function StripWithDelimiters($string, $start, $end) { function StripCDATA($string) {
while (strpos($string, $start) !== false) { $string = str_replace('<![CDATA[', '', $string);
$section_to_remove = substr($string, strpos($string, $start)); $string = str_replace(']]>', '', $string);
$section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); return $string;
$string = str_replace($section_to_remove, '', $string); }
} return $string;
}
function CleanArticle($article_html) { function StripWithDelimiters($string, $start, $end) {
$article_html = StripWithDelimiters($article_html, '<script', '</script>'); while (strpos($string, $start) !== false) {
$article_html = StripWithDelimiters($article_html, '<h1 class="cleanprint-title"', '</h1>'); $section_to_remove = substr($string, strpos($string, $start));
return $article_html; $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end));
} $string = str_replace($section_to_remove, '', $string);
} return $string;
}
$html = $this->getSimpleHTMLDOM(self::URI.'rss/rss.xml') function CleanArticle($article_html) {
or $this->returnServerError('Could not request LeMondeInformatique: ' $article_html = $this->StripWithDelimiters($article_html, '<script', '</script>');
.self::URI.'rss/rss.xml'); $article_html = $this->StripWithDelimiters($article_html, '<h1 class="cleanprint-title"', '</h1>');
$limit = 0; return $article_html;
foreach($html->find('item') as $element) {
if($limit < 5) {
//Retrieve article details
$article_uri = $element->innertext;
$article_uri = substr($article_uri, strpos($article_uri, '<link>') + 6);
$article_uri = substr($article_uri, 0, strpos($article_uri, '</link>'));
$article_html = $this->getSimpleHTMLDOM($article_uri) or $this->returnServerError('Could not request LeMondeInformatique: '.$article_uri);
$article_content = CleanArticle($article_html->find('div#article', 0)->innertext);
$article_title = $article_html->find('h1.cleanprint-title', 0)->plaintext;
//Build and add final item
$item = array();
$item['uri'] = $article_uri;
$item['title'] = $article_title;
$item['author'] = StripCDATA($element->find('dc:creator', 0)->innertext);
$item['timestamp'] = strtotime($element->find('dc:date', 0)->plaintext);
$item['content'] = $article_content;
$this->items[] = $item;
$limit++;
}
}
} }
public function getCacheDuration() { public function getCacheDuration() {

37
bridges/LichessBridge.php
View file

@ -1,39 +1,22 @@
<?php <?php
class LichessBridge extends FeedExpander {
class LichessBridge extends HttpCachingBridgeAbstract
{
const MAINTAINER = 'AmauryCarrade'; const MAINTAINER = 'AmauryCarrade';
const NAME = 'Lichess Blog'; const NAME = 'Lichess Blog';
const URI = 'http://fr.lichess.org/blog'; const URI = 'http://fr.lichess.org/blog';
const DESCRIPTION = 'Returns the 5 newest posts from the Lichess blog (full text)'; const DESCRIPTION = 'Returns the 5 newest posts from the Lichess blog (full text)';
public function collectData() public function collectData(){
{ $this->collectExpandableDatas(self::URI . '.atom', 5);
$xml_feed = $this->getSimpleHTMLDOM(self::URI.'.atom')
or $this->returnServerError('Could not retrieve Lichess blog feed.');
$posts_loaded = 0;
foreach($xml_feed->find('entry') as $entry)
{
if ($posts_loaded < 5)
{
$item = array();
$item['title'] = html_entity_decode($entry->find('title', 0)->innertext);
$item['author'] = $entry->find('author', 0)->find('name', 0)->innertext;
$item['uri'] = $entry->find('id', 0)->plaintext;
$item['timestamp'] = strtotime($entry->find('published', 0)->plaintext);
$item['content'] = $this->retrieve_lichess_post($item['uri']);
$this->items[] = $item;
$posts_loaded++;
}
}
} }
private function retrieve_lichess_post($blog_post_uri) protected function parseItem($newsItem){
{ $item = $this->parseATOMItem($newsItem);
$item['content'] = $this->retrieve_lichess_post($item['uri']);
return $item;
}
private function retrieve_lichess_post($blog_post_uri){
if($this->get_cached_time($blog_post_uri) <= strtotime('-24 hours')) if($this->get_cached_time($blog_post_uri) <= strtotime('-24 hours'))
$this->remove_from_cache($blog_post_uriuri); $this->remove_from_cache($blog_post_uriuri);

36
bridges/NextInpactBridge.php
View file

@ -1,19 +1,25 @@
<?php <?php
class NextInpactBridge extends BridgeAbstract { class NextInpactBridge extends FeedExpander {
const MAINTAINER = "qwertygc"; const MAINTAINER = "qwertygc";
const NAME = "NextInpact Bridge"; const NAME = "NextInpact Bridge";
const URI = "http://www.nextinpact.com/"; const URI = "http://www.nextinpact.com/";
const DESCRIPTION = "Returns the newest articles."; const DESCRIPTION = "Returns the newest articles.";
private function StripCDATA($string) { public function collectData(){
$string = str_replace('<![CDATA[', '', $string); $this->collectExpandableDatas(self::URI . 'rss/news.xml', 10);
$string = str_replace(']]>', '', $string); }
return $string;
protected function parseItem($newsItem){
$item = $this->parseRSS_2_0_Item($newsItem);
$item['content'] = $this->ExtractContent($item['uri']);
return $item;
} }
private function ExtractContent($url) { private function ExtractContent($url) {
$html2 = $this->getSimpleHTMLDOM($url); if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$html2 = $this->get_cached($url);
$text = '<p><em>'.$html2->find('span.sub_title', 0)->innertext.'</em></p>' $text = '<p><em>'.$html2->find('span.sub_title', 0)->innertext.'</em></p>'
.'<p><img src="'.$html2->find('div.container_main_image_article', 0)->find('img.dedicated',0)->src.'" alt="-" /></p>' .'<p><img src="'.$html2->find('div.container_main_image_article', 0)->find('img.dedicated',0)->src.'" alt="-" /></p>'
.'<div>'.$html2->find('div[itemprop=articleBody]', 0)->innertext.'</div>'; .'<div>'.$html2->find('div[itemprop=articleBody]', 0)->innertext.'</div>';
@ -22,22 +28,4 @@ class NextInpactBridge extends BridgeAbstract {
$text = $text.'<p><em>'.$premium_article->innertext.'</em></p>'; $text = $text.'<p><em>'.$premium_article->innertext.'</em></p>';
return $text; return $text;
} }
public function collectData(){
$html = $this->getSimpleHTMLDOM(self::URI.'rss/news.xml') or $this->returnServerError('Could not request NextInpact.');
$limit = 0;
foreach($html->find('item') as $element) {
if($limit < 3) {
$item = array();
$item['title'] = $this->StripCDATA($element->find('title', 0)->innertext);
$item['uri'] = $this->StripCDATA($element->find('guid', 0)->plaintext);
$item['author'] = $this->StripCDATA($element->find('creator', 0)->innertext);
$item['timestamp'] = strtotime($element->find('pubDate', 0)->plaintext);
$item['content'] = $this->ExtractContent($item['uri']);
$this->items[] = $item;
$limit++;
}
}
}
} }

84
bridges/NextgovBridge.php
View file

@ -1,5 +1,5 @@
<?php <?php
class NextgovBridge extends BridgeAbstract { class NextgovBridge extends FeedExpander {
const MAINTAINER = 'ORelio'; const MAINTAINER = 'ORelio';
const NAME = 'Nextgov Bridge'; const NAME = 'Nextgov Bridge';
@ -26,57 +26,45 @@ class NextgovBridge extends BridgeAbstract {
)); ));
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'rss/' . $this->getInput('category') . '/', 10);
}
function ExtractFromDelimiters($string, $start, $end) { protected function parseItem($newsItem){
if (strpos($string, $start) !== false) { $item = $this->parseRSS_2_0_Item($newsItem);
$section_retrieved = substr($string, strpos($string, $start) + strlen($start));
$section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end));
return $section_retrieved;
} return false;
}
function StripWithDelimiters($string, $start, $end) { $item['content'] = '';
while (strpos($string, $start) !== false) {
$section_to_remove = substr($string, strpos($string, $start));
$section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end));
$string = str_replace($section_to_remove, '', $string);
} return $string;
}
$category = $this->getInput('category'); $namespaces = $newsItem->getNamespaces(true);
$url = $this->getURI().'rss/'.$category.'/'; if(isset($namespaces['media'])){
$html = $this->getSimpleHTMLDOM($url) or $this->returnServerError('Could not request Nextgov: '.$url); $media = $newsItem->children($namespaces['media']);
$limit = 0; if(isset($media->content)){
$attributes = $media->content->attributes();
foreach ($html->find('item') as $element) { $item['content'] = '<img src="' . $attributes['url'] . '">';
if ($limit >= 10) {
break;
} }
$article_url = ExtractFromDelimiters($element->innertext, '<link>', '</link>');
$article_author = ExtractFromDelimiters($element->innertext, 'dc/elements/1.1/">', '</dc:creator>');
$article_title = $element->find('title', 0)->plaintext;
$article_subtitle = $element->find('description', 0)->plaintext;
$article_timestamp = strtotime($element->find('pubDate', 0)->plaintext);
$article_thumbnail = ExtractFromDelimiters($element->innertext, '<media:content url="', '"');
$article = $this->getSimpleHTMLDOM($article_url) or $this->returnServerError('Could not request Nextgov: '.$article_url);
$contents = $article->find('div.wysiwyg', 0)->innertext;
$contents = StripWithDelimiters($contents, '<div class="ad-container">', '</div>');
$contents = StripWithDelimiters($contents, '<div', '</div>'); //ad outer div
$contents = StripWithDelimiters($contents, '<script', '</script>');
$contents = ($article_thumbnail == '' ? '' : '<p><img src="'.$article_thumbnail.'" /></p>')
.'<p><b>'.$article_subtitle.'</b></p>'
.trim($contents);
$item = array();
$item['uri'] = $article_url;
$item['title'] = $article_title;
$item['author'] = $article_author;
$item['timestamp'] = $article_timestamp;
$item['content'] = $contents;
$this->items[] = $item;
$limit++;
} }
$item['content'] .= $this->ExtractContent($item['uri']);
return $item;
}
private function StripWithDelimiters($string, $start, $end) {
while (strpos($string, $start) !== false) {
$section_to_remove = substr($string, strpos($string, $start));
$section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end));
$string = str_replace($section_to_remove, '', $string);
} return $string;
}
private function ExtractContent($url){
$article = $this->get_cached($url)
or $this->returnServerError('Could not request Nextgov: ' . $url);
$contents = $article->find('div.wysiwyg', 0)->innertext;
$contents = $this->StripWithDelimiters($contents, '<div class="ad-container">', '</div>');
$contents = $this->StripWithDelimiters($contents, '<div', '</div>'); //ad outer div
return $this->StripWithDelimiters($contents, '<script', '</script>');
$contents = ($article_thumbnail == '' ? '' : '<p><img src="'.$article_thumbnail.'" /></p>')
.'<p><b>'.$article_subtitle.'</b></p>'
.trim($contents);
} }
} }

41
bridges/NiceMatinBridge.php
View file

@ -1,13 +1,25 @@
<?php <?php
class NiceMatinBridge extends BridgeAbstract{ class NiceMatinBridge extends FeedExpander {
const MAINTAINER = "pit-fgfjiudghdf"; const MAINTAINER = "pit-fgfjiudghdf";
const NAME = "NiceMatin"; const NAME = "NiceMatin";
const URI = "http://www.nicematin.com/"; const URI = "http://www.nicematin.com/";
const DESCRIPTION = "Returns the 10 newest posts from NiceMatin (full text)"; const DESCRIPTION = "Returns the 10 newest posts from NiceMatin (full text)";
public function collectData(){
$this->collectExpandableDatas(self::URI . 'derniere-minute/rss', 10);
}
protected function parseItem($newsItem){
$item = $this->parseRSS_2_0_Item($newsItem);
$item['content'] = $this->NiceMatinExtractContent($item['uri']);
return $item;
}
private function NiceMatinExtractContent($url) { private function NiceMatinExtractContent($url) {
$html = $this->getSimpleHTMLDOM($url); if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$html = $this->get_cached($url);
if(!$html) if(!$html)
return 'Could not acquire content from url: ' . $url . '!'; return 'Could not acquire content from url: ' . $url . '!';
@ -19,29 +31,4 @@ class NiceMatinBridge extends BridgeAbstract{
$text = strip_tags($text, '<p><a><img>'); $text = strip_tags($text, '<p><a><img>');
return $text; return $text;
} }
public function collectData(){
$html = $this->getSimpleHTMLDOM(self::URI.'derniere-minute/rss')
or $this->returnServerError('Could not request NiceMatin.');
$limit = 0;
foreach($html->find('item') as $element) {
if($limit >= 10) {
break;
}
// We need to fix the 'link' tag as simplehtmldom cannot parse it (just rename it and load back as dom)
$element_text = $element->outertext;
$element_text = str_replace('<link>', '<url>', $element_text);
$element_text = str_replace('</link>', '</url>', $element_text);
$element = str_get_html($element_text);
$item = array();
$item['title'] = $element->find('title', 0)->innertext;
$item['uri'] = $element->find('url', 0)->innertext;
$item['timestamp'] = strtotime($element->find('pubDate', 0)->plaintext);
$item['content'] = $this->NiceMatinExtractContent($item['uri']);
$this->items[] = $item;
$limit++;
}
}
} }

46
bridges/NumeramaBridge.php
View file

@ -1,5 +1,5 @@
<?php <?php
class NumeramaBridge extends HttpCachingBridgeAbstract { class NumeramaBridge extends FeedExpander {
const MAINTAINER = 'mitsukarenai'; const MAINTAINER = 'mitsukarenai';
const NAME = 'Numerama'; const NAME = 'Numerama';
@ -7,39 +7,23 @@ class NumeramaBridge extends HttpCachingBridgeAbstract {
const DESCRIPTION = 'Returns the 5 newest posts from Numerama (full text)'; const DESCRIPTION = 'Returns the 5 newest posts from Numerama (full text)';
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'feed/', 5);
}
function NumeramaStripCDATA($string) { protected function parseItem($newsItem){
$string = str_replace('<![CDATA[', '', $string); $item = $this->parseRSS_2_0_Item($newsItem);
$string = str_replace(']]>', '', $string); $item['content'] = $this->ExtractContent($item['uri']);
return $string; return $item;
} }
$feed = self::URI.'feed/'; private function ExtractContent($url){
$html = $this->getSimpleHTMLDOM($feed) or $this->returnServerError('Could not request Numerama: '.$feed); if($this->get_cached_time($url) <= strtotime('-24 hours'))
$limit = 0; $this->remove_from_cache($url);
foreach($html->find('item') as $element) { $article_html = $this->get_cached($url) or $this->returnServerError('Could not request Numerama: '.$url);
if($limit < 5) { $contents = $article_html->find('section[class=related-article]', 0)->innertext = ''; // remove related articles block
$item = array(); $contents = '<img alt="" style="max-width:300px;" src="'.$article_html->find('meta[property=og:image]', 0)->getAttribute('content').'">'; // add post picture
$item['title'] = html_entity_decode(NumeramaStripCDATA($element->find('title', 0)->innertext)); return $contents . $article_html->find('article[class=post-content]', 0)->innertext; // extract the post
$item['author'] = NumeramaStripCDATA($element->find('dc:creator', 0)->innertext);
$item['uri'] = NumeramaStripCDATA($element->find('guid', 0)->plaintext);
$item['timestamp'] = strtotime($element->find('pubDate', 0)->plaintext);
$article_url = NumeramaStripCDATA($element->find('guid', 0)->plaintext);
if($this->get_cached_time($article_url) <= strtotime('-24 hours'))
$this->remove_from_cache($article_url);
$article_html = $this->get_cached($article_url) or $this->returnServerError('Could not request Numerama: '.$article_url);
$contents = $article_html->find('section[class=related-article]', 0)->innertext = ''; // remove related articles block
$contents = '<img alt="" style="max-width:300px;" src="'.$article_html->find('meta[property=og:image]', 0)->getAttribute('content').'">'; // add post picture
$contents = $contents.$article_html->find('article[class=post-content]', 0)->innertext; // extract the post
$item['content'] = $contents;
$this->items[] = $item;
$limit++;
}
}
} }
public function getCacheDuration() { public function getCacheDuration() {

17
lib/FeedExpander.php
View file

@ -6,7 +6,7 @@ abstract class FeedExpander extends HttpCachingBridgeAbstract {
private $uri; private $uri;
private $description; private $description;
public function collectExpandableDatas($url){ public function collectExpandableDatas($url, $maxItems = -1){
if(empty($url)){ if(empty($url)){
$this->returnServerError('There is no $url for this RSS expander'); $this->returnServerError('There is no $url for this RSS expander');
} }
@ -25,43 +25,46 @@ abstract class FeedExpander extends HttpCachingBridgeAbstract {
$this->debugMessage('Detected RSS format'); $this->debugMessage('Detected RSS format');
if(isset($rssContent->item[0])){ if(isset($rssContent->item[0])){
$this->debugMessage('Detected RSS 1.0 format'); $this->debugMessage('Detected RSS 1.0 format');
$this->collect_RSS_1_0_data($rssContent); $this->collect_RSS_1_0_data($rssContent, $maxItems);
} else { } else {
$this->debugMessage('Detected RSS 0.9x or 2.0 format'); $this->debugMessage('Detected RSS 0.9x or 2.0 format');
$this->collect_RSS_2_0_data($rssContent); $this->collect_RSS_2_0_data($rssContent, $maxItems);
} }
} elseif(isset($rssContent->entry[0])){ } elseif(isset($rssContent->entry[0])){
$this->debugMessage('Detected ATOM format'); $this->debugMessage('Detected ATOM format');
$this->collect_ATOM_data($rssContent); $this->collect_ATOM_data($rssContent, $maxItems);
} else { } else {
$this->debugMessage('Unknown feed format/version'); $this->debugMessage('Unknown feed format/version');
$this->returnServerError('The feed format is unknown!'); $this->returnServerError('The feed format is unknown!');
} }
} }
protected function collect_RSS_1_0_data($rssContent){ protected function collect_RSS_1_0_data($rssContent, $maxItems){
$this->load_RSS_2_0_feed_data($rssContent->channel[0]); $this->load_RSS_2_0_feed_data($rssContent->channel[0]);
foreach($rssContent->item as $item){ foreach($rssContent->item as $item){
$this->debugMessage('parsing item ' . var_export($item, true)); $this->debugMessage('parsing item ' . var_export($item, true));
$this->items[] = $this->parseItem($item); $this->items[] = $this->parseItem($item);
if($maxItems !== -1 && count($this->items) >= $maxItems) break;
} }
} }
protected function collect_RSS_2_0_data($rssContent){ protected function collect_RSS_2_0_data($rssContent, $maxItems){
$rssContent = $rssContent->channel[0]; $rssContent = $rssContent->channel[0];
$this->debugMessage('RSS content is ===========\n' . var_export($rssContent, true) . '==========='); $this->debugMessage('RSS content is ===========\n' . var_export($rssContent, true) . '===========');
$this->load_RSS_2_0_feed_data($rssContent); $this->load_RSS_2_0_feed_data($rssContent);
foreach($rssContent->item as $item){ foreach($rssContent->item as $item){
$this->debugMessage('parsing item ' . var_export($item, true)); $this->debugMessage('parsing item ' . var_export($item, true));
$this->items[] = $this->parseItem($item); $this->items[] = $this->parseItem($item);
if($maxItems !== -1 && count($this->items) >= $maxItems) break;
} }
} }
protected function collect_ATOM_data($content){ protected function collect_ATOM_data($content, $maxItems){
$this->load_ATOM_feed_data($content); $this->load_ATOM_feed_data($content);
foreach($content->entry as $item){ foreach($content->entry as $item){
$this->debugMessage('parsing item ' . var_export($item, true)); $this->debugMessage('parsing item ' . var_export($item, true));
$this->items[] = $this->parseItem($item); $this->items[] = $this->parseItem($item);
if($maxItems !== -1 && count($this->items) >= $maxItems) break;
} }
} }