blallo/rss-bridge
1
0
Fork 1
Code Issues Pull requests Releases Wiki Activity

[bridges] Define max items and clear caches

Browse source
This commit is contained in:
logmanoriginal 2016-09-05 20:26:45 +02:00
parent f1fb527607
commit 2861a855e4
12 changed files with 43 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bridges/CADBridge.php
View file

@ -6,7 +6,7 @@ class CADBridge extends FeedExpander {
const DESCRIPTION = "Returns the newest articles."; const DESCRIPTION = "Returns the newest articles.";
public function collectData(){ public function collectData(){
$this->collectExpandableDatas('http://cdn2.cad-comic.com/rss.xml'); $this->collectExpandableDatas('http://cdn2.cad-comic.com/rss.xml', 10);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){
@ -16,6 +16,8 @@ class CADBridge extends FeedExpander {
} }
private function CADExtractContent($url) { private function CADExtractContent($url) {
if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$html3 = $this->get_cached($url); $html3 = $this->get_cached($url);
// The request might fail due to missing https support or wrong URL // The request might fail due to missing https support or wrong URL

4
bridges/CommonDreamsBridge.php
View file

@ -7,7 +7,7 @@ class CommonDreamsBridge extends FeedExpander {
const DESCRIPTION = "Returns the newest articles."; const DESCRIPTION = "Returns the newest articles.";
public function collectData(){ public function collectData(){
$this->collectExpandableDatas('http://www.commondreams.org/rss.xml'); $this->collectExpandableDatas('http://www.commondreams.org/rss.xml', 10);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){
@ -17,6 +17,8 @@ class CommonDreamsBridge extends FeedExpander {
} }
private function CommonDreamsExtractContent($url) { private function CommonDreamsExtractContent($url) {
if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$html3 = $this->get_cached($url); $html3 = $this->get_cached($url);
$text = $html3->find('div[class=field--type-text-with-summary]', 0)->innertext; $text = $html3->find('div[class=field--type-text-with-summary]', 0)->innertext;
$html3->clear(); $html3->clear();

6
bridges/DauphineLibereBridge.php
View file

@ -37,7 +37,7 @@ class DauphineLibereBridge extends FeedExpander {
$url = self::URI . $this->getInput('u') . '/rss'; $url = self::URI . $this->getInput('u') . '/rss';
} }
$this->collectExpandableDatas($url); $this->collectExpandableDatas($url, 10);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){
@ -47,7 +47,9 @@ class DauphineLibereBridge extends FeedExpander {
} }
private function ExtractContent($url) { private function ExtractContent($url) {
$html2 = $this->getSimpleHTMLDOM($url); if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$html2 = $this->get_cached($url);
$text = $html2->find('div.column', 0)->innertext; $text = $html2->find('div.column', 0)->innertext;
$text = preg_replace('@<script[^>]*?>.*?</script>@si', '', $text); $text = preg_replace('@<script[^>]*?>.*?</script>@si', '', $text);
return $text; return $text;

4
bridges/DeveloppezDotComBridge.php
View file

@ -7,7 +7,7 @@ class DeveloppezDotComBridge extends FeedExpander {
const DESCRIPTION = "Returns the 15 newest posts from DeveloppezDotCom (full text)."; const DESCRIPTION = "Returns the 15 newest posts from DeveloppezDotCom (full text).";
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'index/rss'); $this->collectExpandableDatas(self::URI . 'index/rss', 15);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){
@ -42,6 +42,8 @@ class DeveloppezDotComBridge extends FeedExpander {
} }
private function DeveloppezDotComExtractContent($url) { private function DeveloppezDotComExtractContent($url) {
if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$articleHTMLContent = $this->get_cached($url); $articleHTMLContent = $this->get_cached($url);
$text = $this->convert_smart_quotes($articleHTMLContent->find('div.content', 0)->innertext); $text = $this->convert_smart_quotes($articleHTMLContent->find('div.content', 0)->innertext);
$text = utf8_encode($text); $text = utf8_encode($text);

16
bridges/FuturaSciencesBridge.php
View file

@ -78,20 +78,22 @@ class FuturaSciencesBridge extends FeedExpander {
) )
)); ));
public function collectData(){ public function collectData(){
$url = self::URI . 'rss/' . $this->getInput('feed') . '.xml'; $url = self::URI . 'rss/' . $this->getInput('feed') . '.xml';
$this->collectExpandableDatas($url); $this->collectExpandableDatas($url, 10);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){
$item = $this->parseRSS_2_0_Item($newsItem); $item = $this->parseRSS_2_0_Item($newsItem);
$item['uri'] = str_replace('#xtor=RSS-8', '', $item['uri']); $item['uri'] = str_replace('#xtor=RSS-8', '', $item['uri']);
if($this->get_cached_time($item['uri']) <= strtotime('-24 hours'))
$this->remove_from_cache($item['uri']);
$article = $this->get_cached($item['uri']) $article = $this->get_cached($item['uri'])
or $this->returnServerError('Could not request Futura-Sciences: ' . $item['uri']); or $this->returnServerError('Could not request Futura-Sciences: ' . $item['uri']);
$item['content'] = $this->ExtractArticleContent($article); $item['content'] = $this->ExtractArticleContent($article);
$item['author'] = empty($this->ExtractAuthor($article)) ? $item['author'] : $this->ExtractAuthor($article); $item['author'] = empty($this->ExtractAuthor($article)) ? $item['author'] : $this->ExtractAuthor($article);
return $item; return $item;
} }
function StripWithDelimiters($string, $start, $end) { function StripWithDelimiters($string, $start, $end) {
while (strpos($string, $start) !== false) { while (strpos($string, $start) !== false) {

4
bridges/LeJournalDuGeekBridge.php
View file

@ -7,7 +7,7 @@ class LeJournalDuGeekBridge extends FeedExpander {
const DESCRIPTION = "Returns the 5 newest posts from LeJournalDuGeek (full text)."; const DESCRIPTION = "Returns the 5 newest posts from LeJournalDuGeek (full text).";
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'rss'); $this->collectExpandableDatas(self::URI . 'rss', 5);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){
@ -17,6 +17,8 @@ class LeJournalDuGeekBridge extends FeedExpander {
} }
private function LeJournalDuGeekExtractContent($url) { private function LeJournalDuGeekExtractContent($url) {
if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$articleHTMLContent = $this->get_cached($url); $articleHTMLContent = $this->get_cached($url);
$text = $articleHTMLContent->find('div.post-content', 0)->innertext; $text = $articleHTMLContent->find('div.post-content', 0)->innertext;

16
bridges/LeMondeInformatiqueBridge.php
View file

@ -6,18 +6,20 @@ class LeMondeInformatiqueBridge extends FeedExpander {
const URI = "http://www.lemondeinformatique.fr/"; const URI = "http://www.lemondeinformatique.fr/";
const DESCRIPTION = "Returns the newest articles."; const DESCRIPTION = "Returns the newest articles.";
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'rss/rss.xml'); $this->collectExpandableDatas(self::URI . 'rss/rss.xml', 10);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){
$item = $this->parseRSS_1_0_Item($newsItem); $item = $this->parseRSS_1_0_Item($newsItem);
if($this->get_cached_time($item['uri']) <= strtotime('-24 hours'))
$this->remove_from_cache($item['uri']);
$article_html = $this->get_cached($item['uri']) $article_html = $this->get_cached($item['uri'])
or $this->returnServerError('Could not request LeMondeInformatique: ' . $item['uri']); or $this->returnServerError('Could not request LeMondeInformatique: ' . $item['uri']);
$item['content'] = $this->CleanArticle($article_html->find('div#article', 0)->innertext); $item['content'] = $this->CleanArticle($article_html->find('div#article', 0)->innertext);
$item['title'] = $article_html->find('h1.cleanprint-title', 0)->plaintext; $item['title'] = $article_html->find('h1.cleanprint-title', 0)->plaintext;
return $item; return $item;
} }
function StripCDATA($string) { function StripCDATA($string) {
$string = str_replace('<![CDATA[', '', $string); $string = str_replace('<![CDATA[', '', $string);

2
bridges/LichessBridge.php
View file

@ -7,7 +7,7 @@ class LichessBridge extends FeedExpander {
const DESCRIPTION = 'Returns the 5 newest posts from the Lichess blog (full text)'; const DESCRIPTION = 'Returns the 5 newest posts from the Lichess blog (full text)';
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . '.atom'); $this->collectExpandableDatas(self::URI . '.atom', 5);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){

4
bridges/NextInpactBridge.php
View file

@ -7,7 +7,7 @@ class NextInpactBridge extends FeedExpander {
const DESCRIPTION = "Returns the newest articles."; const DESCRIPTION = "Returns the newest articles.";
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'rss/news.xml'); $this->collectExpandableDatas(self::URI . 'rss/news.xml', 10);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){
@ -17,6 +17,8 @@ class NextInpactBridge extends FeedExpander {
} }
private function ExtractContent($url) { private function ExtractContent($url) {
if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$html2 = $this->get_cached($url); $html2 = $this->get_cached($url);
$text = '<p><em>'.$html2->find('span.sub_title', 0)->innertext.'</em></p>' $text = '<p><em>'.$html2->find('span.sub_title', 0)->innertext.'</em></p>'
.'<p><img src="'.$html2->find('div.container_main_image_article', 0)->find('img.dedicated',0)->src.'" alt="-" /></p>' .'<p><img src="'.$html2->find('div.container_main_image_article', 0)->find('img.dedicated',0)->src.'" alt="-" /></p>'

2
bridges/NextgovBridge.php
View file

@ -26,7 +26,7 @@ class NextgovBridge extends FeedExpander {
)); ));
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'rss/' . $this->getInput('category') . '/'); $this->collectExpandableDatas(self::URI . 'rss/' . $this->getInput('category') . '/', 10);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){

4
bridges/NiceMatinBridge.php
View file

@ -7,7 +7,7 @@ class NiceMatinBridge extends FeedExpander {
const DESCRIPTION = "Returns the 10 newest posts from NiceMatin (full text)"; const DESCRIPTION = "Returns the 10 newest posts from NiceMatin (full text)";
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'derniere-minute/rss'); $this->collectExpandableDatas(self::URI . 'derniere-minute/rss', 10);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){
@ -17,6 +17,8 @@ class NiceMatinBridge extends FeedExpander {
} }
private function NiceMatinExtractContent($url) { private function NiceMatinExtractContent($url) {
if($this->get_cached_time($url) <= strtotime('-24 hours'))
$this->remove_from_cache($url);
$html = $this->get_cached($url); $html = $this->get_cached($url);
if(!$html) if(!$html)
return 'Could not acquire content from url: ' . $url . '!'; return 'Could not acquire content from url: ' . $url . '!';

2
bridges/NumeramaBridge.php
View file

@ -7,7 +7,7 @@ class NumeramaBridge extends FeedExpander {
const DESCRIPTION = 'Returns the 5 newest posts from Numerama (full text)'; const DESCRIPTION = 'Returns the 5 newest posts from Numerama (full text)';
public function collectData(){ public function collectData(){
$this->collectExpandableDatas(self::URI . 'feed/'); $this->collectExpandableDatas(self::URI . 'feed/', 5);
} }
protected function parseItem($newsItem){ protected function parseItem($newsItem){