blallo/rss-bridge
1
0
Fork 1
Code Issues Pull requests Releases Wiki Activity

[CNET] Update HTML parsing to reflect changes

Browse source 
HTML parsing wasn't working anymore due to changes on CNET's website.
This commit is contained in:
ORelio 2016-01-23 21:45:16 +01:00
parent c375384fba
commit 1e998ab6dd
1 changed files with 28 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
bridges/CNETBridge.php
View file

@ -3,22 +3,22 @@ class CNETBridge extends BridgeAbstract {
private $topicName = ''; private $topicName = '';
public function loadMetadatas() { public function loadMetadatas() {
$this->maintainer = "ORelio"; $this->maintainer = 'ORelio';
$this->name = "CNET News"; $this->name = 'CNET News';
$this->uri = "http://www.cnet.com/"; $this->uri = 'http://www.cnet.com/';
$this->description = "Returns the newest articles. <br /> You may specify a topic, else all topics are selected."; $this->description = 'Returns the newest articles. <br /> You may specify a topic found in some section URLs, else all topics are selected.';
$this->update = "2015-09-10"; $this->update = '2016-01-23';
$this->parameters[] = $this->parameters[] =
'[ '[
{ {
"name" : "Topic name", "name" : "Topic name",
"identifier" : "topic" "identifier" : "topic"
} }
]'; ]';
} }
public function collectData(array $param) { public function collectData(array $param) {
@ -39,7 +39,8 @@ class CNETBridge extends BridgeAbstract {
} }
function CleanArticle($article_html) { function CleanArticle($article_html) {
$article_html = '<p>'.substr($article_html, strpos($article_html, '</script></div><p>') + 18); $article_html = '<p>'.substr($article_html, strpos($article_html, '<p>') + 3);
$article_html = StripWithDelimiters($article_html, '<span class="credit">', '</span>');
$article_html = StripWithDelimiters($article_html, '<script>', '</script>'); $article_html = StripWithDelimiters($article_html, '<script>', '</script>');
$article_html = StripWithDelimiters($article_html, '<div class="shortcode related-links', '</div>'); $article_html = StripWithDelimiters($article_html, '<div class="shortcode related-links', '</div>');
$article_html = StripWithDelimiters($article_html, '<a class="clickToEnlarge">', '</a>'); $article_html = StripWithDelimiters($article_html, '<a class="clickToEnlarge">', '</a>');
@ -53,27 +54,29 @@ class CNETBridge extends BridgeAbstract {
$html = file_get_html($pageUrl) or $this->returnError('Could not request CNET: '.$pageUrl, 500); $html = file_get_html($pageUrl) or $this->returnError('Could not request CNET: '.$pageUrl, 500);
$limit = 0; $limit = 0;
foreach($html->find('div.socialSharingSmall') as $element) { foreach($html->find('div.assetBody') as $element) {
if ($limit < 4) { if ($limit < 8) {
$article_meta = json_decode(ExtractFromDelimiters($element->outertext, 'data-social-counts-options=\'', '\'>')); $article_title = trim($element->find('h2', 0)->plaintext);
$article_title = $article_meta->title; $article_uri = 'http://www.cnet.com'.($element->find('a', 0)->href);
$article_uri = $article_meta->url; $article_thumbnail = $element->parent()->find('img', 0)->src;
$article_timestamp = strtotime($element->find('time.assetTime', 0)->plaintext);
$article_author = trim($element->find('a[rel=author]', 0)->plaintext);
if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) { if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) {
$article_html = file_get_html($article_uri) or $this->returnError('Could not request CNET: '.$article_uri, 500); $article_html = file_get_html($article_uri) or $this->returnError('Could not request CNET: '.$article_uri, 500);
$article_timestamp = strtotime(ExtractFromDelimiters($article_html->innertext, '<time itemprop="datePublished" class="', '">'));
$article_thumbnail = $article_html->find('div.originalImage', 0); if (is_null($article_thumbnail))
$article_thumbnail = $article_html->find('div.originalImage', 0);
if (is_null($article_thumbnail)) if (is_null($article_thumbnail))
$article_thumbnail = $article_html->find('span.imageContainer', 0); $article_thumbnail = $article_html->find('span.imageContainer', 0);
if (!is_null($article_thumbnail)) if (is_object($article_thumbnail))
$article_thumbnail = $article_thumbnail->find('img', 0)->src; $article_thumbnail = $article_thumbnail->find('img', 0)->src;
$article_content = trim(CleanArticle(ExtractFromDelimiters($article_html, '</div></div></div><div class="col-8">', '<footer>'))); $article_content = trim(CleanArticle(ExtractFromDelimiters($article_html, '</div></div></div><div class="col-8">', '<footer>')));
$article_author = trim($article_html->find('a.author', 0)->plaintext);
$item = new \Item(); $item = new \Item();
$item->uri = $article_uri; $item->uri = $article_uri;