move readability libs to system location

lib/readability/JSLikeHTMLElement.php

@@ -107,4 +107,4 @@ class JSLikeHTMLElement extends DOMElement
 		return '['.$this->tagName.']';
 	}
 }
-?>
+?>

lib/readability/Readability.php

@@ -2,6 +2,8 @@
 /** 
 * Arc90's Readability ported to PHP for FiveFilters.org
 * Based on readability.js version 1.7.1 (without multi-page support)
+* Updated to allow HTML5 parsing with html5lib
+* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds
 * ------------------------------------------------------
 * Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
 * Arc90's project URL: http://lab.arc90.com/experiments/readability/
@@ -10,7 +12,7 @@
 * More information: http://fivefilters.org/content-only/
 * License: Apache License, Version 2.0
 * Requires: PHP5
-* Date: 2010-10-29
+* Date: 2012-09-19
 * 
 * Differences between the PHP port and the original
 * ------------------------------------------------------
@@ -44,9 +46,10 @@
 
 // This class allows us to do JavaScript like assignements to innerHTML
 require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');
+libxml_use_internal_errors(true);
 
 // Alternative usage (for testing only!)
-// uncomment the lins below and call Readability.php in your browser 
+// uncomment the lines below and call Readability.php in your browser 
 // passing it the URL of the page you'd like content from, e.g.:
 // Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php
 
@@ -72,6 +75,7 @@ class Readability
 	public $dom;
 	public $url = null; // optional - URL where HTML was retrieved
 	public $debug = false;
+	public $lightClean = true; // preserves more content (experimental) added 2012-09-19
 	protected $body = null; // 
 	protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
 	protected $flags = 7; // 1 | 2 | 4;   // Start with all flags set.
@@ -82,17 +86,17 @@ class Readability
 	* Defined up here so we don't instantiate them repeatedly in loops.
 	**/
 	public $regexps = array(
-		'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i',
+		'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',
 		'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
-		'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i',
+		'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',
-		'negative' => '/combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
+		'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
 		'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
 		'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
 		'replaceFonts' => '/<(\/?)font[^>]*>/i',
 		// 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
 		'normalize' => '/\s{2,}/',
 		'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',
-		'video' => '/http:\/\/(www\.)?(youtube|vimeo)\.com/i',
+		'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
 		'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
 	);	
 	
@@ -105,19 +109,24 @@ class Readability
 	* Create instance of Readability
 	* @param string UTF-8 encoded string
 	* @param string (optional) URL associated with HTML (used for footnotes)
+	* @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
 	*/	
-	function __construct($html, $url=null)
+	function __construct($html, $url=null, $parser='libxml')
 	{
+		$this->url = $url;
 		/* Turn all double br's into p's */
 		$html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
 		$html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
 		$html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
-		$this->dom = new DOMDocument();
-		$this->dom->preserveWhiteSpace = false;
-		$this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
 		if (trim($html) == '') $html = '<html></html>';
-		@$this->dom->loadHTML($html);
+		if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {
-		$this->url = $url;
+			// all good
+		} else {
+			$this->dom = new DOMDocument();
+			$this->dom->preserveWhiteSpace = false;
+			@$this->dom->loadHTML($html);
+		}
+		$this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
 	}
 
 	/**
@@ -150,6 +159,7 @@ class Readability
 	**/
 	public function init()
 	{
+		if (!isset($this->dom->documentElement)) return false;
 		$this->removeScripts($this->dom);
 		//die($this->getInnerHTML($this->dom->documentElement));
 		
@@ -212,7 +222,7 @@ class Readability
 	* Debug
 	*/
 	protected function dbg($msg) {
-		if ($this->debug) echo '* ',$msg, '<br />', "\n";
+		if ($this->debug) echo '* ',$msg, "\n";
 	}
 	
 	/**
@@ -293,7 +303,6 @@ class Readability
 			$this->body = $this->dom->createElement('body');
 			$this->dom->documentElement->appendChild($this->body);
 		}
-		
 		$this->body->setAttribute('id', 'readabilityBody');
 
 		/* Remove all style tags in head */
@@ -420,7 +429,7 @@ class Readability
 		* If there is only one h2, they are probably using it
 		* as a header and not a subheader, so remove it since we already have a header.
 		***/
-		if ($articleContent->getElementsByTagName('h2')->length == 1) {
+		if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {
 			$this->clean($articleContent, 'h2'); 
 		}
 		$this->clean($articleContent, 'iframe');
@@ -439,8 +448,9 @@ class Readability
 			$imgCount    = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
 			$embedCount  = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
 			$objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
+			$iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;
 			
-			if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
+			if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
 			{
 				$articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
 			}
@@ -664,9 +674,20 @@ class Readability
 		if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
 		{
 			$topCandidate = $this->dom->createElement('div');
-			$topCandidate->innerHTML = ($page instanceof DOMDocument) ? $page->saveXML($page->documentElement) : $page->innerHTML;
+			if ($page instanceof DOMDocument) {
-			$page->innerHTML = '';
+				if (!isset($page->documentElement)) {
-			$page->appendChild($topCandidate);
+					// we don't have a body either? what a mess! :)
+				} else {
+					$topCandidate->innerHTML = $page->documentElement->innerHTML;
+					$page->documentElement->innerHTML = '';
+					$this->reinitBody();
+					$page->documentElement->appendChild($topCandidate);
+				}
+			} else {
+				$topCandidate->innerHTML = $page->innerHTML;
+				$page->innerHTML = '';
+				$page->appendChild($topCandidate);
+			}
 			$this->initializeNode($topCandidate);
 		}
 
@@ -677,7 +698,11 @@ class Readability
 		$articleContent        = $this->dom->createElement('div');
 		$articleContent->setAttribute('id', 'readability-content');
 		$siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
-		$siblingNodes          = $topCandidate->parentNode->childNodes;
+		$siblingNodes          = @$topCandidate->parentNode->childNodes;
+		if (!isset($siblingNodes)) {
+			$siblingNodes = new stdClass;
+			$siblingNodes->length = 0;
+		}
 
 		for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
 		{
@@ -769,7 +794,9 @@ class Readability
 		**/
 		if (strlen($this->getInnerText($articleContent, false)) < 250)
 		{
-			$this->body->innerHTML = $this->bodyCache;
+			// TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7
+			// in the meantime, we check and create an empty element if it's not there.
+			$this->reinitBody();
 			
 			if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
 				$this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
@@ -846,6 +873,7 @@ class Readability
 	* @return void
 	*/
 	public function cleanStyles($e) {
+		if (!is_object($e)) return;
 		$elems = $e->getElementsByTagName('*');
 		foreach ($elems as $elem) {
 			$elem->removeAttribute('style');
@@ -928,13 +956,15 @@ class Readability
 	* Clean a node of all elements of type "tag".
 	* (Unless it's a youtube/vimeo video. People love movies.)
 	*
+	* Updated 2012-09-18 to preserve youtube/vimeo iframes
+	*
 	* @param DOMElement $e
 	* @param string $tag
 	* @return void
 	*/
 	public function clean($e, $tag) {
 		$targetList = $e->getElementsByTagName($tag);
-		$isEmbed = ($tag == 'object' || $tag == 'embed');
+		$isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
 		
 		for ($y=$targetList->length-1; $y >= 0; $y--) {
 			/* Allow youtube and vimeo videos through as people usually want to see those. */
@@ -999,12 +1029,19 @@ class Readability
 				$img    = $tagsList->item($i)->getElementsByTagName('img')->length;
 				$li     = $tagsList->item($i)->getElementsByTagName('li')->length-100;
 				$input  = $tagsList->item($i)->getElementsByTagName('input')->length;
+				$a 		= $tagsList->item($i)->getElementsByTagName('a')->length;
 
 				$embedCount = 0;
 				$embeds = $tagsList->item($i)->getElementsByTagName('embed');
 				for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
 					if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
-					$embedCount++; 
+						$embedCount++; 
+					}
+				}
+				$embeds = $tagsList->item($i)->getElementsByTagName('iframe');
+				for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
+					if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
+						$embedCount++; 
 					}
 				}
 
@@ -1012,23 +1049,58 @@ class Readability
 				$contentLength = strlen($this->getInnerText($tagsList->item($i)));
 				$toRemove      = false;
 
-				if ( $img > $p ) {
+				if ($this->lightClean) {
-					$toRemove = true;
+					$this->dbg('Light clean...');
-				} else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
+					if ( ($img > $p) && ($img > 4) ) {
-					$toRemove = true;
+						$this->dbg(' more than 4 images and more image elements than paragraph elements');
-				} else if ( $input > floor($p/3) ) {
+						$toRemove = true;
-					$toRemove = true; 
+					} else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
-				} else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
+						$this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
-					$toRemove = true;
+						$toRemove = true;
-				} else if($weight < 25 && $linkDensity > 0.2) {
+					} else if ( $input > floor($p/3) ) {
-					$toRemove = true;
+						$this->dbg(' too many <input> elements');
-				} else if($weight >= 25 && $linkDensity > 0.5) {
+						$toRemove = true; 
-					$toRemove = true;
+					} else if ($contentLength < 10 && ($embedCount === 0 && ($img === 0 || $img > 2))) {
-				} else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
+						$this->dbg(' content length less than 10 chars, 0 embeds and either 0 images or more than 2 images');
-					$toRemove = true;
+						$toRemove = true;
+					} else if($weight < 25 && $linkDensity > 0.2) {
+						$this->dbg(' weight smaller than 25 and link density above 0.2');
+						$toRemove = true;
+					} else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
+						$this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');
+						$toRemove = true;
+					} else if($embedCount > 3) {
+						$this->dbg(' more than 3 embeds');
+						$toRemove = true;
+					}
+				} else {
+					$this->dbg('Standard clean...');
+					if ( $img > $p ) {
+						$this->dbg(' more image elements than paragraph elements');
+						$toRemove = true;
+					} else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
+						$this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
+						$toRemove = true;
+					} else if ( $input > floor($p/3) ) {
+						$this->dbg(' too many <input> elements');
+						$toRemove = true; 
+					} else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
+						$this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
+						$toRemove = true;
+					} else if($weight < 25 && $linkDensity > 0.2) {
+						$this->dbg(' weight smaller than 25 and link density above 0.2');
+						$toRemove = true;
+					} else if($weight >= 25 && $linkDensity > 0.5) {
+						$this->dbg(' weight above 25 but link density greater than 0.5');
+						$toRemove = true;
+					} else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
+						$this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');
+						$toRemove = true;
+					}
 				}
 
 				if ($toRemove) {
+					//$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
 					$tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
 				}
 			}
@@ -1063,5 +1135,18 @@ class Readability
 	public function removeFlag($flag) {
 		$this->flags = $this->flags & ~$flag;
 	}
+	
+	/**
+	 * Will recreate previously deleted body property
+	 *
+	 * @return void
+	 */
+	protected function reinitBody() {
+		if (!isset($this->body->childNodes)) {
+			$this->body = $this->dom->createElement('body');
+			$this->body->innerHTML = $this->bodyCache;
+		}
+	}
+		
 }
-?>
+?>

plugins/af_readability/init.php

@@ -96,7 +96,7 @@ class Af_Readability extends Plugin {
 		$key = array_search($article["feed"]["id"], $enabled_feeds);
 		if ($key === FALSE) return $article;
 
-		if (!class_exists("Readability")) require_once(__DIR__ . "/classes/Readability.php");
+		if (!class_exists("Readability")) require_once(dirname(dirname(__DIR__)). "/lib/readability/Readability.php");
 
 		if (function_exists("curl_init")) {
 			$ch = curl_init($article["link"]);

plugins/af_redditimgur/classes/JSLikeHTMLElement.php

@@ -1,110 +0,0 @@
-<?php
-/**
-* JavaScript-like HTML DOM Element
-*
-* This class extends PHP's DOMElement to allow
-* users to get and set the innerHTML property of
-* HTML elements in the same way it's done in 
-* JavaScript.
-*
-* Example usage:
-* @code
-* require_once 'JSLikeHTMLElement.php';
-* header('Content-Type: text/plain');
-* $doc = new DOMDocument();
-* $doc->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
-* $doc->loadHTML('<div><p>Para 1</p><p>Para 2</p></div>');
-* $elem = $doc->getElementsByTagName('div')->item(0);
-* 
-* // print innerHTML
-* echo $elem->innerHTML; // prints '<p>Para 1</p><p>Para 2</p>'
-* echo "\n\n";
-* 
-* // set innerHTML
-* $elem->innerHTML = '<a href="http://fivefilters.org">FiveFilters.org</a>';
-* echo $elem->innerHTML; // prints '<a href="http://fivefilters.org">FiveFilters.org</a>'
-* echo "\n\n";
-* 
-* // print document (with our changes)
-* echo $doc->saveXML();
-* @endcode
-*
-* @author Keyvan Minoukadeh - http://www.keyvan.net - keyvan@keyvan.net
-* @see http://fivefilters.org (the project this was written for)
-*/
-class JSLikeHTMLElement extends DOMElement
-{
-	/**
-	* Used for setting innerHTML like it's done in JavaScript:
-	* @code
-	* $div->innerHTML = '<h2>Chapter 2</h2><p>The story begins...</p>';
-	* @endcode
-	*/
-	public function __set($name, $value) {
-		if ($name == 'innerHTML') {
-			// first, empty the element
-			for ($x=$this->childNodes->length-1; $x>=0; $x--) {
-				$this->removeChild($this->childNodes->item($x));
-			}
-			// $value holds our new inner HTML
-			if ($value != '') {
-				$f = $this->ownerDocument->createDocumentFragment();
-				// appendXML() expects well-formed markup (XHTML)
-				$result = @$f->appendXML($value); // @ to suppress PHP warnings
-				if ($result) {
-					if ($f->hasChildNodes()) $this->appendChild($f);
-				} else {
-					// $value is probably ill-formed
-					$f = new DOMDocument();
-					$value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8');
-					// Using <htmlfragment> will generate a warning, but so will bad HTML
-					// (and by this point, bad HTML is what we've got).
-					// We use it (and suppress the warning) because an HTML fragment will 
-					// be wrapped around <html><body> tags which we don't really want to keep.
-					// Note: despite the warning, if loadHTML succeeds it will return true.
-					$result = @$f->loadHTML('<htmlfragment>'.$value.'</htmlfragment>');
-					if ($result) {
-						$import = $f->getElementsByTagName('htmlfragment')->item(0);
-						foreach ($import->childNodes as $child) {
-							$importedNode = $this->ownerDocument->importNode($child, true);
-							$this->appendChild($importedNode);
-						}
-					} else {
-						// oh well, we tried, we really did. :(
-						// this element is now empty
-					}
-				}
-			}
-		} else {
-			$trace = debug_backtrace();
-			trigger_error('Undefined property via __set(): '.$name.' in '.$trace[0]['file'].' on line '.$trace[0]['line'], E_USER_NOTICE);
-		}
-	}
-
-	/**
-	* Used for getting innerHTML like it's done in JavaScript:
-	* @code
-	* $string = $div->innerHTML;
-	* @endcode
-	*/	
-	public function __get($name)
-	{
-		if ($name == 'innerHTML') {
-			$inner = '';
-			foreach ($this->childNodes as $child) {
-				$inner .= $this->ownerDocument->saveXML($child);
-			}
-			return $inner;
-		}
-
-		$trace = debug_backtrace();
-		trigger_error('Undefined property via __get(): '.$name.' in '.$trace[0]['file'].' on line '.$trace[0]['line'], E_USER_NOTICE);
-		return null;
-	}
-
-	public function __toString()
-	{
-		return '['.$this->tagName.']';
-	}
-}
-?>

plugins/af_redditimgur/init.php

@@ -243,7 +243,7 @@ class Af_RedditImgur extends Plugin {
 			if (function_exists("curl_init") && !$found && $this->host->get($this, "enable_readability") &&
 				mb_strlen(strip_tags($article["content"])) <= 150) {
 
-				if (!class_exists("Readability")) require_once(__DIR__ . "/classes/Readability.php");
+				if (!class_exists("Readability")) require_once(dirname(dirname(__DIR__)). "/lib/readability/Readability.php");
 
 				$content_link = $xpath->query("(//a[contains(., '[link]')])")->item(0);