1: <?php
2:
3: /**
4: * Forgivingly lexes HTML (SGML-style) markup into tokens.
5: *
6: * A lexer parses a string of SGML-style markup and converts them into
7: * corresponding tokens. It doesn't check for well-formedness, although its
8: * internal mechanism may make this automatic (such as the case of
9: * HTMLPurifier_Lexer_DOMLex). There are several implementations to choose
10: * from.
11: *
12: * A lexer is HTML-oriented: it might work with XML, but it's not
13: * recommended, as we adhere to a subset of the specification for optimization
14: * reasons. This might change in the future. Also, most tokenizers are not
15: * expected to handle DTDs or PIs.
16: *
17: * This class should not be directly instantiated, but you may use create() to
18: * retrieve a default copy of the lexer. Being a supertype, this class
19: * does not actually define any implementation, but offers commonly used
20: * convenience functions for subclasses.
21: *
22: * @note The unit tests will instantiate this class for testing purposes, as
23: * many of the utility functions require a class to be instantiated.
24: * This means that, even though this class is not runnable, it will
25: * not be declared abstract.
26: *
27: * @par
28: *
29: * @note
30: * We use tokens rather than create a DOM representation because DOM would:
31: *
32: * @par
33: * -# Require more processing and memory to create,
34: * -# Is not streamable, and
35: * -# Has the entire document structure (html and body not needed).
36: *
37: * @par
38: * However, DOM is helpful in that it makes it easy to move around nodes
39: * without a lot of lookaheads to see when a tag is closed. This is a
40: * limitation of the token system and some workarounds would be nice.
41: */
42: class HTMLPurifier_Lexer
43: {
44:
45: /**
46: * Whether or not this lexer implements line-number/column-number tracking.
47: * If it does, set to true.
48: */
49: public $tracksLineNumbers = false;
50:
51: // -- STATIC ----------------------------------------------------------
52:
53: /**
54: * Retrieves or sets the default Lexer as a Prototype Factory.
55: *
56: * By default HTMLPurifier_Lexer_DOMLex will be returned. There are
57: * a few exceptions involving special features that only DirectLex
58: * implements.
59: *
60: * @note The behavior of this class has changed, rather than accepting
61: * a prototype object, it now accepts a configuration object.
62: * To specify your own prototype, set %Core.LexerImpl to it.
63: * This change in behavior de-singletonizes the lexer object.
64: *
65: * @param $config Instance of HTMLPurifier_Config
66: * @return Concrete lexer.
67: */
68: public static function create($config) {
69:
70: if (!($config instanceof HTMLPurifier_Config)) {
71: $lexer = $config;
72: trigger_error("Passing a prototype to
73: HTMLPurifier_Lexer::create() is deprecated, please instead
74: use %Core.LexerImpl", E_USER_WARNING);
75: } else {
76: $lexer = $config->get('Core.LexerImpl');
77: }
78:
79: $needs_tracking =
80: $config->get('Core.MaintainLineNumbers') ||
81: $config->get('Core.CollectErrors');
82:
83: $inst = null;
84: if (is_object($lexer)) {
85: $inst = $lexer;
86: } else {
87:
88: if (is_null($lexer)) { do {
89: // auto-detection algorithm
90:
91: if ($needs_tracking) {
92: $lexer = 'DirectLex';
93: break;
94: }
95:
96: if (
97: class_exists('DOMDocument') &&
98: method_exists('DOMDocument', 'loadHTML') &&
99: !extension_loaded('domxml')
100: ) {
101: // check for DOM support, because while it's part of the
102: // core, it can be disabled compile time. Also, the PECL
103: // domxml extension overrides the default DOM, and is evil
104: // and nasty and we shan't bother to support it
105: $lexer = 'DOMLex';
106: } else {
107: $lexer = 'DirectLex';
108: }
109:
110: } while(0); } // do..while so we can break
111:
112: // instantiate recognized string names
113: switch ($lexer) {
114: case 'DOMLex':
115: $inst = new HTMLPurifier_Lexer_DOMLex();
116: break;
117: case 'DirectLex':
118: $inst = new HTMLPurifier_Lexer_DirectLex();
119: break;
120: case 'PH5P':
121: $inst = new HTMLPurifier_Lexer_PH5P();
122: break;
123: default:
124: throw new HTMLPurifier_Exception("Cannot instantiate unrecognized Lexer type " . htmlspecialchars($lexer));
125: }
126: }
127:
128: if (!$inst) throw new HTMLPurifier_Exception('No lexer was instantiated');
129:
130: // once PHP DOM implements native line numbers, or we
131: // hack out something using XSLT, remove this stipulation
132: if ($needs_tracking && !$inst->tracksLineNumbers) {
133: throw new HTMLPurifier_Exception('Cannot use lexer that does not support line numbers with Core.MaintainLineNumbers or Core.CollectErrors (use DirectLex instead)');
134: }
135:
136: return $inst;
137:
138: }
139:
140: // -- CONVENIENCE MEMBERS ---------------------------------------------
141:
142: public function __construct() {
143: $this->_entity_parser = new HTMLPurifier_EntityParser();
144: }
145:
146: /**
147: * Most common entity to raw value conversion table for special entities.
148: */
149: protected $_special_entity2str =
150: array(
151: '"' => '"',
152: '&' => '&',
153: '<' => '<',
154: '>' => '>',
155: ''' => "'",
156: ''' => "'",
157: ''' => "'"
158: );
159:
160: /**
161: * Parses special entities into the proper characters.
162: *
163: * This string will translate escaped versions of the special characters
164: * into the correct ones.
165: *
166: * @warning
167: * You should be able to treat the output of this function as
168: * completely parsed, but that's only because all other entities should
169: * have been handled previously in substituteNonSpecialEntities()
170: *
171: * @param $string String character data to be parsed.
172: * @returns Parsed character data.
173: */
174: public function parseData($string) {
175:
176: // following functions require at least one character
177: if ($string === '') return '';
178:
179: // subtracts amps that cannot possibly be escaped
180: $num_amp = substr_count($string, '&') - substr_count($string, '& ') -
181: ($string[strlen($string)-1] === '&' ? 1 : 0);
182:
183: if (!$num_amp) return $string; // abort if no entities
184: $num_esc_amp = substr_count($string, '&');
185: $string = strtr($string, $this->_special_entity2str);
186:
187: // code duplication for sake of optimization, see above
188: $num_amp_2 = substr_count($string, '&') - substr_count($string, '& ') -
189: ($string[strlen($string)-1] === '&' ? 1 : 0);
190:
191: if ($num_amp_2 <= $num_esc_amp) return $string;
192:
193: // hmm... now we have some uncommon entities. Use the callback.
194: $string = $this->_entity_parser->substituteSpecialEntities($string);
195: return $string;
196: }
197:
198: /**
199: * Lexes an HTML string into tokens.
200: *
201: * @param $string String HTML.
202: * @return HTMLPurifier_Token array representation of HTML.
203: */
204: public function tokenizeHTML($string, $config, $context) {
205: trigger_error('Call to abstract class', E_USER_ERROR);
206: }
207:
208: /**
209: * Translates CDATA sections into regular sections (through escaping).
210: *
211: * @param $string HTML string to process.
212: * @returns HTML with CDATA sections escaped.
213: */
214: protected static function escapeCDATA($string) {
215: return preg_replace_callback(
216: '/<!\[CDATA\[(.+?)\]\]>/s',
217: array('HTMLPurifier_Lexer', 'CDATACallback'),
218: $string
219: );
220: }
221:
222: /**
223: * Special CDATA case that is especially convoluted for <script>
224: */
225: protected static function escapeCommentedCDATA($string) {
226: return preg_replace_callback(
227: '#<!--//--><!\[CDATA\[//><!--(.+?)//--><!\]\]>#s',
228: array('HTMLPurifier_Lexer', 'CDATACallback'),
229: $string
230: );
231: }
232:
233: /**
234: * Special Internet Explorer conditional comments should be removed.
235: */
236: protected static function removeIEConditional($string) {
237: return preg_replace(
238: '#<!--\[if [^>]+\]>.*?<!\[endif\]-->#si', // probably should generalize for all strings
239: '',
240: $string
241: );
242: }
243:
244: /**
245: * Callback function for escapeCDATA() that does the work.
246: *
247: * @warning Though this is public in order to let the callback happen,
248: * calling it directly is not recommended.
249: * @params $matches PCRE matches array, with index 0 the entire match
250: * and 1 the inside of the CDATA section.
251: * @returns Escaped internals of the CDATA section.
252: */
253: protected static function CDATACallback($matches) {
254: // not exactly sure why the character set is needed, but whatever
255: return htmlspecialchars($matches[1], ENT_COMPAT, 'UTF-8');
256: }
257:
258: /**
259: * Takes a piece of HTML and normalizes it by converting entities, fixing
260: * encoding, extracting bits, and other good stuff.
261: * @todo Consider making protected
262: */
263: public function normalize($html, $config, $context) {
264:
265: // normalize newlines to \n
266: if ($config->get('Core.NormalizeNewlines')) {
267: $html = str_replace("\r\n", "\n", $html);
268: $html = str_replace("\r", "\n", $html);
269: }
270:
271: if ($config->get('HTML.Trusted')) {
272: // escape convoluted CDATA
273: $html = $this->escapeCommentedCDATA($html);
274: }
275:
276: // escape CDATA
277: $html = $this->escapeCDATA($html);
278:
279: $html = $this->removeIEConditional($html);
280:
281: // extract body from document if applicable
282: if ($config->get('Core.ConvertDocumentToFragment')) {
283: $e = false;
284: if ($config->get('Core.CollectErrors')) {
285: $e =& $context->get('ErrorCollector');
286: }
287: $new_html = $this->extractBody($html);
288: if ($e && $new_html != $html) {
289: $e->send(E_WARNING, 'Lexer: Extracted body');
290: }
291: $html = $new_html;
292: }
293:
294: // expand entities that aren't the big five
295: $html = $this->_entity_parser->substituteNonSpecialEntities($html);
296:
297: // clean into wellformed UTF-8 string for an SGML context: this has
298: // to be done after entity expansion because the entities sometimes
299: // represent non-SGML characters (horror, horror!)
300: $html = HTMLPurifier_Encoder::cleanUTF8($html);
301:
302: // if processing instructions are to removed, remove them now
303: if ($config->get('Core.RemoveProcessingInstructions')) {
304: $html = preg_replace('#<\?.+?\?>#s', '', $html);
305: }
306:
307: return $html;
308: }
309:
310: /**
311: * Takes a string of HTML (fragment or document) and returns the content
312: * @todo Consider making protected
313: */
314: public function extractBody($html) {
315: $matches = array();
316: $result = preg_match('!<body[^>]*>(.*)</body>!is', $html, $matches);
317: if ($result) {
318: return $matches[1];
319: } else {
320: return $html;
321: }
322: }
323:
324: }
325:
326: // vim: et sw=4 sts=4
327: