svn-gvsig-desktop / tags / v1_0_2_Build_898 / extensions / extScripting / scripts / jython / Lib / sgmllib.py @ 10513
History | View | Annotate | Download (17 KB)
1 |
"""A parser for SGML, using the derived class as a static DTD."""
|
---|---|
2 |
|
3 |
# XXX This only supports those SGML features used by HTML.
|
4 |
|
5 |
# XXX There should be a way to distinguish between PCDATA (parsed
|
6 |
# character data -- the normal case), RCDATA (replaceable character
|
7 |
# data -- only char and entity references and end tags are special)
|
8 |
# and CDATA (character data -- only end tags are special).
|
9 |
|
10 |
|
11 |
import re |
12 |
import string |
13 |
|
14 |
__all__ = ["SGMLParser"]
|
15 |
|
16 |
# Regular expressions used for parsing
|
17 |
|
18 |
interesting = re.compile('[&<]')
|
19 |
incomplete = re.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|'
|
20 |
'<([a-zA-Z][^<>]*|'
|
21 |
'/([a-zA-Z][^<>]*)?|'
|
22 |
'![^<>]*)?')
|
23 |
|
24 |
entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
|
25 |
charref = re.compile('&#([0-9]+)[^0-9]')
|
26 |
|
27 |
starttagopen = re.compile('<[>a-zA-Z]')
|
28 |
shorttagopen = re.compile('<[a-zA-Z][-.a-zA-Z0-9]*/')
|
29 |
shorttag = re.compile('<([a-zA-Z][-.a-zA-Z0-9]*)/([^/]*)/')
|
30 |
piopen = re.compile('<\?')
|
31 |
piclose = re.compile('>')
|
32 |
endtagopen = re.compile('</[<>a-zA-Z]')
|
33 |
endbracket = re.compile('[<>]')
|
34 |
special = re.compile('<![^<>]*>')
|
35 |
commentopen = re.compile('<!--')
|
36 |
commentclose = re.compile(r'--\s*>')
|
37 |
tagfind = re.compile('[a-zA-Z][-_.a-zA-Z0-9]*')
|
38 |
attrfind = re.compile( |
39 |
r'\s*([a-zA-Z_][-.a-zA-Z_0-9]*)(\s*=\s*'
|
40 |
r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./:;+*%?!&$\(\)_#=~\'"]*))?')
|
41 |
|
42 |
declname = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*')
|
43 |
declstringlit = re.compile(r'(\'[^\']*\'|"[^"]*")\s*')
|
44 |
|
45 |
|
46 |
class SGMLParseError(RuntimeError): |
47 |
"""Exception raised for all parse errors."""
|
48 |
pass
|
49 |
|
50 |
|
51 |
# SGML parser base class -- find tags and call handler functions.
|
52 |
# Usage: p = SGMLParser(); p.feed(data); ...; p.close().
|
53 |
# The dtd is defined by deriving a class which defines methods
|
54 |
# with special names to handle tags: start_foo and end_foo to handle
|
55 |
# <foo> and </foo>, respectively, or do_foo to handle <foo> by itself.
|
56 |
# (Tags are converted to lower case for this purpose.) The data
|
57 |
# between tags is passed to the parser by calling self.handle_data()
|
58 |
# with some data as argument (the data may be split up in arbitrary
|
59 |
# chunks). Entity references are passed by calling
|
60 |
# self.handle_entityref() with the entity reference as argument.
|
61 |
|
62 |
class SGMLParser: |
63 |
|
64 |
# Interface -- initialize and reset this instance
|
65 |
def __init__(self, verbose=0): |
66 |
self.verbose = verbose
|
67 |
self.reset()
|
68 |
|
69 |
# Interface -- reset this instance. Loses all unprocessed data
|
70 |
def reset(self): |
71 |
self.rawdata = '' |
72 |
self.stack = []
|
73 |
self.lasttag = '???' |
74 |
self.nomoretags = 0 |
75 |
self.literal = 0 |
76 |
|
77 |
# For derived classes only -- enter literal mode (CDATA) till EOF
|
78 |
def setnomoretags(self): |
79 |
self.nomoretags = self.literal = 1 |
80 |
|
81 |
# For derived classes only -- enter literal mode (CDATA)
|
82 |
def setliteral(self, *args): |
83 |
self.literal = 1 |
84 |
|
85 |
# Interface -- feed some data to the parser. Call this as
|
86 |
# often as you want, with as little or as much text as you
|
87 |
# want (may include '\n'). (This just saves the text, all the
|
88 |
# processing is done by goahead().)
|
89 |
def feed(self, data): |
90 |
self.rawdata = self.rawdata + data |
91 |
self.goahead(0) |
92 |
|
93 |
# Interface -- handle the remaining data
|
94 |
def close(self): |
95 |
self.goahead(1) |
96 |
|
97 |
# Internal -- handle data as far as reasonable. May leave state
|
98 |
# and data to be processed by a subsequent call. If 'end' is
|
99 |
# true, force handling all data as if followed by EOF marker.
|
100 |
def goahead(self, end): |
101 |
rawdata = self.rawdata
|
102 |
i = 0
|
103 |
n = len(rawdata)
|
104 |
while i < n:
|
105 |
if self.nomoretags: |
106 |
self.handle_data(rawdata[i:n])
|
107 |
i = n |
108 |
break
|
109 |
match = interesting.search(rawdata, i) |
110 |
if match: j = match.start(0) |
111 |
else: j = n
|
112 |
if i < j: self.handle_data(rawdata[i:j]) |
113 |
i = j |
114 |
if i == n: break |
115 |
if rawdata[i] == '<': |
116 |
if starttagopen.match(rawdata, i):
|
117 |
if self.literal: |
118 |
self.handle_data(rawdata[i])
|
119 |
i = i+1
|
120 |
continue
|
121 |
k = self.parse_starttag(i)
|
122 |
if k < 0: break |
123 |
i = k |
124 |
continue
|
125 |
if endtagopen.match(rawdata, i):
|
126 |
k = self.parse_endtag(i)
|
127 |
if k < 0: break |
128 |
i = k |
129 |
self.literal = 0 |
130 |
continue
|
131 |
if commentopen.match(rawdata, i):
|
132 |
if self.literal: |
133 |
self.handle_data(rawdata[i])
|
134 |
i = i+1
|
135 |
continue
|
136 |
k = self.parse_comment(i)
|
137 |
if k < 0: break |
138 |
i = i+k |
139 |
continue
|
140 |
if piopen.match(rawdata, i):
|
141 |
if self.literal: |
142 |
self.handle_data(rawdata[i])
|
143 |
i = i+1
|
144 |
continue
|
145 |
k = self.parse_pi(i)
|
146 |
if k < 0: break |
147 |
i = i+k |
148 |
continue
|
149 |
match = special.match(rawdata, i) |
150 |
if match:
|
151 |
if self.literal: |
152 |
self.handle_data(rawdata[i])
|
153 |
i = i+1
|
154 |
continue
|
155 |
# This is some sort of declaration; in "HTML as
|
156 |
# deployed," this should only be the document type
|
157 |
# declaration ("<!DOCTYPE html...>").
|
158 |
k = self.parse_declaration(i)
|
159 |
if k < 0: break |
160 |
i = k |
161 |
continue
|
162 |
elif rawdata[i] == '&': |
163 |
match = charref.match(rawdata, i) |
164 |
if match:
|
165 |
name = match.group(1)
|
166 |
self.handle_charref(name)
|
167 |
i = match.end(0)
|
168 |
if rawdata[i-1] != ';': i = i-1 |
169 |
continue
|
170 |
match = entityref.match(rawdata, i) |
171 |
if match:
|
172 |
name = match.group(1)
|
173 |
self.handle_entityref(name)
|
174 |
i = match.end(0)
|
175 |
if rawdata[i-1] != ';': i = i-1 |
176 |
continue
|
177 |
else:
|
178 |
raise SGMLParseError('neither < nor & ??') |
179 |
# We get here only if incomplete matches but
|
180 |
# nothing else
|
181 |
match = incomplete.match(rawdata, i) |
182 |
if not match: |
183 |
self.handle_data(rawdata[i])
|
184 |
i = i+1
|
185 |
continue
|
186 |
j = match.end(0)
|
187 |
if j == n:
|
188 |
break # Really incomplete |
189 |
self.handle_data(rawdata[i:j])
|
190 |
i = j |
191 |
# end while
|
192 |
if end and i < n: |
193 |
self.handle_data(rawdata[i:n])
|
194 |
i = n |
195 |
self.rawdata = rawdata[i:]
|
196 |
# XXX if end: check for empty stack
|
197 |
|
198 |
# Internal -- parse comment, return length or -1 if not terminated
|
199 |
def parse_comment(self, i): |
200 |
rawdata = self.rawdata
|
201 |
if rawdata[i:i+4] != '<!--': |
202 |
raise SGMLParseError('unexpected call to parse_comment()') |
203 |
match = commentclose.search(rawdata, i+4)
|
204 |
if not match: |
205 |
return -1 |
206 |
j = match.start(0)
|
207 |
self.handle_comment(rawdata[i+4: j]) |
208 |
j = match.end(0)
|
209 |
return j-i
|
210 |
|
211 |
# Internal -- parse declaration.
|
212 |
def parse_declaration(self, i): |
213 |
rawdata = self.rawdata
|
214 |
j = i + 2
|
215 |
# in practice, this should look like: ((name|stringlit) S*)+ '>'
|
216 |
while 1: |
217 |
c = rawdata[j:j+1]
|
218 |
if c == ">": |
219 |
# end of declaration syntax
|
220 |
self.handle_decl(rawdata[i+2:j]) |
221 |
return j + 1 |
222 |
if c in "\"'": |
223 |
m = declstringlit.match(rawdata, j) |
224 |
if not m: |
225 |
# incomplete or an error?
|
226 |
return -1 |
227 |
j = m.end() |
228 |
elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ": |
229 |
m = declname.match(rawdata, j) |
230 |
if not m: |
231 |
# incomplete or an error?
|
232 |
return -1 |
233 |
j = m.end() |
234 |
elif i == len(rawdata): |
235 |
# end of buffer between tokens
|
236 |
return -1 |
237 |
else:
|
238 |
raise SGMLParseError(
|
239 |
"unexpected char in declaration: %s" % `rawdata[i]`)
|
240 |
assert 0, "can't get here!" |
241 |
|
242 |
# Internal -- parse processing instr, return length or -1 if not terminated
|
243 |
def parse_pi(self, i): |
244 |
rawdata = self.rawdata
|
245 |
if rawdata[i:i+2] != '<?': |
246 |
raise SGMLParseError('unexpected call to parse_pi()') |
247 |
match = piclose.search(rawdata, i+2)
|
248 |
if not match: |
249 |
return -1 |
250 |
j = match.start(0)
|
251 |
self.handle_pi(rawdata[i+2: j]) |
252 |
j = match.end(0)
|
253 |
return j-i
|
254 |
|
255 |
__starttag_text = None
|
256 |
def get_starttag_text(self): |
257 |
return self.__starttag_text |
258 |
|
259 |
# Internal -- handle starttag, return length or -1 if not terminated
|
260 |
def parse_starttag(self, i): |
261 |
self.__starttag_text = None |
262 |
start_pos = i |
263 |
rawdata = self.rawdata
|
264 |
if shorttagopen.match(rawdata, i):
|
265 |
# SGML shorthand: <tag/data/ == <tag>data</tag>
|
266 |
# XXX Can data contain &... (entity or char refs)?
|
267 |
# XXX Can data contain < or > (tag characters)?
|
268 |
# XXX Can there be whitespace before the first /?
|
269 |
match = shorttag.match(rawdata, i) |
270 |
if not match: |
271 |
return -1 |
272 |
tag, data = match.group(1, 2) |
273 |
self.__starttag_text = '<%s/' % tag |
274 |
tag = tag.lower() |
275 |
k = match.end(0)
|
276 |
self.finish_shorttag(tag, data)
|
277 |
self.__starttag_text = rawdata[start_pos:match.end(1) + 1] |
278 |
return k
|
279 |
# XXX The following should skip matching quotes (' or ")
|
280 |
match = endbracket.search(rawdata, i+1)
|
281 |
if not match: |
282 |
return -1 |
283 |
j = match.start(0)
|
284 |
# Now parse the data between i+1 and j into a tag and attrs
|
285 |
attrs = [] |
286 |
if rawdata[i:i+2] == '<>': |
287 |
# SGML shorthand: <> == <last open tag seen>
|
288 |
k = j |
289 |
tag = self.lasttag
|
290 |
else:
|
291 |
match = tagfind.match(rawdata, i+1)
|
292 |
if not match: |
293 |
raise SGMLParseError('unexpected call to parse_starttag') |
294 |
k = match.end(0)
|
295 |
tag = rawdata[i+1:k].lower()
|
296 |
self.lasttag = tag
|
297 |
while k < j:
|
298 |
match = attrfind.match(rawdata, k) |
299 |
if not match: break |
300 |
attrname, rest, attrvalue = match.group(1, 2, 3) |
301 |
if not rest: |
302 |
attrvalue = attrname |
303 |
elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ |
304 |
attrvalue[:1] == '"' == attrvalue[-1:]: |
305 |
attrvalue = attrvalue[1:-1] |
306 |
attrs.append((attrname.lower(), attrvalue)) |
307 |
k = match.end(0)
|
308 |
if rawdata[j] == '>': |
309 |
j = j+1
|
310 |
self.__starttag_text = rawdata[start_pos:j]
|
311 |
self.finish_starttag(tag, attrs)
|
312 |
return j
|
313 |
|
314 |
# Internal -- parse endtag
|
315 |
def parse_endtag(self, i): |
316 |
rawdata = self.rawdata
|
317 |
match = endbracket.search(rawdata, i+1)
|
318 |
if not match: |
319 |
return -1 |
320 |
j = match.start(0)
|
321 |
tag = rawdata[i+2:j].strip().lower()
|
322 |
if rawdata[j] == '>': |
323 |
j = j+1
|
324 |
self.finish_endtag(tag)
|
325 |
return j
|
326 |
|
327 |
# Internal -- finish parsing of <tag/data/ (same as <tag>data</tag>)
|
328 |
def finish_shorttag(self, tag, data): |
329 |
self.finish_starttag(tag, [])
|
330 |
self.handle_data(data)
|
331 |
self.finish_endtag(tag)
|
332 |
|
333 |
# Internal -- finish processing of start tag
|
334 |
# Return -1 for unknown tag, 0 for open-only tag, 1 for balanced tag
|
335 |
def finish_starttag(self, tag, attrs): |
336 |
try:
|
337 |
method = getattr(self, 'start_' + tag) |
338 |
except AttributeError: |
339 |
try:
|
340 |
method = getattr(self, 'do_' + tag) |
341 |
except AttributeError: |
342 |
self.unknown_starttag(tag, attrs)
|
343 |
return -1 |
344 |
else:
|
345 |
self.handle_starttag(tag, method, attrs)
|
346 |
return 0 |
347 |
else:
|
348 |
self.stack.append(tag)
|
349 |
self.handle_starttag(tag, method, attrs)
|
350 |
return 1 |
351 |
|
352 |
# Internal -- finish processing of end tag
|
353 |
def finish_endtag(self, tag): |
354 |
if not tag: |
355 |
found = len(self.stack) - 1 |
356 |
if found < 0: |
357 |
self.unknown_endtag(tag)
|
358 |
return
|
359 |
else:
|
360 |
if tag not in self.stack: |
361 |
try:
|
362 |
method = getattr(self, 'end_' + tag) |
363 |
except AttributeError: |
364 |
self.unknown_endtag(tag)
|
365 |
else:
|
366 |
self.report_unbalanced(tag)
|
367 |
return
|
368 |
found = len(self.stack) |
369 |
for i in range(found): |
370 |
if self.stack[i] == tag: found = i |
371 |
while len(self.stack) > found: |
372 |
tag = self.stack[-1] |
373 |
try:
|
374 |
method = getattr(self, 'end_' + tag) |
375 |
except AttributeError: |
376 |
method = None
|
377 |
if method:
|
378 |
self.handle_endtag(tag, method)
|
379 |
else:
|
380 |
self.unknown_endtag(tag)
|
381 |
del self.stack[-1] |
382 |
|
383 |
# Overridable -- handle start tag
|
384 |
def handle_starttag(self, tag, method, attrs): |
385 |
method(attrs) |
386 |
|
387 |
# Overridable -- handle end tag
|
388 |
def handle_endtag(self, tag, method): |
389 |
method() |
390 |
|
391 |
# Example -- report an unbalanced </...> tag.
|
392 |
def report_unbalanced(self, tag): |
393 |
if self.verbose: |
394 |
print '*** Unbalanced </' + tag + '>' |
395 |
print '*** Stack:', self.stack |
396 |
|
397 |
# Example -- handle character reference, no need to override
|
398 |
def handle_charref(self, name): |
399 |
try:
|
400 |
n = int(name)
|
401 |
except ValueError: |
402 |
self.unknown_charref(name)
|
403 |
return
|
404 |
if not 0 <= n <= 255: |
405 |
self.unknown_charref(name)
|
406 |
return
|
407 |
self.handle_data(chr(n)) |
408 |
|
409 |
# Definition of entities -- derived classes may override
|
410 |
entitydefs = \ |
411 |
{'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''} |
412 |
|
413 |
# Example -- handle entity reference, no need to override
|
414 |
def handle_entityref(self, name): |
415 |
table = self.entitydefs
|
416 |
if table.has_key(name):
|
417 |
self.handle_data(table[name])
|
418 |
else:
|
419 |
self.unknown_entityref(name)
|
420 |
return
|
421 |
|
422 |
# Example -- handle data, should be overridden
|
423 |
def handle_data(self, data): |
424 |
pass
|
425 |
|
426 |
# Example -- handle comment, could be overridden
|
427 |
def handle_comment(self, data): |
428 |
pass
|
429 |
|
430 |
# Example -- handle declaration, could be overridden
|
431 |
def handle_decl(self, decl): |
432 |
pass
|
433 |
|
434 |
# Example -- handle processing instruction, could be overridden
|
435 |
def handle_pi(self, data): |
436 |
pass
|
437 |
|
438 |
# To be overridden -- handlers for unknown objects
|
439 |
def unknown_starttag(self, tag, attrs): pass |
440 |
def unknown_endtag(self, tag): pass |
441 |
def unknown_charref(self, ref): pass |
442 |
def unknown_entityref(self, ref): pass |
443 |
|
444 |
|
445 |
class TestSGMLParser(SGMLParser): |
446 |
|
447 |
def __init__(self, verbose=0): |
448 |
self.testdata = "" |
449 |
SGMLParser.__init__(self, verbose)
|
450 |
|
451 |
def handle_data(self, data): |
452 |
self.testdata = self.testdata + data |
453 |
if len(`self.testdata`) >= 70: |
454 |
self.flush()
|
455 |
|
456 |
def flush(self): |
457 |
data = self.testdata
|
458 |
if data:
|
459 |
self.testdata = "" |
460 |
print 'data:', `data` |
461 |
|
462 |
def handle_comment(self, data): |
463 |
self.flush()
|
464 |
r = `data` |
465 |
if len(r) > 68: |
466 |
r = r[:32] + '...' + r[-32:] |
467 |
print 'comment:', r |
468 |
|
469 |
def unknown_starttag(self, tag, attrs): |
470 |
self.flush()
|
471 |
if not attrs: |
472 |
print 'start tag: <' + tag + '>' |
473 |
else:
|
474 |
print 'start tag: <' + tag, |
475 |
for name, value in attrs: |
476 |
print name + '=' + '"' + value + '"', |
477 |
print '>' |
478 |
|
479 |
def unknown_endtag(self, tag): |
480 |
self.flush()
|
481 |
print 'end tag: </' + tag + '>' |
482 |
|
483 |
def unknown_entityref(self, ref): |
484 |
self.flush()
|
485 |
print '*** unknown entity ref: &' + ref + ';' |
486 |
|
487 |
def unknown_charref(self, ref): |
488 |
self.flush()
|
489 |
print '*** unknown char ref: &#' + ref + ';' |
490 |
|
491 |
def close(self): |
492 |
SGMLParser.close(self)
|
493 |
self.flush()
|
494 |
|
495 |
|
496 |
def test(args = None): |
497 |
import sys |
498 |
|
499 |
if not args: |
500 |
args = sys.argv[1:]
|
501 |
|
502 |
if args and args[0] == '-s': |
503 |
args = args[1:]
|
504 |
klass = SGMLParser |
505 |
else:
|
506 |
klass = TestSGMLParser |
507 |
|
508 |
if args:
|
509 |
file = args[0]
|
510 |
else:
|
511 |
file = 'test.html'
|
512 |
|
513 |
if file == '-': |
514 |
f = sys.stdin |
515 |
else:
|
516 |
try:
|
517 |
f = open(file, 'r') |
518 |
except IOError, msg: |
519 |
print file, ":", msg |
520 |
sys.exit(1)
|
521 |
|
522 |
data = f.read() |
523 |
if f is not sys.stdin: |
524 |
f.close() |
525 |
|
526 |
x = klass() |
527 |
for c in data: |
528 |
x.feed(c) |
529 |
x.close() |
530 |
|
531 |
|
532 |
if __name__ == '__main__': |
533 |
test() |