ó
BÜXQc @ sü d Z d d l Z d d l Z d d l Z d d l Z d d l m Z d d l m Z m
Z
d e f d „ ƒ YZ d d d „ ƒ YZ
d „ Z d
d d „ ƒ YZ d e e j f d
„ ƒ YZ d e e j f d „ ƒ YZ d „ Z e d k rø e ƒ n d S( sà A simple "pull API" for HTML parsing, after Perl's HTML::TokeParser.
Examples
This program extracts all links from a document. It will print one
line for each link, containing the URL and the textual description
between the ... tags:
import pullparser, sys
f = file(sys.argv[1])
p = pullparser.PullParser(f)
for token in p.tags("a"):
if token.type == "endtag": continue
url = dict(token.attrs).get("href", "-")
text = p.get_compressed_text(endat=("endtag", "a"))
print "%s %s" % (url, text)
This program extracts the from the document:
import pullparser, sys
f = file(sys.argv[1])
p = pullparser.PullParser(f)
if p.get_tag("title"):
title = p.get_compressed_text()
print "Title: %s" % title
Copyright 2003-2006 John J. Lee <jjl@pobox.com>
Copyright 1998-2001 Gisle Aas (original libwww-perl code)
This code is free software; you can redistribute it and/or modify it
under the terms of the BSD or ZPL 2.1 licenses.
iÿÿÿÿN( t saxutils( t unescapet unescape_charreft NoMoreTokensErrorc B s e Z RS( ( t __name__t
__module__( ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR , s t Tokenc B sG e Z d Z d d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z RS( sk Represents an HTML tag, declaration, processing instruction etc.
Behaves as both a tuple-like object (ie. iterable) and has attributes
.type, .data and .attrs.
>>> t = Token("starttag", "a", [("href", "http://www.python.org/")])
>>> t == ("starttag", "a", [("href", "http://www.python.org/")])
True
>>> (t.type, t.data) == ("starttag", "a")
True
>>> t.attrs == [("href", "http://www.python.org/")]
True
Public attributes
type: one of "starttag", "endtag", "startendtag", "charref", "entityref",
"data", "comment", "decl", "pi", after the corresponding methods of
HTMLParser.HTMLParser
data: For a tag, the tag name; otherwise, the relevant data carried by the
tag, as a string
attrs: list of (name, value) pairs representing HTML attributes
(or None if token does not represent an opening tag)
c C s | | _ | | _ | | _ d S( N( t typet datat attrs( t selfR R R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt __init__G s c C s t | j | j | j f ƒ S( N( t iterR R R ( R
( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt __iter__K s c C sH | \ } } } | j | k r@ | j | k r@ | j | k r@ t St Sd S( N( R R R t Truet False( R
t otherR R R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt __eq__M s c C s | j | ƒ S( N( R ( R
R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt __ne__U s c C s< d j t t | j | j | j g ƒ ƒ } | j j d | S( Ns , s (%s)( t joint mapt reprR R R t __class__R ( R
t args( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt __repr__V s *c C sX | j d k rP d j g | j D]% \ } } d | t j | ƒ f ^ q ƒ } n d } | j d k rv d | j | f S| j d k r– d | j | f S| j d k r° d | j S| j d k rÊ d
| j S| j d k rä d | j S| j d
k rú | j S| j d k rd | j S| j d k r.d | j S| j d k rHd | j St sTt ‚ d S( sH
>>> print Token("starttag", "br")
<br>
>>> print Token("starttag", "a",
... [("href", "http://www.python.org/"), ("alt", '"foo"')])
<a href="http://www.python.org/" alt='"foo"'>
>>> print Token("startendtag", "br")
<br />
>>> print Token("startendtag", "br", [("spam", "eggs")])
<br spam="eggs" />
>>> print Token("endtag", "p")
</p>
>>> print Token("charref", "38")
&
>>> print Token("entityref", "amp")
&
>>> print Token("data", "foo\nbar")
foo
bar
>>> print Token("comment", "Life is a bowl\nof cherries.")
<!--Life is a bowl
of cherries.-->
>>> print Token("decl", "decl")
<!decl>
>>> print Token("pi", "pi")
<?pi>
t s %s=%st starttags <%s%s>t startendtags <%s%s />t endtags </%s>t charrefs &#%s;t entityrefs &%s;R t comments <!--%s-->t decls <!%s>t pis <?%s>N( R t NoneR R t quoteattrR R R t AssertionError( R
t kt vR ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt __str__Z s. 8 N(
R R t __doc__R" R R
R R R R' ( ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR . s c o s6 x/ y | | | Ž VWq | k
r. t ‚ q Xq d S( N( t
StopIteration( t fnt exceptionR t kwds( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt iter_until_exception s
t _AbstractParserc B sý e Z d Z e j d ƒ Z i d d 6d d 6d d d „ Z d „ Z d „ Z d „ Z
d
„ Z d „ Z d „ Z
d
„ Z d d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z RS( i s \s+t altt imgt applett asciic C sI | | _ g | _ | | _ | | _ | d k r< t j } n | | _ d S( s‰
fh: file-like object (only a .read() method is required) from which to
read HTML to be parsed
textify: mapping used by .get_text() and .get_compressed_text() methods
to represent opening tags as text
encoding: encoding used to encode numeric character references by
.get_text() and .get_compressed_text() ("ascii" by default)
entitydefs: mapping like {"amp": "&", ...} containing HTML entity
definitions (a sensible default is used). This is used to unescape
entities in .get_text() (and .get_compressed_text()) and attribute
values. If the encoding can not represent the character, the entity
reference is left unescaped. Note that entity references (both
numeric - e.g. { or ઼ - and non-numeric - e.g. &) are
unescaped in attribute values and the return value of .get_text(), but
not in data outside of tags. Instead, entity references outside of
tags are represented as tokens. This is a bit odd, it's true :-/
If the element name of an opening tag matches a key in the textify
mapping then that tag is converted to text. The corresponding value is
used to specify which tag attribute to obtain the text from. textify
maps from element names to either:
- an HTML attribute name, in which case the HTML attribute value is
used as its text value along with the element name in square
brackets (e.g. "alt text goes here[IMG]", or, if the alt attribute
were missing, just "[IMG]")
- a callable object (e.g. a function) which takes a Token and returns
the string to be used as its text value
If textify has no key for an element name, nothing is substituted for
the opening tag.
Public attributes:
encoding and textify: see above
N( t _fht _tokenstackt textifyt encodingR" t htmlentitydefst name2codepointt _entitydefs( R
t fhR5 R6 t
entitydefs( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR › s ( c C s | S( N( ( R
( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR
Ë s c G s t | j t | Œ S( N( R- t get_tagR ( R
t names( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt tagsÍ s c G s t | j t | Œ S( N( R- t get_tokenR ( R
t
tokentypes( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt tokensÐ s c C s/ y | j ƒ SWn t k
r* t ƒ ‚ n Xd S( N( R? R R) ( R
( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt nextÔ s
c G s€ xy x? | j rD | j j d ƒ } | r= | j | k rA | Sq | Sq W| j j | j ƒ } | sl t ƒ ‚ n | j | ƒ q d S( s< Pop the next Token object from the stack of parsed tokens.
If arguments are given, they are taken to be token types in which the
caller is interested: tokens representing other elements will be
skipped. Element names must be given in lower case.
Raises NoMoreTokensError.
i N( R4 t popR R3 t readt chunkR t feed( R
R@ t tokenR ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR? Ú s
c C s | j j d | ƒ d S( s! Push a Token back onto the stack.i N( R4 t insert( R
RG ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt unget_tokenñ s c G sK xD | j ƒ } | j d k r$ q n | r@ | j | k rD | Sq | Sq d S( sA Return the next Token that represents an opening or closing tag.
If arguments are given, they are taken to be element names in which the
caller is interested: tags representing other elements will be skipped.
Element names must be given in lower case.
Raises NoMoreTokensError.
R R R N( s starttags endtags startendtag( R? R R ( R
R= t tok( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR< õ s
c C sï g } d
} xÓy | j ƒ } Wn( t k
rI | rE | j | ƒ n Pn X| j d k rl | j | j ƒ q | j d k rª t d | j | j | j ƒ } | j | ƒ q | j d k rÞ t
| j | j ƒ } | j | ƒ q | j d k r | j } | j d k rª| j j | ƒ } | d
k rªt
| ƒ rE| j | | ƒ ƒ q§| j d
k r§x3 | j D]( \ } } | | k r^| j | ƒ q^q^W| j d | j ƒ ƒ q§qªn | d
k sË| | j | f k rß| j | ƒ Pqßq q d j | ƒ S(
s¯ Get some text.
endat: stop reading text at this tag (the tag is included in the
returned text); endtag is a tuple (type, name) where type is
"starttag", "endtag" or "startendtag", and name is the element name of
the tag (element names must be given in lower case)
If endat is not given, .get_text() will stop at the next opening or
closing tag, or when there are no more tokens (no exception is raised).
Note that .get_text() includes the text representation (if any) of the
opening tag, but pushes the opening tag back onto the stack. As a
result, if you want to call .get_text() again, you need to call
.get_tag() first (unless you want an empty string returned when you
next call .get_text()).
Entity references are translated using the value of the entitydefs
constructor argument (a mapping from names to characters like that
provided by the standard module htmlentitydefs). Named entity
references that are not in this mapping are left unchanged.
The textify attribute is used to translate opening tags into text: see
the class docstring.
R R s &%s;R R R R s [%s]R N( s starttags endtags startendtag( s starttags startendtag( R" R? R RI R t appendR R R9 R6 R R5 t gett callableR t upperR ( R
t endatt textRJ t tt tag_nameR/ R% R& ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt get_text sB
!
c O s1 | j | | Ž } | j ƒ } | j j d | ƒ S( s²
As .get_text(), but collapses each group of contiguous whitespace to a
single space character, and removes all initial and trailing
whitespace.
t ( RS t stript compress_ret sub( R
R R, RP ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt get_compressed_textD s c C s | j j t d | | ƒ ƒ d S( NR ( R4 RK R ( R
t tagR ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt handle_startendtagO s c C s | j j t d | | ƒ ƒ d S( NR ( R4 RK R ( R
RY R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt handle_starttagQ s c C s | j j t d | ƒ ƒ d S( NR ( R4 RK R ( R
RY ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt
handle_endtagS s c C s | j j t d | ƒ ƒ d S( NR ( R4 RK R ( R
t name( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt handle_charrefU s c C s | j j t d | ƒ ƒ d S( NR ( R4 RK R ( R
R] ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt handle_entityrefW s c C s | j j t d | ƒ ƒ d S( NR ( R4 RK R ( R
R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt handle_dataY s c C s | j j t d | ƒ ƒ d S( NR ( R4 RK R ( R
R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt handle_comment[ s c C s | j j t d | ƒ ƒ d S( NR ( R4 RK R ( R
R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt handle_decl] s c C s | j j t d | ƒ ƒ d S( NR ( R4 RK R ( R
R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt unknown_decl_ s c C s | j j t d | ƒ ƒ d S( NR! ( R4 RK R ( R
R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt handle_pic s c C s t | | j | j ƒ S( N( R R9 R6 ( R
R] ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt
unescape_attrf s c C s= g } x0 | D]( \ } } | j | | j | ƒ f ƒ q
W| S( N( RK Re ( R
R t
escaped_attrst keyt val( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt unescape_attrsh s N( R R RE t ret compileRV R" R R
R> RA RB R? RI R< RS RX RZ R[ R\ R^ R_ R` Ra Rb Rc Rd Re Ri ( ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR. ˜ s2 / ; t
PullParserc B s e Z d „ Z d „ Z RS( c O s' t j j | ƒ t j | | | Ž d S( N( t
HTMLParserR R. ( R
R R, ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR o s c C s
| j | ƒ S( N( Re ( R
R] ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR r s ( R R R R ( ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyRl n s t TolerantPullParserc B s# e Z d „ Z d „ Z d „ Z RS( c O s' t j j | ƒ t j | | | Ž d S( N( t sgmllibt
SGMLParserR R. ( R
R R, ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyR x s c C s/ | j | ƒ } | j j t d | | ƒ ƒ d S( NR ( Ri R4 RK R ( R
RY R ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt unknown_starttag{ s c C s | j j t d | ƒ ƒ d S( NR ( R4 RK R ( R
RY ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt unknown_endtag~ s ( R R R Rq Rr ( ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyRn w s c C s% d d l } d d l } | j | ƒ S( Niÿÿÿÿ( t doctestt _pullparsert testmod( Rs Rt ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt _test‚ s t __main__( ( ( R( Rj R7 t
_sgmllib_copyRo Rm t xml.saxR t _htmlR R t ExceptionR R R- R. Rl Rp Rn Rv R ( ( ( s: /scratch/rashmi/Condor_Script/src/mechanize/_pullparser.pyt <module>" s b Ö