BXQc@sdZddlZddlmZddlZddlZddlZddlZddlZddl Z ddl m Z m Z ddlmZddlmZmZddlmZddlmZddlZddlZejd jZejd jZd Zd Zd efdYZdd$dYZ de ejfdYZ!de e j"fdYZ#dZ$de fdYZ%dej&fdYZ'de fdYZ(de fdYZ)de fdYZ*d Z+d!Z,d"e fd#YZ-dS(%sUHTTP related handlers. Note that some other HTTP handlers live in more specific modules: _auth.py, _gzip.py, etc. Copyright 2002-2006 John J Lee This code is free software; you can redistribute it and/or modify it under the terms of the BSD or ZPL 2.1 licenses (see the file COPYING.txt included with the distribution). iN(tStringIO(t HTTPErrort BaseHandler(tis_html(tunescapetunescape_charref(tRequest(tresponse_seek_wrappert mechanizesmechanize.robotsislatin-1tEndOfHeadErrorcBseZRS((t__name__t __module__(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyR 0stAbstractHeadParserc BsneZdZejZeZd Zd Z d Z d Z d Z dZ dZdZdZRS(thtmltheadttitletbasetscripttstyletmetatlinktobjectcCs g|_dS(N(t http_equiv(tself((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt__init__9scCsd}}xP|D]H\}}|dkr;|j|}q|dkr|j|}qqW|dk r|dk r|jj||fndS(Ns http-equivtcontent(tNonetunescape_attr_if_requiredRtappend(RtattrsRRtkeytvalue((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt start_meta<s   cCs tdS(N(R (R((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pytend_headFscCs'|jtd||j|jdS(Ns&%s;(t handle_dataRt _entitydefst _encoding(Rtname((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pythandle_entityrefIs cCs|jt||jdS(N(R"RR$(RR%((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pythandle_charrefNscCst||j|jS(N(RR#R$(RR%((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt unescape_attrRscCs:i}x-|jD]\}}|j|||RR(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyR4ds    t HeadParsercBs>eZdZdZdZdZdZdZRS(cCsts tdS(N(tFalsetAssertionError(R((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt _not_calledscCs!tjj|tj|dS(N(tsgmllibt SGMLParserRR (R((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRscCs8||jkrtn|dkr4||ndS(NR(R0R (RR:R;R((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyR<s  cCs|j||j|dS(N(R<RB(RR:R((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pytunknown_starttagscCs&||jkr|n tdS(N(R0R (RR:R;((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyR>s cCs |j|S(N(R((RR%((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRs(R R RBRR<RER>R(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyR?s      cCsXxN|jt}y|j|Wntk r7PnXt|tkrPqq|jS(s"Return a list of key, value pairs.(treadtCHUNKtfeedR tlenR(tfileobjtparsertdata((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt parse_heads tHTTPEquivProcessorcBs2eZdZdZeedZdZeZRS(s7Append META HTTP-EQUIV headers to regular HTTP headers.i,cCs||_||_dS(N(thead_parser_classt _allow_xhtml(RROti_want_broken_xhtml_support((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRs c Cst|dst|}n|j}|j}|jd}t|||jry.zt||j}Wd|j dXWnt j t j fk rqXxf|D][\}}||j|j<|d|} x+| jdD]} |jj| dqWqWn|S(Ntseeks content-typeis: s (thasattrRtinfotgeturlt getheadersRRPRMRORRR5tHTMLParseErrorRCtSGMLParseErrortdicttlowertsplittheadersR( Rtrequesttresponset http_messageturltct_hdrst html_headersthdrR+ttexttline((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt http_responses(  ( R R t__doc__t handler_orderR?R@RRfthttps_response(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRNs   tMechanizeRobotFileParsercBs5eZdddZddZdZdZRS(tcCs,tjj||||_tj|_dS(N(t robotparsertRobotFileParserRt_openert_sockettimeoutt_GLOBAL_DEFAULT_TIMEOUTt_timeout(RR`topener((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRs cCs4ddl}|dkr'|j}n||_dS(Ni(RnRtOpenerDirector(RRrRn((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt set_openers  cCs ||_dS(N(Rq(Rttimeout((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt set_timeoutscCsi|jdkr|jnt|jdtdtd|j}y|jj|}WnIt k rk}n7t t j t fk r}td|j|fdSXg}|j}x)|r|j|j|j}qW|j}|dks|dkrt|_tdnN|d kr9t|_td n,|d kre|retd |j|ndS( s4Reads the robots.txt URL and feeds it to the parser.t unverifiabletvisitRusignoring error opening %r: %sNiis disallow allis allow allis parse lines(RnRRtRR`tTrueR@RqtopenRtIOErrortsocketterrortOSErrort debug_robotstreadlineRtstriptcodet disallow_allt allow_alltparse(RtreqtftexctlinesRetstatus((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRFs6           N(R R RRRtRvRF(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRjs  tRobotExclusionErrorcBseZdZRS(cGs$ttj|f|||_dS(N(tapplyRRR](RR]targs((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyR s(R R R(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyR stHTTPRobotRulesProcessorcBsceZdZyddlmZWnddlmZeZnXeZedZ dZ e Z RS(i i(t HTTPMessage(tMessagecCs||_d|_d|_dS(N(t rfp_classRtrfpt_host(RR((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRs  c Cs|j}|d kr|S|jdkr2|S|j}t|dd}|dk r|jdkr|j|kr|S||jkr)|j|_y|jj|j Wn%t k rt d|jj nX|jj |d|d|jj|j|jj||_n|jdd}|jj||jrZ|Sd }t||jd ||jtt|dS( Nthttpthttpss /robots.txtt _origin_reqs'%r instance does not support set_openers://s User-agentRks request disallowed by robots.txti(shttpshttps(tget_typet get_selectortget_hostR8RRRRRttparentR9tdebugt __class__tset_urlRvRuRFt get_headert can_fetcht get_full_urlRthttp_response_classR(RR]tschemethostt origin_reqtuatmsg((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt http_requests>        ( R R RhthttplibRt mimetoolsRRRjRRt https_request(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRs   ,tHTTPRefererProcessorcBs5eZdZdZdZdZeZeZRS(sYAdd Referer header to requests. This only makes sense if you use each RefererProcessor for a single chain of requests only (so, for example, if you use a single HTTPRefererProcessor to fetch a series of URLs extracted from a single page, this will break). There's a proper implementation of this in mechanize.Browser. cCs d|_dS(N(Rtreferer(R((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRXscCs9|jdk r5|jd r5|jd|jn|S(NtReferer(RRt has_headertadd_unredirected_header(RR]((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyR[scCs|j|_|S(N(RUR(RR]R^((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRfas(R R RgRRRfRRi(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRMs     cCs\|jdr|jds<|jdrL|jdrL|dd!}ntj|dS(Nt"t'iislatin-1(t startswithtendswitht_rfc3986t clean_url(R`((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pytclean_refresh_urliscCs|jd}|dkrt|| ||d}}|jd}d}|dkr|| ||d}}t|}n|dks|jjdkrtqnt|d}}||fS(s >>> parse_refresh_header("1; url=http://example.com/") (1.0, 'http://example.com/') >>> parse_refresh_header("1; url='http://example.com/'") (1.0, 'http://example.com/') >>> parse_refresh_header("1") (1.0, None) >>> parse_refresh_header("blah") # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ValueError: invalid literal for float(): blah t;iit=R`N(tfindtfloatRRRRZt ValueError(trefreshtiitpauset newurl_spectjjRtnewurl((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pytparse_refresh_headerps  $tHTTPRefreshProcessorcBs2eZdZdZdedZdZeZRS(s>Perform HTTP Refresh redirections. Note that if a non-200 HTTP code has occurred (for example, a 30x redirect), this processor will do nothing. By default, only zero-time Refresh headers are redirected. Use the max_time attribute / constructor argument to allow Refresh with longer pauses. Use the honor_time attribute / constructor argument to control whether the requested pause is honoured (with a time.sleep()) or skipped in favour of immediate redirection. Public attributes: max_time: see above honor_time: see above iicCs"||_||_tj|_dS(N(tmax_timet honor_timettimetsleept_sleep(RRR((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRs  c Cs(|j|j|j}}}|dkr$|jdr$|jdd}yt|\}}Wn tk rtd||SX|dkr|j }n|j dks||j kr|dkr|j r|j |n||d<|j jd||d||}q$td|n|S( NiRisbad Refresh header: %rgMbP?tlocationRsRefresh header ignored: %r(RRRTthas_keyRVRRRRRURRRRR}( RR]R^RRthdrsRRR((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRfs&      (R R RgRhRyRRfRi(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyRs  ((.RgR5t cStringIORR1tloggingRlR|Rt _sgmllib_copyRCt _urllib2_forkRRt _headersutilRt_htmlRRt_requestRt _responseRRRot getLoggerRRRGR3t ExceptionR R R4RDR?RMRNRmRjRRRRRR(((s4/scratch/rashmi/Condor_Script/src/mechanize/_http.pyt sB          3& '.?