a s h9?@sdZddlmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlmZmZmZddlmZddlmZddlmZddlmZmZmZdd lmZdd lmZdd l m!Z!m"Z"dd l#m$Z$dd l%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6e 7e8Z9ee:e:fZ;dddddZdddd d!Z?Gd"d#d#e=Z@dd$dd%d&d'ZAdd$dd%d(d)ZBd*dd+d,d-ZCGd.d/d/ZDGd0d1d1eZEd1d1d2d3d4ZFeFd5d6d7d8d9ZGed:d;Gdd>eZIdQd?d@dAddBdCdDZJdRddEd5dFdGdHZKd?d$dIdJdKdLZLGdMdNdNeZMGdOdPdPZNdS)SzO The main purpose of this module is to expose LinkCollector.collect_sources(). ) annotationsN)IterableMutableMappingSequence) dataclass) HTMLParser)Values)Callable NamedTupleProtocol)requests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_fileredact_auth_from_url)vcs)CandidatesFromPage LinkSource build_sourcestr str | NoneurlreturncCs6tjD]*}||r|t|dvr|SqdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)r schemer'Y/var/www/html/assistant/venv/lib/python3.9/site-packages/pip/_internal/index/collector.py_match_vcs_scheme.s  r)cs&eZdZddddfdd ZZS)_NotAPIContentrNone) content_type request_descr!cst||||_||_dSN)super__init__r,r-)selfr,r- __class__r'r(r0:sz_NotAPIContent.__init__)__name__ __module__ __qualname__r0 __classcell__r'r'r2r(r*9sr*r r+)responser!cCs6|jdd}|}|dr$dSt||jjdS)z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr#r$r*requestmethod)r8r,content_type_lr'r'r(_ensure_api_header@srAc@s eZdZdS)_NotHTTPN)r4r5r6r'r'r'r(rBVsrBr)r sessionr!cCsFtj|\}}}}}|dvr$t|j|dd}t|t|dS)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httpshttpT)allow_redirectsN)urllibparseurlsplitrBheadrrA)r rCr&netlocpathqueryfragmentrespr'r'r(_ensure_api_responseZs rPcCsxtt|jrt||dtdt||j|dgdddd}t |t |tdt||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. rCzGetting page %sz, )r;z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r<zFetched page %s as %sr9r:) rrfilenamerPloggerdebugrr=joinrrAr<)r rCrOr'r'r(_get_simple_responsels&   rWResponseHeaders)r<r!cCs<|r8d|vr8tj}|d|d<|d}|r8t|SdS)z=Determine if we have any encoding information in our headers.r9z content-typecharsetN)emailmessageMessage get_paramr)r<mrYr'r'r(_get_encoding_from_headerss    r_c@s:eZdZdddddZdddd d Zd d d dZdS)CacheablePageContent IndexContentr+pager!cCs|js J||_dSr.)cache_link_parsingrcr1rcr'r'r(r0s zCacheablePageContent.__init__objectbool)otherr!cCst|t|o|jj|jjkSr.) isinstancetypercr )r1rhr'r'r(__eq__szCacheablePageContent.__eq__intr!cCs t|jjSr.)hashrcr r1r'r'r(__hash__szCacheablePageContent.__hash__N)r4r5r6r0rkrpr'r'r'r(r`sr`c@seZdZdddddZdS) ParseLinksraIterable[Link]rbcCsdSr.r'rer'r'r(__call__zParseLinks.__call__N)r4r5r6rsr'r'r'r(rqsrq)fnr!cs>tjdddfdd tdddfdd }|S) z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. r`z list[Link])cacheable_pager!cst|jSr.)listrc)rv)rur'r(wrappersz*with_cached_index_content..wrapperrarbcs|jrt|St|Sr.)rdr`rw)rcrurxr'r(wrapper_wrappers z2with_cached_index_content..wrapper_wrapper) functoolscachewraps)rurzr'ryr(with_cached_index_contents r~rarrrbc cs|j}|drTt|j}|dgD]"}t||j }|durHq,|Vq,dSt |j }|j pfd}| |j ||j }|jp|}|jD]$} tj| ||d}|durq|VqdS)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r;filesNzutf-8)page_urlbase_url)r,r#r$jsonloadscontentr=r from_jsonr HTMLLinkParserencodingfeeddecoderanchorsZ from_element) rcr@datafilelinkparserrr ranchorr'r'r( parse_linkss&       rT)frozenc@sLeZdZUdZded<ded<ded<ded<d Zd ed <dd d dZdS)raaRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. bytesrrr,rrr TrgrdrmcCs t|jSr.)rr ror'r'r(__str__szIndexContent.__str__N)r4r5r6__doc____annotations__rdrr'r'r'r(ras   csJeZdZdZdddfdd Zddddd d Zdd d d dZZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. rr+rcs$tjdd||_d|_g|_dS)NT)convert_charrefs)r/r0r rr)r1r r2r'r(r0szHTMLLinkParser.__init__zlist[tuple[str, str | None]])tagattrsr!cCsH|dkr,|jdur,||}|durD||_n|dkrD|jt|dS)Nbasea)rget_hrefrappenddict)r1rrhrefr'r'r(handle_starttags  zHTMLLinkParser.handle_starttagr)rr!cCs"|D]\}}|dkr|SqdS)Nrr')r1rnamevaluer'r'r(r!s  zHTMLLinkParser.get_href)r4r5r6rr0rrr7r'r'r2r(r srrzstr | ExceptionzCallable[..., None] | None)rreasonmethr!cCs|durtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rTrU)rrrr'r'r(_handle_get_simple_fail(srrg)r8rdr!cCs&t|j}t|j|jd||j|dS)Nr9)rr rd)r_r<rarr )r8rdrr'r'r(_make_index_content2s rIndexContent | None)rrCr!c Cs|jddd}t|}|r0td||dStj|\}}}}}}|dkrtj tj |r| dsv|d7}tj|d}td|zt||d }WnLtytd |Yn<ty}z"td ||j|jWYd}~n d}~0ty(}zt||WYd}~nd}~0tyV}zt||WYd}~nd}~0ty}z,d } | t|7} t|| tjd WYd}~nld}~0tjy}zt|d|WYd}~n6d}~0tjyt|dYn0t||jdSdS)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.r/z index.htmlz# file: URL is directory, getting %srQz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)rd) r splitr)rTwarningrGrHurlparseosrLisdirr> url2pathnameendswithurljoinrUrWrBr*r-r,rrrrrinfor ConnectionErrorTimeoutrrd) rrCr vcs_schemer&_rLrOexcrr'r'r(_get_index_content?sV     $$rc@seZdZUded<ded<dS)CollectedSourceszSequence[LinkSource | None] find_links index_urlsN)r4r5r6rr'r'r'r(r|s rc@speZdZdZddddddZeddd d dd d d ZeddddZdddddZ ddddddZ dS) LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. rrr+)rC search_scoper!cCs||_||_dSr.)rrC)r1rCrr'r'r(r0szLinkCollector.__init__Frrg)rCoptionssuppress_no_indexr!cCsd|jg|j}|jr8|s8tdddd|Dg}|jp@g}tj|||jd}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdSr.r).0r r'r'r( rtz'LinkCollector.create..)rrno_index)rCr) index_urlextra_index_urlsrrTrUrVrrcreater)clsrCrrrrrlink_collectorr'r'r(rs$   zLinkCollector.createz list[str]rmcCs|jjSr.)rrror'r'r(rszLinkCollector.find_linksrr)locationr!cCst||jdS)z> Fetch an HTML page containing package links. rQ)rrC)r1rr'r'r(fetch_responseszLinkCollector.fetch_responserrr) project_namecandidates_from_pager!cstfddjD}tfddjD}ttj rddt ||D}t |ddg|}t d|tt|t|d S) Nc 3s&|]}t|jjdddVqdS)FrZpage_validator expand_dirrdrNrrCis_secure_originrlocrrr1r'r(rs z0LinkCollector.collect_sources..c 3s&|]}t|jjdddVqdS)TrNrrrr'r(rs cSs*g|]"}|dur|jdurd|jqS)Nz* )r)rsr'r'r( sz1LinkCollector.collect_sources..z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrrT isEnabledForloggingDEBUG itertoolschainr%rUrVrrw)r1rrZindex_url_sourcesZfind_links_sourceslinesr'rr(collect_sourcess*     zLinkCollector.collect_sourcesN)F) r4r5r6rr0 classmethodrpropertyrrrr'r'r'r(rs!r)N)T)Or __future__rr email.messagerZr{rrrr urllib.parserGurllib.requestcollections.abcrrrZ dataclassesr html.parserroptparsertypingr r r pip._vendorr Zpip._vendor.requestsr Zpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.vcsrsourcesrrr getLoggerr4rTrrXr) Exceptionr*rArBrPrWr_r`rqr~rrarrrrrrr'r'r'r(sf                 ?    =