wgrddlZddlZddlZddlZddlZddlmZddlmZddlZddl Z ddl m Z m Z m Z mZddlmZddlmZddlmZddlmZmZmZmZmZd d lmZej:d Zej:d Zej@d Z!dZ"Gdde Z#GddeZ$ej:dZ%dZ&GddeZ'Gdde Z(ddZ)ddZ*ddZ+ee+Z,y)N)copy)urlparse)AbstractAsyncStreamedFileAsyncFileSystemsync sync_wrapper)DEFAULT_CALLBACK)FSTimeoutError)AbstractBufferedFile)DEFAULT_BLOCK_SIZEglob_translate isfilelike nullcontexttokenize)AllBytesz6<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P[^"']+)z-(?Phttp[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)z fsspec.httpc4Ktjdi|Sw)N)aiohttp ClientSession)kwargss `/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/fsspec/implementations/http.py get_clientr s  *6 **sc eZdZdZdZdddddddddedf fd ZedZd Z e d Z d Z e d Ze fd ZddZddZeeZdZd dZdefdZdeddfdZdZdZ d!dZd"dZdZdZd#dZdZ xZ!S)$HTTPFileSystema2 Simple File-System for fetching data via HTTP(S) ``ls()`` is implemented by loading the parent page and doing a regex match on the result. If simple_link=True, anything of the form "http(s)://server.com/stuff?thing=other"; otherwise only links within HTML href tags will be used. /TNbytesFc  t||f||d| ||nt|_||_||_||_||_| xsi|_| |_ | |_ | |_ d|_ t| } | jdd|_| jdd| jdd| jdd| |_ y)a NB: if this is called async, you must await set_client Parameters ---------- block_size: int Blocks to read bytes; if 0, will default to raw requests file-like objects instead of HTTPFile instances simple_links: bool If True, will consider both HTML tags and anything that looks like a URL; if False, will consider only the former. same_scheme: True When doing ls/glob, if this is True, only consider paths that have http/https matching the input URLs. size_policy: this argument is deprecated client_kwargs: dict Passed to aiohttp.ClientSession, see https://docs.aiohttp.org/en/stable/client_reference.html For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}`` get_client: Callable[..., aiohttp.ClientSession] A callable which takes keyword arguments and constructs an aiohttp.ClientSession. It's state will be managed by the HTTPFileSystem class. storage_options: key-value Any other parameters passed on to requests cache_type, cache_options: defaults used in open ) asynchronousloopNuse_listings_cacheFlistings_expiry_time max_pathsskip_instance_cache)super__init__r block_size simple_links same_schema cache_type cache_options client_kwargsrencodedr_sessionrpopr!)selfr(r' same_scheme size_policyr*r+rr r,rr-storage_optionsrequest_options __class__s rr&zHTTPFileSystem.__init__0sT WLtWW(2(>*DV(&$**0b$ %  /"1"5"56JE"R2D9K.148% cy)Nhttprr0s rfsidzHTTPFileSystem.fsidqsr6cDtj||jS)N)r-)yarlURLr-r0urls r encode_urlzHTTPFileSystem.encode_urlusxxT\\22r6c|*|jr t||jdyt |dd}||jyy#ttt f$rY7wxYw)Ng?)timeout _connector) is_runningrclose TimeoutErrorr NotImplementedErrorgetattr_close)r session connectors r close_sessionzHTTPFileSystem.close_sessionxsl   1 T7==#6G\48      !!.2EF  sAA%$A%c,K|jw|jdd|ji|jd{|_|js6t j ||j|j|j|jS7Ww)Nr r)r.rr r,rweakreffinalizerLr9s r set_sessionzHTTPFileSystem.set_sessionsr == "1$//"Wtyy"WDDVDV"WWDM$$  t'9'9499dmmT}}Xs8BBABc|S)z-For HTTP, we always want to keep the full URLr)clspaths r_strip_protocolzHTTPFileSystem._strip_protocols  r6cDt||}t|dkDr|Sy)N)r%_parentlen)rRrSparr5s rrXzHTTPFileSystem._parents&god# s8a<Jr6c K|jj}|j|tj ||j d{}|j |j|fi|j4d{}|j|| |jd{}|jr=tj|tj|Dcgc]}|d c}z} n%tj|Dcgc]}|d } }dddd{t} t!|}  D]B} t#| t$r| d} | j'dr+t)| dkDr| j*d| j,| } | j'dr|j.r5| j'|j1ddzr| j3| | j5ddj'|j5ddj1ddzs| j3| | dvs| j3dj7|j1d| j9dgE| s;|j;dr*|j=|j1dd d{} |r'| Dcgc]}|d|j;drd nd d c}St?| S777Wcc}wcc}w#t$rg} YwxYw7#1d{7swY xYw7vcc}ww) Nrrz://r8https)z..z../F)detail directoryfilenamesizetype) rrupdateloggerdebugrPgetr@_raise_not_found_for_statustextr(ex2findallexUnicodeDecodeErrorsetr isinstancetuple startswithrYschemenetlocr)rstripaddreplacejoinlstripendswith_ls_realsorted) r0r?r^rkwrJrrjulinksoutpartsls rr{zHTTPFileSystem._ls_reals [[    & S((**7;;ts3Ct{{C  q  , ,Q 4 VVX~$$KK-rzz$?O0P!10PPE+-::d+;EsAM L4M  L  M L/#L6L 7;L2 L>L L'L) M 4L,5DM ;M A;M M M  M9M  M  L L L)%L/(L))L/,M /M5L8 6M= M M cK|jr||jvr|j|}|S|j|fd|i|d{}||j|<|S7w)Nr^)r!dircacher{)r0r?r^rrs r_lszHTTPFileSystem._lssg  " "sdmm';--$C & cC&CFCCC!$DMM#  DsAAAAcX|jdk(r t||jy)zU Raises FileNotFoundError for 404s, otherwise uses raise_for_status. iN)statusFileNotFoundErrorraise_for_status)r0responser?s rriz*HTTPFileSystem._raise_not_found_for_statuss' ??c !#C( (!!#r6cbK|jj}|j|tj |||I||k(ry|j dij}|j |||d{|d<||d<|jd{}|j|j|fi|4d{}|jd{} |j||dddd{| S77m7D7.7#1d{7swY SxYww)Nr6headersRange) rrrerfrgr/_process_limitsrPrhr@readri) r0r?startendrr}rrJr~rs r _cat_filezHTTPFileSystem._cat_files [[    & S  |ffY+002G%)%9%9#uc%JJGG #ByM((**7;;ts3:r: 5 5a.C  , ,Q 4 5 5 K* 5  5 5 5 5 s~BD/DD/#D$*D/DD/D&D'D= D/D D/D/D/DD/D,D" D,'D/iPcnK|jj}|j|tj ||j d{}|j |j|fi|4d{} t|jd} |j| |j||t|r|} n t|d} d} | rQ|j j#|d{} | j%| |j't)| | rQt|s| j+ dddd{y7 7#ttf$rd} YwxYw7{#t|s| j+wwxYw7D#1d{7swYyxYww)Nzcontent-lengthwbT)rrrerfrgrPrhr@intr ValueErrorKeyErrorset_sizeriropencontentrwriterelative_updaterYrE) r0rpathlpath chunk_sizecallbackrr}rJr~rcoutfilechunks r _get_filezHTTPFileSystem._get_files~[[    & U((**7;;tu5<< $ $ 199%567   d #  , ,Q 6% ud+ $"#)).."<.gen_chunkss%+%e, ud+ /A%%affQl3FF1I%%ga&>?z*K,,SZ8FF:.E / / /s9C)BC C)C&"C))rputz.method has to be either 'post' or 'put', not: data) rGrrrerPlowerrrHr@ri) r0rrrrmethodmoderrr}rJmethresps ` `` r _put_filezHTTPFileSystem._put_file s ; %&78 8 /.[[    &((**  (@ K w'.HZ\HRH : :D  , ,T5 9 : : :+ : : : : :s[AC4CAC41C2C45C C4CC4C4C4C1%C( &C1-C4cK|jj}|j| tj ||j d{}|j |j|fi|d{}|4d{|jdkcdddd{S7Y717(7 #1d{7swYyxYw#tj$rYywxYww)NiF) rrrerfrgrPrhr@rr ClientError)r0rSrr}rJr~s r_existszHTTPFileSystem._exists?s [[    &  LL  ,,..G!gkk$//$"7>2>>A & &xx#~ & & &/> & & & & &""  s,C'(CB1)CB3 C B5 CB9 C+B7,C0C'1C3C5C7C9C ?C C C C' CC$!C'#C$$C'cDK|j|fi|d{S7wN)r)r0rSrs r_isfilezHTTPFileSystem._isfileKs"!T\\$1&1111s   c "|dk7rt||n |j}|jj} |j| d<| j |i} |xs)| j |j |fi|xs| d}t|j|j} |rO|rM| jddr;t||f| ||||xs |j|xs |j|jd| St||f||j| d| S)aMake a file-like object Parameters ---------- path: str Full URL with protocol mode: string must be "rb" block_size: int or None Bytes to download in one request; use instance value if None. If zero, will return a streaming Requests file-like instance. kwargs: key-value Any other parameters, passed to requests calls rrrcpartialT)rJr'rrcr*r+r )rr rJ)rGr'rrrreinforr rPrhHTTPFiler*r+HTTPStreamFile) r0rSrr' autocommitr*r+rcrr}rrJs r_openzHTTPFileSystem._openNs,2 4<% %#-#9Zt [[   !..> &Mt{{9499T#s rukeyzHTTPFileSystem.ukeysT[[$--88r6c Ki}|jd{}dD]W} |jt|j|f||d|j|d{|j dnY|dd|d d iS7k7*#t $r2}|dk(r t||tjd|Yd}~d}~wwxYww) aHGet info of URL Tries to access location via HEAD, and then GET methods, but does not fetch the data. It is possible that the server does not supply any size information, in which case size will be given as None (and certain operations on the corresponding file will not work). N)headrh)r2rJrcrhrW)exc_inforbrcrdr`) rPre _file_infor@rrh Exceptionrrfrg)r0r?rrrJpolicyexcs rrzHTTPFileSystem._infos((**% /F / $,$* '++  ! 88F#/0 /&TBTB66BB++  /U?+C0c9 R# ..  /sJCB C8BB B5CB C(B>9C>CCc K||dkr tdddl}|jd}|j|}|xs|jd}|j ddk\r|j dn t |}|j ddk\r|j dn t |}t ||} |jd d } t|sC|j|fi|d{r"| s|gS||j|fi|d{iS| sgSiSd|d| vr7|d| jd} |d| dz} || dzdjddz} nd } || dzdjddz} d |vr5|1|j d } || djddz}| |z |z} nd} |j| f| d d d|d{}t||rdnd z}|j|}t!|j#Dcic]T\}}|j%|j'dr/|s*|ddk(r"|jdr|j'dn||V}}}| r|St)|S77l7cc}}ww)a Find files by glob-matching. This implementation is idntical to the one in AbstractFileSystem, but "?" is not considered as a character for globbing, because it is so common in URLs, often identifying the "query" part. Nr\zmaxdepth must be at least 1rr)z/**z/**[r^FrWz**T)maxdepthwithdirsr^rdr_)rrerzrTfindrYminr/ has_magicrrrindexcount_findr compiler|itemsmatchrulist)r0rSrrrends_with_slashappend_slash_to_dirnameidx_star idx_bracemin_idxr^rootdepthidx_double_starsdepth_double_starsallpathspatternprrs r_globzHTTPFileSystem._globs  HqL:; ;--,##D)"1"QT]]=5Q%)YYs^q%8499S>c$i&*iin&9DIIcNs4y h *He,!T\\$1&111 6M  4(B6(B"BCCII D'N "8Gn++C0G 'A+&D1'--c2Q6ED1'--c2Q6E 4<##'99T? %)*:*;%<%B%B3%G!%K" 22X=#   4 @F  !R!HI"**W%"(.."23  4}}QXXc]+ /LK/JJsO       J9 a2#C,   sKC-J/I90 JI<B8J I? AJAJ'J<J?JJc~K t|j|d{S7#ttf$rYywxYww)NF)boolrrr)r0rSs r_isdirzHTTPFileSystem._isdirs: dhhtn,- -,!:.  s(=(&(=(:=:=)T)NN)rNNNNN)rNr)"__name__ __module__ __qualname____doc__seprr&propertyr:r@ staticmethodrLrP classmethodrTrXr{rrlsrirr rrrrrrrrrr __classcell__r5s@rrr$s C ?&B3  0d c B$&(1;K$D! 0:d 2  7r 9 CDGRr6rcreZdZdZ dfd Zdfd ZdZeeZdZ dZ ee Z xZ S) ra A file-like object pointing to a remote HTTP(S) resource Supports only reading, with read-ahead of a predetermined block-size. In the case that the server does not supply the filesize, only reading of the complete file in one go is supported. Parameters ---------- url: str Full URL of the remote resource, including the protocol session: aiohttp.ClientSession or None All calls will be made within this session, to avoid restarting connections where the server allows this block_size: int or None The amount of read-ahead to do, in bytes. Default is 5MB, or the value configured for the FileSystem creating this file size: None or int If given, this is the size of the file in bytes, and we don't attempt to call the server to find the value. kwargs: all other key-values are passed to requests calls. c |dk7r td| |_| |_||_||_||dd|_t |d||||||d| y)NrzFile mode not supportedr`ra)fsrSrr'r*r+r)rGrr r?rJdetailsr%r&) r0rr?rJr'rr*r+rcr rrr5s rr&zHTTPFile.__init__'sq 4<%&?@ @(  #T6B   !!'   r6c>|dkrD|jdk(r5|j|j|jks|j|j|dkr4|jn#t |j|jz |}t ||S)a5Read bytes from file Parameters ---------- length: int Read up to this many bytes. If negative, read all content to end of file. If the server has not supplied the filesize, attempting to read only part of the data will raise a ValueError. r)locrc blocksize _fetch_allrr%r)r0lengthr5s rrz HTTPFile.readFs|aZDHHMYY*tyyDNN/J OO  99 z!TXX-v6Fw|F##r6c0Ktjd|t|jts|j j |jj|jfi|jd{}|4d{|j|jd{}t t|dd||_t||_dddd{yy7v7m7G7 #1d{7swYyxYww)zRead whole file in one shot, without caching This is only called when position is still at zero, and read() is called without a byte-count. zFetch all for N)rcfetcherrr)rfrgrpcacherrJrhrr@r?rrrrYrc)r0r~rs rasync_fetch_allzHTTPFile.async_fetch_all]s  ~dV,-$**h/&dll&&tww'9'9$(('CSt{{SSA % %""$FFHn%S44c  H  % % %0S %$ % % % %slBDC9 D C;D$D5C=61D' D2C?3D;D=D?DDD DDc|jdd}tjd|}|sy|ddk(rdx}}n.|djdDcgc] }t |c}\}}|d dk(rdn t |d }|||fScc}w) zParse the Content-Range header Content-RangerWzbytes (\d+-\d+|\*)/(\d+|\*))NNNr\rN-r)rhrrsplitr)r0rsmrrxtotals r_parse_content_rangezHTTPFile._parse_content_rangeps KK , HH3Q 7# Q43; EC*+A$**S/:Q#a&:JE3! QqTc5  ;sBcKtjd|d|d||jj}|j dij}d|d|dz |d<tj|j d|d|j j|jj|j fd|i|d {}|4d {|jd k(r d d d d {y |j|jd k(xsQ|j|jd |k(xs.t|jjd|dz||z k}|r|jd {}n{|d kDr t!dd }g} |j"jdd {} | r(|j%| |t'| z }|||z kDrnnOd j)|d ||z }|cd d d d {S7L7D7'77e7#1d {7swYy xYww)a3Download a block of data The expectation is that the server returns only the requested bytes, with HTTP code 206. If this is not the case, we first check the headers, and then stream the output - if the data size is bigger than we requested, an exception is raised. zFetch range for z: rrbytes=r\rz : Nir6rContent-LengthzThe HTTP server doesn't appear to support range requests. Only reading this file from the beginning is supported. Open with block_size=0 for a streaming file interface.i)rfrgrrr/r?rJrhrr@rrrrrrrrappendrYrx) r0rrrrr~response_is_rangerclrs rasync_fetch_rangezHTTPFile.async_fetch_range~sM  'vRwau=>!!#**Y+002#E7!C!G95 zWW%5$678"$,,"" GG  txx ( 29 =C  ) ) xx3) ) )     CP,,QYY7:eCPqyy}}%5sQw?@C%KO  !FFHn M"#)).."77E 5)c%j(e +!hhsmMcEk2S) ) )  ) ) $%8?) ) ) ) sCIH! I H$!I$H05 IH'IBH0 H*8H0H,AH0 IH.I$I'I*H0,H0.I0I6H9 7I>I)NNrrNNNF) rrrrr&rr rrrr _fetch_rangerrs@rrrsU8    >$.%"o.J !9v 12Lr6rz([*[])c4tj|}|duSr) magic_checksearch)rrs rrrs   q !E  r6cPeZdZdfd ZddZddZeeZdZfdZ xZ S) rc jdd__|_|_|dk7rt dd_t d||ddfd}tj|_ |j_y) NrFrrnonerrSrr*cKjjjjfij d{}jj ||S7"wr)rJrhrr@ __aenter__ri)r~rr0r?s rcorz$HTTPStreamFile.__init__..cors\&dll&&tww'9'9#'>I&ITTVVA GG / /3 7HWsAA0 A. #A0r) r/rr?r rJrrr%r&rr~) r0rr?rr rJrr*r5s ` ` ` rr&zHTTPStreamFile.__init__s"JJ~u=  4<  #T2  QBStQ&Q  dii%GG r6cX|dk(r|dk(ry||jk(r|dk(rytd)Nrr\zCannot seek streaming HTTP file)rr)r0rwhences rrzHTTPStreamFile.seeks2 !8!  $((?v{ :;;r6cK|jjj|d{}|xjt |z c_|S7$wr)r~rrrrY)r0numrs r_readzHTTPStreamFile._reads=FFNN'',, CH -s)AA%Ac@K|jjywr)r~rEr9s rrIzHTTPStreamFile._closes  sc~tj|j|jt|yr)asynciorun_coroutine_threadsaferIr r%rEr0r5s rrEzHTTPStreamFile.closes&(( B  r6)rNN)rr) rrrr&rr/rrrIrErrs@rrrs-$<  Dr6rc4eZdZ dfd ZddZfdZxZS)rc ||_||_d|_|dk7rt|dd|_||_t ||||d||_y)Nrrr&r') r?rJr~rrrr%r&rc) r0rr?rr rJrcrr5s rr&zAsyncStreamFile.__init__sW  4<  #T2   BStG r6cK|j|jj|jj |j fi|j jd{}|jj||j ||_|jjj|d{}|xjt|z c_ |S7~7&wr) r~rJrhrr@r?rr)rirrrrY)r0r.r~rs rrzAsyncStreamFile.reads 66>&dll&&""488,04 jlA GG / /488 <DFFFNN'',, CH  -s%A(C-*C)+AC-C+%C-+C-cK|j!|jjd|_t|d{y7wr)r~rEr%r4s rrEzAsyncStreamFile.closes4 66  FFLLNDFgmos?A AA )rNNNr)rrrr&rrErrs@rrrs@D  r6rcK|j}|jdij}d|d|dz |d<|j|fd|i|d{}|j|4d{|j d{}dddd{|r8t |d5} | j || jdddyS77j7T7F#1d{7swYVxYw#1swYyxYww)Nrrrr\rzr+b)rr/rhrrrrr) rJr?rrr`rrr~rrs r get_ranger: s [[]FjjB',,.Gwaay1GGgkk#9w9&99AFFHn $  ! FF5M GGCL    :  sADCD4C5D8C% C! C% DC#D/#C: DD!C%#D%C7+C. ,C73D:D?DcbKtjd||j}|jdd}|j dij}d|d<||d<i}|dk(r|j |fd|i|d{}n1|d k(r|j|fd|i|d{}nt d ||4d{|jd |jvr;d |jvs|jd d vrWt|jd |d<n;d|jvr-t|jdjdd|d<d|jvr$|jdjdd|d<|jj ddk(rd|d<t|j|d<dD]0}|jj |s|j|||<2dddd{|S77j7S7#1d{7swY|SxYww)zCall HEAD on the server to get details about the file (size/checksum etc.) Default operation is to explicitly allow redirects and use encoding 'identity' (no compression) to get the true size of the target. zRetrieve file size for %sallow_redirectsTridentityzAccept-EncodingrNrhz)size_policy must be "head" or "get", got rzContent-Encoding)r=rWrcrrr\z Content-Type;rmimetypez Accept-Rangesr&Frr?)ETagz Content-MD5Digest)rfrgrr/rhr TypeErrorrrrr partitionstrr?) r?rJr2rarrrr~checksum_fields rrrs6  LL,c2 [[]F %t ,B ::i $ ) ) +D(D F9 Df',,sABA&A A  '++c@2@@ @CK=QRRAA  qyy ("2aii@R6SX7 #199-=#>?V  )qyy9??DQGHDL QYY & yy8BB3GJD  99== )V 3$DO!%%jU ? ANyy}}^,'(yy'@^$ A1AA8 KC B @AAAAA8 KssBH/H"H/'H(H/HH/D"H(H< H/HH/H/H/H/H,H" H,'H/cK|td{}t|g|d|i|d{}|jdS7.7w)NrJrc)rrrh)r?rJargsrrs r _file_sizerIQsI" $CB4BB6B BD 88F % BsAAAAAAr)r)-r2rloggingrrNr urllib.parserrr< fsspec.asynrrrrfsspec.callbacksr fsspec.exceptionsr fsspec.specr fsspec.utilsr r rrrcachingrrrmrk getLoggerrfrrrr"rrrr:rrI file_sizerr6rrTs  ! VV-,,RZZMNbjjEF   = )+g_gTk3#k3\bjj"  &)&R/@"0f  $ r6