wg4ddlmZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZddlmZmZddlmZddlmZddlmZmZdd lmZdd lmZdd lmZdd lm Z dd l!m"Z"ddl#m$Z$e rddlm%Z%ejLdZ'Gdde"Z(GddeZ)Gdde)Z*Gdde*Z+GddZ,y)) annotationsN)rmtree) TYPE_CHECKINGAnyCallableClassVar)AbstractFileSystem filesystem)DEFAULT_CALLBACK)compr) BaseCache MMapCache)BlocksizeMismatchError)create_cache_mapper) CacheMetadata)AbstractBufferedFile) Transaction)infer_compression)AbstractCacheMapperz fsspec.cachedceZdZddZy)WriteCachedTransactioncf|jDcgc]}|j}}|jDcgc]}|j}}|r|jj |||jj d|j_d|j_d|_ycc}wcc}wNF)filespathfnfsputclear_intrans _transaction)selfcommitfrpathslpathss b/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/fsspec/implementations/cached.pycompletezWriteCachedTransaction.completes"&**-Q!&&-- $ +1!$$++  GGKK '  #.+s B)B.NT)__name__ __module__ __qualname__r(r'rrsr.rceZdZUdZdZded< d dfd ZedZdZ dZ d Z d Z d Z d Zd ZddZdZ ddZdZddZdZddZfdZdZdZxZS)CachingFileSystemaSLocally caching filesystem, layer over any other FS This class implements chunk-wise local storage of remote files, for quick access after the initial download. The files are stored in a given directory with hashes of URLs for the filenames. If no directory is given, a temporary one is used, which should be cleaned up by the OS after the process ends. The files themselves are sparse (as implemented in :class:`~fsspec.caching.MMapCache`), so only the data which is accessed takes up space. Restrictions: - the block-size must be the same for each access of a given file, unless all blocks of the file have already been read - caching can only be applied to file-systems which produce files derived from fsspec.spec.AbstractBufferedFile ; LocalFileSystem is also allowed, for testing ) blockcachecachedzClassVar[str | tuple[str, ...]]protocolc  Ntd i| | | td|du|duz s td|dk(r9tj} | g} t j j| nt|tr|g} n|} tj| dd| _ |xsi_ |_|_|_| _d_| | td| | _nt)||nd _t|tr|n4t|j*tr |j*n|j*d _t/j_j3||nt5|fij_fd }|_y) a Parameters ---------- target_protocol: str (optional) Target filesystem protocol. Provide either this or ``fs``. cache_storage: str or list(str) Location to store files. If "TMP", this is a temporary directory, and will be cleaned up by the OS when this process ends (or later). If a list, each location will be tried in the order given, but only the last will be considered writable. cache_check: int Number of seconds between reload of cache metadata check_files: bool Whether to explicitly see if the UID of the remote file matches the stored one before using. Warning: some file systems such as HTTP cannot reliably give a unique hash of the contents of some path, so be sure to set this option to False. expiry_time: int The time in seconds after which a local copy is considered useless. Set to falsy to prevent expiry. The default is equivalent to one week. target_options: dict or None Passed to the instantiation of the FS, if fs is None. fs: filesystem instance The target filesystem to run against. Provide this or ``protocol``. same_names: bool (optional) By default, target URLs are hashed using a ``HashCacheMapper`` so that files from different backends with the same basename do not conflict. If this argument is ``true``, a ``BasenameCacheMapper`` is used instead. Other cache mapper options are available by using the ``cache_mapper`` keyword argument. Only one of this and ``cache_mapper`` should be specified. compression: str (optional) To decompress on download. Can be 'infer' (guess from the URL name), one of the entries in ``fsspec.compression.compr``, or None for no decompression. cache_mapper: AbstractCacheMapper (optional) The object use to map from original filenames to cached filenames. Only one of this and ``same_names`` should be specified. Nz9Please provide filesystem instance(fs) or target_protocolz@Both filesystems (fs) and target_protocol may not be both given.TMPTexist_okzMCannot specify both same_names and cache_mapper in CachingFileSystem.__init__Frcjjjtj|SN)r_strip_protocoltype)rr"s r'r;z3CachingFileSystem.__init__.._strip_protocols'77**4:+E+Ed+KL Lr.r-)super__init__ ValueErrortempfilemkdtempweakreffinalize_remove_tempdir isinstancestrosmakedirsstoragekwargs cache_check check_filesexpiry compression _cache_size_mapperrr3target_protocolr _metadata load_cacher rr;)r"rQ cache_storagerKrL expiry_timetarget_optionsr same_namesrN cache_mapperrJtempdirrIr; __class__s` r'r>zCachingFileSystem.__init__?sn "6" :/1K d $67R  E !&&(GiG   T4#7#7 A--(/' GBK$/ $* &&! &    !l&>-   #'DL.(4 %DL /3/ !+BKK!="++2;;q>  't||4 "J,V$++,V M*9r.c: t|y#t$rYywxYwr:)r Exception)rYs r'rDz!CachingFileSystem._remove_tempdirs   7O   s  cLtj|jddy)Nr6Tr7)rGrHrIr"s r'_mkcachezCachingFileSystem._mkcaches DLL$t4r.c|j0|jd}tdj|d|_|jS)zReturn size of cache in bytes. If more than one cache directory is in use, only the size of the last one (the writable cache directory) is returned. r6fileT)withdirs)rOrIr du)r" cache_dirs r' cache_sizezCachingFileSystem.cache_sizesG    # R(I)&144Y4ND r.c|jj|jtj|_y)z#Read set of stored blocks from fileN)rRloadr_time last_cacher^s r'rSzCachingFileSystem.load_caches(  ))+r.c|j|jjtj|_d|_y)z#Save set of stored blocks from fileN)r_rRsaverhrirOr^s r' save_cachezCachingFileSystem.save_caches0  ))+r.c|j|jsytj|jz |jkD}t d|j D}|s|s|j yy)z0Reload caches if time elapsed or any disappearedNc3ZK|]#}tjj|%ywr:)rGrexists).0rIs r' z1CachingFileSystem._check_cache..sLGw/Ls)+)r_rKrhriallrIrS)r"timecond existconds r' _check_cachezCachingFileSystem._check_caches^  99;043C3CCLt||LL 9 OO %r.c||j|}|j|jj||S)z Is path in cache and still valid)r;rurR check_filer"rs r' _check_filezCachingFileSystem._check_files5##D) ~~((t44r.cbt|jd|jd|_y)zRemove all files and metadata from the cache In the case of multiple cache locations, this clears only the last one, which is assumed to be the read/write one. r6N)rrIrSrOr^s r' clear_cachezCachingFileSystem.clear_caches' t||B  r.cV|s |j}|j|jj|\}}|D]7}tj j |s#t j|9|r(t|jd|jd|_ y)aRemove all expired files and metadata from the cache In the case of multiple cache locations, this clears only the last one, which is assumed to be the read/write one. Parameters ---------- expiry_time: int The time in seconds after which a local copy is considered useless. If not defined the default is equivalent to the attribute from the file caching instantiation. r6N) rMrurR clear_expiredrGrroremoverrIrSrO)r"rU expired_fileswritable_cache_emptyrs r'clear_expired_cachez%CachingFileSystem.clear_expired_caches++K .2nn.J.J;.W+ + Bww~~b! "   4<<# $ OO r.c|j|}|jj|}|tj|d|_y)zRemove cached version of given file Deletes local copy of the given (remote) path. If it is found in a cache location which is not the last, it is assumed to be read-only, and raises PermissionError N)r;rRpop_filerGr~rO)r"rrs r'pop_from_cachez CachingFileSystem.pop_from_cache s@##D) ^^ $ $T * > IIbMr.c  j|}jj|}d|vr"jj|f||||d|Sj|}|rL|\}}|d|d} } | dur"tj d|t ||Stj d|nj|} tjjjd| }t} || | tjjj|d }jj!||tj d |j#jj|f||||d d | j$r6j$d k(r t'|n j$} t)|  d d|vr1|d j*k7r.t-d|dd j*d j*|d<t/ j* j0 j2||  _ j6 fd _j9 S)a!Wrap the target _open If the whole file exists in the cache, just open it locally and return that. Otherwise, open the file on the target FS, and make it have a mmap cache pointing to the location which we determine, in our cache. The ``blocks`` instance is shared, so as the mmap cache instance updates, so does the entry in our ``cached_files`` attribute. We monkey-patch this file, so that when it closes, we call ``close_and_update`` to save the state of the blocks. r)mode block_size autocommit cache_optionsrblocksTOpening local copy of %sz#Opening partially cached copy of %sr6originalrrrhuidz!Creating local sparse file for %snone)rrrr cache_typeinferrbr blocksizezDCached file must be reopened with same block size as original (old: z, new )c(jSr:)close_and_update)closer$r"sr'z)CachingFileSystem._open..os$//59r.)r;r_openryloggerdebugopenrPrGrjoinrIsetrhukeyrR update_filer_rNrr rrr _fetch_rangesizecacherrl)r"rrrrrrJdetailrhashrcomprr$s` @@r'rzCachingFileSystem._opensf*##D)ww&&t, d? 477==%%+    !!$' JFB!$<)9&D~ 7>B~% LL> E<<%Ddll2.5BUF   ww||D) F NN & &tV 4 LL*rcatgetheadinforpipetailrisdir__eq__roisfileto_dictto_json__hash__r>r_cat_filer open_many pipe_file __reduce__re cat_ranges get_mapperrS local_file read_blockrlryr{ commit_manyrurend_transaction__getattribute___paths_from_pathrstart_transaction_make_local_detailsrcXttj|i|Sr:)getattrr<__get__)rkwitemr"s r'rz4CachingFileSystem.__getattribute__..s0'NwtDz4'@'H'H'N((r.) __reduce_ex__) transaction)_cachetransaction_typerZ__dict__r__self__)AttributeErrorr<rrrobjectrrrinspect isfunctionisdatadescriptorhasattrrr=)r"rdrclsmrZs`` r'rz"CachingFileSystem.__getattribute__s: + + \  $ $ ? ":))11$7 7 1 14:t, , ; :   # #D* 5 UU4  19T7N ^r{{"{{4((r(CT"A""1%)A)A!)DAz*ajj.@yyS))H7+D1 1r.c||uryt|t|sy|j|jk(xr|j|jk(xr|j|jk(xr|j |j k(xrj|j |j k(xrO|j|jk(xr4|j|jk(xr|j|jk(S)zTest for equality.TF) rEr<rIrJrKrLrMrNrPrQ)r"others r'rzCachingFileSystem.__eq__s 5=%d, LLEMM ) > u||+ >  E$5$55 >  E$5$55 > u||+  >   E$5$55  >   -  >$$(=(== r.ctt|jtt|jz t|j z t|j z t|jz t|jz t|jz t|jz S)zCalculate hash.) rtuplerIrFrJrKrLrMrNrPrQr^s r'rzCachingFileSystem.__hash__s t||$ %3t{{#$ %4##$ %4##$ %4;;  4##$  % 4<<  !4''( ) r.) Nr5 Fi: NNNNN)rWz bool | NonerXzAbstractCacheMapper | Noner:)rNTN)rrFrrreturnrFr))r*r+r,__doc__r3__annotations__r> staticmethodrDr_rerSrlruryr{rrrrrrrrrr __classcell__rZs@r'r0r0)s&1IH-H "&37o9 o91o9b 5  &   5   <   Yv%" "(O2b "  r.r0c@eZdZdZdZdZdZdZdZdde fd Z d d Z y ) WholeFileCacheFileSystema Caches whole remote files on first access This class is intended as a layer over any other file system, and will make a local copy of each file accessed, so that all subsequent reads are local. This is similar to ``CachingFileSystem``, but without the block-wise functionality and so can work even when sparse files are not allowed. See its docstring for definition of the init arguments. The class still needs access to the remote store for listing files, and may refresh cached files. filecacheTc |Dcgc]}|j}}d|jvr|jnl|Dcgc]`}t|j|f|jt jj |jd|j|d|bc}S|jrt|Dcgc]}|j|}}t||D cgc] \}} | r | } }} t||D cgc]A\}} t jj |jd|j|C} }} t| |D cgc] \} } | r |  } } } | r|jj| | | Dcgc]D}||j|dtj|jj|dF}}t| |D]!\}}|j j#||#|j%d}t|| Dcgc]&\}}t'|r||n||j(c}}Scc}wcc}wcc}wcc} }wcc} }wcc} } wcc}wcc}}w)Nrr6rrTrc0t|tr|dS|S)N)rErrs r' firstpartz5WholeFileCacheFileSystem.open_many..firstpart<s&r512a5 9r 9r.r)rrr_ LocalTempFilerrGrrIrPrNNotImplementedErrorryziprrhrrRrrlr)r" open_filesrJofpathsrspdetailsprdownpathdownfn0rdownfn newdetailrrfn0fn1s r'rz"WholeFileCacheFileSystem.open_manysX#-.R.. *// ! MMO" GG$ww||DLL$4dll46HI        % %278B4##B'88"%eW"5?$!QQA??E7+ 1 GGLLb)4<<? ;  #&gw"7AQq"AA  GGKK& )%  !%,,t," IIK77<<-  I !$Hi 8 9 f**48 9 OO  :  1 S 33Cjoo F  _/ 9? B $ s=IA%I";I'$ I,/I,AI2 I8(I8A I>.+Jcp|jj|Dcgc]}|jc}|Dcgc]}|jc}|Dcgc]}|j c}|D]"} t j |j$d|_ ycc}wcc}wcc}w#t$rYGwxYwr:) rrrrrrGr~nameFileNotFoundErrorrO)r"rr$s r'rz$WholeFileCacheFileSystem.commit_manyEs  :.aQTT.0LA0LM&'q' A  !&&!   /0L' %  s"BB B$1B)) B54B5cL|j|}tjj|jd|}||dt j |j j|d}|jj||tjd||S)Nr6TrCopying %s to local cache) rPrGrrrIrhrrrRrrr)r"rrrrs r'rz,WholeFileCacheFileSystem._make_local_detailsPs||D! WW\\$,,r*D 1IIK77<<%   ""40 0$7 r.Fraisec p|j|||jd}g}g}g} i} |jD]t} |j| } | s4|j | } |j | |j | nt | tr| nd| f\} } | j | v|r,|jj|||j|jt|t|| D]>\} } t| d5}|j!| | <ddd|j#d@t |t$rt|dk(r |dur| |d} | S#t$r,}|dk(r|dk(r|| | <|j| Yd}~hd}~wwxYw#1swYxYw) Nmaxdepth) recursiverr rrrFr) expand_pathrcopyryrappendrErr\r~rrlset_sizelenrrreadrelative_updaterF)r"rron_errorcallbackrJrgetpaths storepathsfnsoutrrrer$s r'rzWholeFileCacheFileSystem.cat^s  I :0F!   A ))!,11!4BOOA&%%b)+5fe+D4QW.JFB 2 "  GGKK* - OO #e*%_ (EArb$ "1A "  $ $Q ' ( dC SZ1_e9KeAh-C % w&x'CF Q   " "s%A1E4F,4 F)=!F$$F),F5 c |j|}d|vru|j|}tjj |j d|}|j Dcic] \}}|dvr||}}}t||f||d|S|j|} | r\| \} }| d| d} } | dur:tjd|t||} | jd | _ | Std |d |j|}||d <|j!|j"r|j$j&|fi|5} t|d 5} t)| t*r0t-d| j.j0| j2| _|j"dk(r t5|n |j"}t7|| d} d}|r2t9| dd}| j;|}| j=||r2ddddddn|j$j?|||jA|j'||Scc}}w#1swYVxYw#1swY=xYw)Nrr6rrrrrrTrrz&Attempt to open partially cached file z as a wholly cached filerwbrrrrrP)!r;rPrGrrrIitemsrryrrrrrr?rr_rNrrrErr rfetcherrrr rrwriteget_filerl)r"rrrJrrkvuser_specified_kwargsr_rr$f2rdatablocks r'rzWholeFileCacheFileSystem._opensf##D) d?<<%Ddll2.5B#LLN%AqII1% !% !tW$2WAVW W!!$' JFBt fX&6vA~ 7>TN#ZZ 3  E66%=DHHTN # # # GG  T2 & zz$%%i%D # # # #s+'I I$B,II$I! I$$I-Nr) r*r+r,rr3rrrrr rrr-r.r'rrs9 HJ3 j   "! +Z9&r.rcxeZdZdZdZdZeZfdZdZ dZ dZ dfd Z dd Z d Zdd Z dfd Zdd ZxZS)SimpleCacheFileSystemaCaches whole remote files on first access This class is intended as a layer over any other file system, and will make a local copy of each file accessed, so that all subsequent reads are local. This implementation only copies whole files, and does not keep any metadata about the download time or file details. It is therefore safer to use in multi-threaded/concurrent situations. This is the only of the caching filesystems that supports write: you will be given a real local open file, and upon close and commit, it will be uploaded to the target filesystem; the writability or the target URL is not checked until that time. simplecacheTc |j}dD]}d||< t|di||jD]9}tj j |r#t j|d;y)N)rKrUrLFTr7r-)rr=r>rIrGrrorH)r"rJrkeyrIrZs r'r>zSimpleCacheFileSystem.__init__sg [[]@ CBsG  2|| 4G77>>'* Gd3 4r.c|j|j|}|jD]E}tjj ||}tjj |sC|cSyr:)rurPrIrGrrro)r"rsharIrs r'ryz!SimpleCacheFileSystem._check_filesX ll4 || Ggs+Bww~~b!  r.cyr:r-r^s r'rlz SimpleCacheFileSystem.save_cache r.cyr:r-r^s r'rSz SimpleCacheFileSystem.load_cacher7r.c |jr-|j|d5}|j|dddyt|||y#1swYyxYw)Nr!)r rr%r=r)r"rvaluerJr$rZs r'rzSimpleCacheFileSystem.pipe_filesP ==4& !   G dE *  s A  Ac |j|}g} |jj|fddi|j}d}|j rK|j ddz}|jjD]}|j|k(r2|j||jxs|jddE|jj|sa|jjd|jdk(r<|j|j|jxs|jdddj|jj!dd|jddz} |j| ddd ||s||r|St#d |DS#t$r }|}Yd}~d}~wwxYw) NrT/rar rr<rr directoryc3&K|] }|d yw)r Nr-)rpr*s r'rqz+SimpleCacheFileSystem.ls..s1Aai1s)r;rrrr r rstriprrrrrtell startswithcountrsplitsorted) r"rrrJrexrpath1r$dnames r'rzSimpleCacheFileSystem.lss##D)  dggjj!%+df  B ==KK$s*E%%++ X66T>NN!%qvv/A6RVV&&u-vv||C(EKK,<<%&VVQVV5GqvvxQWX!$c):;QU[[=MPQ=Q)R Sq+'VW X >'H N1111/! B s-F88 GG  Gc |j|jr|jjDcgc]}|jk(s|}}|rT|dj r,t jj|djn|dj}|ddStfd|jjD}|rdddS|jjfi|Scc}w)Nrrar=c3ZK|]"}|jjdz$yw)r<N)rrB)rpr*rs r'rqz-SimpleCacheFileSystem.info.. s$RaAFF%%dSj1Rs(+r>) r;r rrrrrGgetsizerrAanyrr)r"rrJr*r$rs ` r'rzSimpleCacheFileSystem.infos##D) == ,,22EqaffnEAE34Q4;;rwwqtww/AaDIIK $dFCCR4;K;K;Q;QRRA $aEEtww||D+F++Fs C5 C5c t|tr$|j|j||fi|yt|tr<|j D](\}}|j|j||fi|*yt d)Nzpath must be str or dict)rErFrr;dictr#r?)r"rr:rJr'r(s r'rzSimpleCacheFileSystem.pipe%s dC DNN4//5u G G d #  E1t33A6DVD E78 8r.c P|Dcgc]}|j|}}t||D cgc] \} }| dus |} } }t||D cgc] \} }| dus | }} }|jj| |t ||||f||d|Scc}wcc}} wcc}} w)NF)max_gapr)ryrrrr=r) r"rstartsendsrPrrJrr&lr%rZs r'rz SimpleCacheFileSystem.cat_ranges.s055!$""1%55 #FE 2A1a5j!AA #FE 2A1a5j!AA  FF#w! 64 )08 GM  6AAsB BB B"B"c |j|}|j|}d|vrptjj |j d|}|j Dcic] \}}|dvr||}}}t||f||j |d|S|j|}|r t||Stjj |j d|}tjd|||d<|jd|_|jr|j j"|fi|5} t|d5} t%| t&r0t)d| j*j,| j.| _|jd k(r t1|n |j} t3| | d } d } | r2t5| d d} | j7| } | j9| | r2ddddddn|j j;|||j#||Scc}}w#1swYFxYw#1swY-xYw)Nrr6r )rrrr rr!rrrrTrr")r;rPrGrrrIr#rr ryrrrr_rOrNrrrErr rr$rrr rrr%r&)r"rrrJr5rr'r(r)r$r+rr,r-s r'rzSimpleCacheFileSystem._open9s4##D)ll4  d?dll2.4B#LLN%AqII1% !% !#}},  (    d # D> ! WW\\$,,r*C 0 0$7v    t.v. #!T"d^ #ra!56'177??AFFCAG''72&d+)) $K-#A{I>E66%=DHHTN # # # GG  T2 &zz$%%S%0 # # # #s+'H7> I  B,H=8I =I I  Ir:r))Nrr.)r*r+r,rr3rrrr>ryrlrSrrrrrrrrs@r'r0r0sS HJ-4  +2@ ,9;C  /&r.r0cZeZdZdZd dZdZdZdZdZdZ dZ e d Z dd Z d Zy )rz8A temporary local file, which will be uploaded on commitc ||_t|||_||_|r|jj |||_d|_||_d|_||_ ||_ yr) rrfhrseekrrrrrrJ)r"rrrrrrXrJs r'r>zLocalTempFile.__init__nsZr4.  GGLL    $ r.ct|j|j|jd|j|j ffS)Nzr+b)rrrrrrAr^s r'rzLocalTempFile.__reduce__{s7  WWdii%$))+ N  r.c|jSr:)rWr^s r' __enter__zLocalTempFile.__enter__s wwr.c$|jyr:)r)r"exc_typeexc_valexc_tbs r'__exit__zLocalTempFile.__exit__s  r.c|jry|jjd|_|jr|j yy)NT)rrWrrr#r^s r'rzLocalTempFile.closes5 ;;    ?? KKM r.cv|jjtj|jyr:)rWrrGr~rr^s r'discardzLocalTempFile.discards   $''r.c||jj|j|jfi|jyr:)rrrrrJr^s r'r#zLocalTempFile.commits' DGGTYY6$++6r.c|jSr:rr^s r'r zLocalTempFile.names wwr.c d|jS)NzLocalTempFile: )rr^s r'__repr__zLocalTempFile.__repr__s  ,,r.c.t|j|Sr:)rrW)r"rs r' __getattr__zLocalTempFile.__getattr__stww%%r.N)r!Tr)rrF)r*r+r,rr>rr[r`rrcr#propertyr rgrir-r.r'rrksFB  7-&r.r)- __future__rrloggingrGr@rhrBshutilrtypingrrrrfsspecr r fsspec.callbacksr fsspec.compressionr fsspec.corer rfsspec.exceptionsr#fsspec.implementations.cache_mapperr%fsspec.implementations.cache_metadatar fsspec.specrfsspec.transactionr fsspec.utilsrr getLoggerrrr0rr0rr-r.r'rzs"  991-$,4C?,**G   ? + [ S *S lE&0E&Pa&4a&H6&6&r.