wgjddlZddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z ddl Z ddlZddlmZddlmZddl mZmZmZddlmZdd lmZmZmZej>d Z Gd d e!Z"d Z#dZ$dZ%GddejLjNZ(GddejLjRZ*dZ+GddejLjXZ-GddeZ.y#e$r e sddlZYwxYw)N) lru_cache)chain) TYPE_CHECKINGLiteral)AsyncFileSystem)DEFAULT_CALLBACK) filesystemopensplit_protocol)AsyncFileSystemWrapper) isfilelikemerge_offset_ranges other_pathszfsspec.referencec$eZdZfdZdZxZS)ReferenceNotReachablec:t||||_||_yN)super__init__ referencetarget)selfrrargs __class__s e/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/fsspec/implementations/reference.pyrzReferenceNotReachable.__init__s $" c:d|jd|jS)Nz Reference "z" failed to fetch target )rrrs r__str__zReferenceNotReachable.__str__#sT^^,,Edkk]SSr)__name__ __module__ __qualname__rr __classcell__rs@rrrs TrrcFtt|jSr)nextitervalues)ds r_firstr*'s QXXZ !!rc|j|}t|ttfr/t|dtr|drt |ddS|dSyyNr)get isinstancelisttuplestrr )path referencesrefs r_prot_in_referencesr5+sW .. C#e}%*SVS*A,/F~c!f%a(>A>+B%rct|trt|||giSi}|D]/}t||}|j|gj |1|Sr)r.r1r5 setdefaultappend)pathsr3outr2protocols r_protocol_groupsr<1s`%#E:6@@ C2&tZ8 x$++D12 JrceZdZdZy)RefsValuesViewc#K|jjjD]'}tj|j )|jj jEd{|jjD]c}|jj|}t|dk(r|j|dzA|jj|Ed{ey77 w)Nr/0) _mapping zmetadatar(jsondumpsencode_itemslistdir_get_chunk_sizeslen_generate_all_records)rvalfield chunk_sizess r__iter__zRefsValuesView.__iter__<s==**113 +C**S/((* * +==''..000]]**, BE--88?K;1$mmEDL11}}::5A A A  B 1 Bs%A6D8C>9A=D6D7DDNr r!r"rNrrr>r>;s Brr>ceZdZdZy) RefsItemsViewczt|jj|jjSr)ziprAkeysr(rs rrNzRefsItemsView.__iter__Is)4==%%')=)=)?@@rNrOrPrrrRrRHsArrRchd}d}t|ddd|dddD]\}}|||zz }||z}|S)Nr)rT)idxsizesrKmultiss rravel_multi_indexr^MsS C DC"IuTrT{+1 q4x   JrceZdZdZedZedZ d dedfdZdZ d Z e d!d Z e d Zd"d Zd Ze ddZdZdZdZdZdZdZdZdZe dZdZd#dZd#dZdZdZdZ dZ!y)$LazyReferenceMapperaoThis interface can be used to read/write references from Parquet stores. It is not intended for other types of references. It can be used with Kerchunk's MultiZarrToZarr method to combine references into a parquet store. Examples of this use-case can be found here: https://fsspec.github.io/kerchunk/advanced.html?highlight=parquet#parquet-storagecddl}|Sr,)numpy)rnps rrczLazyReferenceMapper.np_s  rcddl}|Sr,)pandas)rpds rrfzLazyReferenceMapper.pdes  rNengine) fastparquetpyarrowc*||_i|_|xs |j|_||_||_||_|jdz|_|tjdn||_ ddl m }|jdk(r|d tdyy)a This instance will be writable, storing changes in memory until full partitions are accumulated or .flush() is called. To create an empty lazy store, use .create() Parameters ---------- root : str Root of parquet store fs : fsspec.AbstractFileSystem fsspec filesystem object, default is local filesystem. cache_size : int, default=128 Maximum size of LRU cache, where cache_size*record_size denotes the total number of references that can be loaded in memory at once. categorical_threshold : int Encode urls as pandas.Categorical to reduce memory footprint if the ratio of the number of unique urls to total number of refs for each variable is greater than or equal to this number. (default 10) engine: Literal["fastparquet","pyarrow"] Engine choice for reading parquet files. (default is "fastparquet") z/{field}/refs.{record}.parqNfiler) find_specriz)engine choice `pyarrow` is not installed.) rootrMout_root cat_threshrg cache_sizeurlfsspecr fsimportlib.utilrl ImportError)rrmrsrnrpcategorical_thresholdrgrls rrzLazyReferenceMapper.__init__ksB  -DII / $99<</1z&##F+r, ;;) # )(<(DIJ J)E #rc^|dvr|j|j|St|)N)rF record_sizerB)setup__dict__AttributeErrorritems r __getattr__zLazyReferenceMapper.__getattr__s. 9 9 JJL==& &T""rcPi_jjdjjdgjd<t j jd}|d_|d_tjfd}|_ y)N/ .zmetadatarxmetadata)maxsizecrjj||}tjjj |} j j|j}|jDcic]}|||j}}|Scc}w#t$rd}Y|SwxYw)zcached parquet file loader)rLrecord)rgN) rqformatioBytesIOrscat_filerf read_parquetrgcolumnsto_numpyOSError)rLrr2datadfcrefsrs r open_refsz,LazyReferenceMapper.setup..open_refss88??v?>D::dgg..t45D WW))$t{{)C57ZZ@2a5>>++@@KA K s$5B'B"B'"B'' B65B6) rFrsrjoinrmrCloadsrxrBrrpr)rmetrs` rryzLazyReferenceMapper.setups $(GG$4$4 HHdii. /%  L!jj\23}-Z 4?? +  , #rc ni|d}|'tjj|fi|xsi\}}|j|r|j |d|j |d|j dj|dgtj|jt||fi|S)aJMake empty parquet reference set First deletes the contents of the given directory, if it exists. Parameters ---------- root: str Directory to contain the output; will be created storage_options: dict | None For making the filesystem to use for writing is fs is None fs: FileSystem | None Filesystem for writing record_size: int Number of references per parquet file kwargs: passed to __init__ Returns ------- LazyReferenceMapper instance rrxT recursiveexist_okrr) rrcore url_to_fsexistsrmmakedirspiperrCrDrEr`)rmstorage_optionsrsrxkwargsrs rcreatezLazyReferenceMapper.creates,k: :{{,,TMo6KMHB 99T? EE$$E ' D4 ( $ -. 30F0F0HI"46v66rc>d|jD}t|S)zList top-level directoriesc3jK|]+}|jdr|jddd-yw).zrrWrN) startswithrsplit.0ps r z.LazyReferenceMapper.listdir..s,V1<.s5dT5 c3 K|]}|ywrrPrs rrz)LazyReferenceMapper.ls..s2dT2rF directoryrrtypesizerkc |dS)NrrP)r]s rz(LazyReferenceMapper.ls..s AfIrkeyrX)rstriprGrlstriprrBrFupdatesortedrIrCrDr_keys_in_fieldr/rJrT)rr2detailpathdashdirnamesr)rfothersrdirinfofileinforLrUrecsrecrecinfos rlszLazyReferenceMapper.lss{{3!%4#:2<<> ||H%#QXXh5G*G   !N5dnn52dkk2 <<)c(9K.K F d#f~%RVW$k1EWGW# !"4>>1 4>>$#78![[. H 'H,2EF F"nn Ld4& 0KT L $ Kt$qz/JtK L   t~~-JJt~~d34T*     ""5) U?<$t*, ,))%0!t_ c1v63r7 ;  '!!w X  MK  " s=-J -J% J8AJ3J! J!!J& ;J& AJ+-J0c||jvr|j|S||jvr0tj|j|j Sd|vs|j |r t ||jdd\}}|j|\}}}|jj||fij|d}|t |r|S|dk(ry |j||}gd} | D cgc]} | |vr|| |nd} } | d} | | S| d t d | dd ddgk(r| ddS| dd S#tttf$r} t || d} ~ wwxYwcc} w) zfGet the reference for one key Returns bytes, one-element list or three-element list. rrWFNrrr2offsetrrawrXz1This reference does not exist or has been deleted)rFrBrCrDrE_is_metaKeyErrorr_key_to_recordr-r ValueError TypeErrorFileNotFoundError)rrrL_rri chunk_sizemayberexcrr selectionrs r _load_one_keyz!LazyReferenceMapper._load_one_key!s $++ ;;s# # DNN "::dnnS1299; ; ^t}}S13- ::c1%q!%!4!4S!9J 488UC =N L 1_ )>>%0D4AHIAAIT!WR[47I Im ?J Q< NO O Qq>aV #Ra= !}I'89 )3-S ( )Js'EE-E* E%%E*ic4|jdd\}}|j|}t|dk(ry|jdDcgc] }t |}}t ||}||j z}||j z} || t|fScc}w)z3Details needed to construct a reference for one keyrrWr)rrr.)rrHrIsplitintr^rx) rrrLchunkrMr chunk_idx chunk_numberrrs rrz"LazyReferenceMapper._key_to_recordIszz#q) u++E2 { q %*[[%56SV6 6(K@ !1!11 D,, ,r3{+++ 7s Bc||jvr`|j|d}t|d|dDcgc]\}}tj||z }}}|xsdg|j|<|j|Scc}}w)z6The number of chunks along each axis for a given fieldz/.zarrayshapechunksrW)rMrBrTmathceil)rrLzarrayr]r size_ratios rrHz$LazyReferenceMapper._get_chunk_sizesVs (( (^^ugX$67F-0&BR-S%)Q !a% J'1&7QCD  U #&& s"A>c|j||}tt|j}t |dk(r d|DSt |dk(r|dSd|DS)z8The references for a given parquet file of a given fieldrc32K|]}t|ywrr/rts rrz7LazyReferenceMapper._generate_record..fs(DG(rWrc3LK|]}|dst|ddn|dyw)rNrrs rrz7LazyReferenceMapper._generate_record..ls)B!qtD2AK15Bs"$)rr'rTr(rI)rrLrrits r_generate_recordz$LazyReferenceMapper._generate_record`s`~~eV, #t{{}% & t9>(R( ( Y!^; CrB Brc#Kd}|j|D]}||z} tj||jz }t |D]}|j ||Ed{y7w)zJLoad all the references within a field by iterating over the parquet filesrWN)rHrrrxranger)rrLnrecchrs rrJz)LazyReferenceMapper._generate_all_recordsnss''. B BJD yy 0 001Dk rs rr(zLazyReferenceMapper.valuesws d##rct|Sr)rRrs ritemszLazyReferenceMapper.itemszs T""rct|Sr)idrs r__hash__zLazyReferenceMapper.__hash__}s $xrc$|j|Sr)r)rrs r __getitem__zLazyReferenceMapper.__getitem__s!!#&&rcd|vr|j|sy|jdd\}}|j|\}}}|jj ||fi}|||<t ||j k(r|j||yy||j|<tjt|tr|jn|} i|jj|i| |j|<yNrrW)rrrrFr7rIrxwriterCrr.bytesdecoderBr-) rrvaluerLrrr\rsubdict new_values r __setitem__zLazyReferenceMapper.__setitem__s #:dmmC0::c1-LE5..s3LFAqkk,,eV_bAGGAJ7|t/// 5&)0 %DKK  ",UE": I#OT^^%7%7R%@"NI"NDNN3 rc0|jdxsd|vS)Nrz/.zrrs rrzLazyReferenceMapper._is_metas~~d#3u|3rc||jvr|j|=y||jvr|j|=yd|vr|j|sy|jdd\}}|j |\}}}|jj ||fi}d||<t ||jk(r|j||yyd|j|<yr) rFrBrrrr7rIrxr)rrrLrrr\r s r __delitem__zLazyReferenceMapper.__delitem__s $++  C DNN "s#cz$--"4::c1-q#2237 1++00%"E! w<4#3#33JJuf-4$( C rc tddl}ddl}ddl}|j||f}d} t ||j kr |j ||} | r| d} | d} | d} | d} n|j|j |jd} |j|j d } |j|j d } |j|j |jd} |jD]\}}t|trot| jd k(r/|d| jj vr| j#|d} |d| |<t |d kDst|d | |<|d | |<|d| |<d| |<d| |<d| |<|j$j'|| |<|j)| | | | d d}|j*j-|j*j/xsd z |j0kDr|dj3d |d<ddd}ddg}|xs |j4d|d|d}|j6j9|xs |j4d|d|j:dk(rddi}n.|j:dk(rd||d}nt=|j:d|j>|f|j:|xstA|j6ddddd||jC|jjE||fy#t$rYwxYw)NrFr2rrrO)dtypeint64categoryrWr)copyrutf8)rr2rz/refs.z.parqTrriwrite_statisticsrh)statsobject_encoding has_nullsz not supportedrzstd)rgr compressionindex)# kerchunk.dfrbrerFrIrxrrfullnanzerosrr.r/r1r categoriesadd_categoriesr _proc_raw DataFramer2countnuniqueroastypernrsmkdirsrgNotImplementedError to_parquetgetattrclearpop)rrLrbase_urlrkerchunkrcrf partitionoriginalr9offsetsrZrawsjrrrrfndf_backend_kwargss rrzLazyReferenceMapper.writesJKK0  y>D,, , >>%8 V$Ex(GV$EE?DGGD,,bffCG@Ehht//wh?GHHT--WH=E774++RVV37?D ( 6GAt$% $ 2Qu{{'='==!00a9E7at9q=!%aGAJ#AwE!Ha aQ#++//5Q' 6*\\!     77==?bggoo/41 5 GF**:6BvJ")6:UO )DMM*!E7& F (3dmm4AeW=M ;;) #!3U ;  [[M )#2&!  & ^&DE E  ;;+9tww 148      (S  sL** L76L7ct|jD]-}t|ts|\}}|j ||||/t|jD]H}|dk7s d|vst j |jj||j|<J|j|jd}|jjt j|j|jd<|jjdj|xs |j dg|jd|j"j%y)zOutput any modified or deleted keys Parameters ---------- base_url: str Location of the output )r1rrrrrN)r/rFr.r0rrCrr0rBrxr/rDrErsrrrnr cache_clear)rr1rthingrLrkrs rflushzLazyReferenceMapper.flushs'$++& E%' % v %$3  dkk" CAL TQY$(JJt{{q/A$Bq! C >>$:J:JK $(JJsO$:$:$< L!  HHh/$--> ? KK % ""$rcd}|jD]@}|jdr|dz }|tj|j |z }B|t |j z }|td|jDz }|S)NrrrWc3BK|]}t|trdyw)rWN)r.r0)rrs rrz.LazyReferenceMapper.__len__..+sH1:a3GQHs) rGrrprodrHrIrBsumrF)rr(rLs r__len__zLazyReferenceMapper.__len__!s\\^ AE$ 4#8#8#?@@  A T^^$$ H HHH rc# Kt|j}|j|j|D]}t |t s||j D]!}|j|D] }||vs| #ywr)rrBrrFr.r1rGr)rmetasbitrLr=s rrNzLazyReferenceMapper.__iter__.sDNN# T[[! C#s#  \\^ E((/ 9G  sAB 1B; BcF |j|y#t$rYywxYw)NTF)rrr|s r __contains__z LazyReferenceMapper.__contains__;s*    t $  s   c #K|j|}t|dk(r|dzytjd|D}|D]2}|dzdj |Dcgc] }t |c}z4ycc}ww)z|List key names in given field Produces strings like "field/x.y" appropriate from the chunking of the array rr@Nc32K|]}t|ywr)r)rr\s rrz5LazyReferenceMapper._keys_in_field..Ks"A58"Arrr)rHrI itertoolsproductrr1)rrLrMindsindrs rrz"LazyReferenceMapper._keys_in_fieldBs ++E2 { q $,    "A["AB @C#+#)>Q#a&)> ?? ? @)>sABA; -B)NN rh)NNi')rTNN)"r r!r"__doc__propertyrcrfrrr~ry staticmethodrrrGrrrrHrrJr(rrrr rrrr>rCrNrHrrPrrr`r`VsY   4A.K01.K`##077<[ @"D&Pt_ , ,' C<$#'O 44("T)l!%F   @rr`ceZdZdZdZ dfd Zd dZd dZd dZdZ dZ e fd Z d!d Z d"d Zd#d Zd Zd#dZdZdZdZd$dZd%dZdZdZdZd%dZd&fd ZdZdZdZd'dZd'dZ dZ!xZ"S)(ReferenceFileSystemaView byte ranges of some other file as a file system Initial version: single file system target, which must support async, and must allow start and end args in _cat_file. Later versions may allow multiple arbitrary URLs for the targets. This FileSystem is read-only. It is designed to be used with async targets (for now). This FileSystem only allows whole-file access, no ``open``. We do not get original file details from the target FS. Configuration is by passing a dict of references at init, or a URL to a JSON file containing the same; this dict can also contain concrete data for some set of paths. Reference dict format: {path0: bytes_data, path1: (target_url, offset, size)} https://github.com/fsspec/kerchunk/blob/main/README.md rc t|d i|||_| |_| |_i|_i|_i|_| |_| |_ t|trtd i|xs|xsid|i}tjj|fi|\}}|j!|r^tj"|dfi|5}t$j'd|t)j*|}ddd|j-| n#-R[[%"@ Abkk )+DHH_ %  "~~,,. ,C=%C$kk88= ! 8#HG1E2GB)+DHHX&  ,  "--/ C=%Cc4(SV"(++"<"OF0D"FB(*DHH_ %1z&1HHNN$ 8DAq<<4Q7  8(( s;,O!AO!Oc |j|}tjd| |j|}t |tr|j}t |trGtjd|d|jdrtj|dd}|ddfSt|dk(r%tjd|d||d}||}}nV|\}} } tjd|d |d | d | | | z} ||dk\r| |z}n| |z}n| }||dk\r| |z}n| |z}n| }| |j}|||fS#t$r}t ||d}~wwxYw) Nzcat: z Reference: z , type bytesbase64:rWz, whole file => rz => z , offset z, size )_strip_protocolrcdebugr3rrr.r1rErrbase64 b64decoderIr) rr2startendpartrrqstart1end1start0rend0s r _cat_commonzReferenceFileSystem._cat_commons##D) uTF^$ 3??4(D dC ;;=D dE " LL;tfL9 :z*''QR1t# # t9> LL;tf,s %rcK|j|rtj|dS|j|d{}t |d5}|j |dddy7+#1swYyxYwwNTrwb)isdirosrrr r)rrpathlpathrrrs r _get_filezReferenceFileSystem._get_fileBsj ::e ;;ut4 4^^E** %  ! GGDM  +  s(=A9A+A9A-" A9-A62A9c |j|rtj|dS|j|fi|}|j t |t |r|j|n&t|d5}|j|ddd|jt |y#1swY$xYwr) rrrrset_sizerIr rr absolute_update)rrrcallbackrrrs rget_filezReferenceFileSystem.get_fileIs ::e ;;ut4 4t}}U-f-#d)$ e  KK eT" a    T+  s ?B44B=c |r|jd|j||}tjdd}t ||}|r2|j |Dcgc]}|j |r|c}}n|j |}t||D]\} } | |vs |j| || !ycc}w)NrrrkT) auto_mkdir) r expand_pathrrr rcatrrTr) rrrrrrstargetsrrremotelocals rr-zReferenceFileSystem.getUs  GGBK  ) <   v$ 7eU+ 88C1TZZ]QCDD88E?D 0 2MFE~ UDL1 2Ds B>2B>c t|tr|rtt|tr|st d|Drtt ||j }i}|jD]\}}|j|} ggggf\} } } } |D]t} |j|\}}}t|ttfs0 | j|| j|| j|| j|vg}g}g}g}t}t| | | | D]u\}}}}t|tr|||<|!|j||j||j||j||j|wt| | | | D]S\}}}}| ||vs|j||j||j||j|Ut!t|t|t|d|j"|j$\}}}| j'|||}t| | | | D]\}}}}||vr t||||D]i\}}}}||k(r#||t|t(r|||<'|||||<0||k(s6||k\s<||ksBt|t(r|||<X|||z ||z xsd||<k|j+jD]\\} }!t|!t(s| |j vs&|| }"t-| |j | }#|"|#_|dk(r|#|dk7sX|#|| <^t1|dk(rt|trd|vr t3|S|S#t$r}|dk(r|dk7r|||<Yd}~/d}~wwxYw)Nc3$K|]}d|v yw)*NrPrs rrz*ReferenceFileSystem.cat..gs7OQq7OsraiseomitT)sortr_r`rWr)r.r1r,r/anyr<r3rr]rrr8rrrTaddrr_r` cat_rangesrrr __cause__rIr*)$rr2ron_errorr proto_dictr:protor9rsurlsstartsends valid_pathsrur]rerrurls2starts2ends2paths2 whole_files new_paths new_startsnew_ends bytes_outrcnsnebr=vexnew_exs$ rrzReferenceFileSystem.catds dC Y% % dD !yC7O$7O4O% %%dDOO< &,,.I BLE5%B.0"b"n +D&$  **"..q1GAq!%a%6 7KKNMM!$KKN&&q)# *(EGEF%K!$kB % 1aa'CFY OOA&LLONN1%LLOMM!$ %"$kB % 1a=Qk%9LLONN1%LLOMM!$  %/BU W U  .. / +Iz8 iXFI"$kB B 1a8%(J)%T BMBBQwBJ"*%a3%&CF%&qVCFqQ"Wb%a3%&CF%&q2vR0@D%ACF B ByI BVHHJ$$& $DAq!Y'A,@V.q$//!2DE#% w& L'#CF $ s8q=Zc2s$#;  [)%7*6)!$A %s+N O'N==Oc|jdd}||j|y|dk(r|j||ytd|)NversionrW)rZz Unknown reference spec version: )r-_process_references0_process_references1r)rr3rZverss rrfz'ReferenceFileSystem._process_referencessS~~i. <  % %j 1 QY  % %jEW % X?vFG Grc t|trH|jDcic]-\}}|t|trtj|n|/}}}||_ycc}}w)z&Make reference dict for Spec Version 0N)r.rarrCrDr3)rr3rrKs rrz(ReferenceFileSystem._process_references0s^ j$ ' !+ 0 0 2C 3(=TZZ_3FJ% s2A!cjr jrddli_j |j dit dfd}|j dijD]/\}}t|trF|jdr%tj|ddj|<|j|<]t|tr#tj|j|<jr|d}d|vrQjr=|j!dd j!d d j"dij}n||}t%|d k(r|gn ||d |d gj|<!|j|<2jj'j)|j dgy)Nrr\icZj|jdijSNrP)Templaterenderr\)rjinja2rs r _render_jinjaz?ReferenceFileSystem._process_references1.._render_jinjas&,6??1%,,>t~~> >rrzbase64:rv{{{z}}}rWrgenrP)r[r\rr3_process_templatesr-rrr.r1rryrzrarCrDreplacerrIr _process_gen)rr3rZrr=rrrs` @rrz(ReferenceFileSystem._process_references1s$$   {B ?@ 4 ?  ?NN62.446 'DAq!S!<< *)/)9)9!AB%)@DOOA&%&"At$%)ZZ]"aD19,,$AIIdC0$WT3/#V6&*nn6 *!,,/FaKaSa1qt_"%&") '* t00r1JKLrci|_|j|j|j|jD]1\}}d|vrddl|ffd |j|<#||j|<3y)NrrcFj|jdi|Sr)rr)temprrs rrz8ReferenceFileSystem._process_templates..s*>V__>&>#!>#r)r\rZrrr)rtmpr=rrs @rrz&ReferenceFileSystem._process_templatessn  " " . JJt.. /IIK &DAqqy01%#q!%&q! &rc^ i}|D]}|djDcic]H\}}|t|tr|n/t|j dd|d|j ddJc}} fdt j jD}|D]}ddl}|j|djd i||j} |j|d jd i||j} d |vrd |vr{t|j|d jd i||j} t|j|d jd i||j} | | | g|| <d |vd |vz r td | g|| <|Scc}}w)N dimensionsr{rstopsteprWc3dK|]'}ttj|)ywr)rarTrU)rr( dimensions rrz3ReferenceFileSystem._process_gen..s,S)623s-0rrqrlengthz^Both 'offset' and 'length' are required for a reference generator entry if either is provided.rP)rr.r/rr-rKrLr(rrrr\rr)rgensr:rr=rproductsprrrrqrrrs @rrz ReferenceFileSystem._process_gen s %C -335  Aq !!T*quuWa0!F)QUU61=MNOI'//1A1A1CDH %8fooc%j188P2PP8fooc%j188P2PPO(c/ =H 6==UUdnnUF!=H 6==UUdnnUF!$VV4CH#o(c/:$K !$uCH' % %B AsA F)cdgi|_|jj}|D]<\}}t|tt fr t |}nt |dk(rd}n|\}}}d|vr|jdddnd}|}|g}|rA||jvr3|j|}|j||r||jvr3|jt||ddD]V\} } | |jvsJ| |jvsJ|j| j| dddg|j| <X|j|j|d|d?y)NrrWrrrrrk) dircacher3rr.rr1rIr_parentr8reverserT) rrr2r}rrparpar0subdirsparentchilds r_dircache_from_itemsz(ReferenceFileSystem._dircache_from_items0shR __ " " $ TJD$$ -4yTa! 1d,/4K$++c1%a(RCDfG4t}}4||D)t$4t}}4 OO !$Wgabk!: * DMM111... f%,,"KC(* e$ * MM#  % %tVT&R S3 Trc N|j|}tj|Sr)rrr)rr2mode block_size cache_optionsrrs r_openzReferenceFileSystem._openNs}}T"zz$rc z|j|}t|jtr |jj ||S|js|j|j|}| t ||r|S|Dcgc]}|d c}S#t $rYnwxYwt d|dcc}w)N'z' is not a known keyr) rwr.r3r`rrrrr_ls_from_cache)rr2rrr:os rrzReferenceFileSystem.lsRs##D) doo': ; ))$77}}  % % '!!$' ;#D) ) J#&'a& ''  #av-A$BC C(sB B8 B&%B&c J|j|xs|j|Sr)rrbrr2rs rrzReferenceFileSystem.existscszz$44;;t#44rc|jr|jvSt|jtr|jj vSt fd|jDS)Nc3FK|]}|jdyw)rNr)rrr2s rrz,ReferenceFileSystem.isdir..ns IAq||tfAJ/Is!)rr.r3r`rGrrr2s `rrzReferenceFileSystem.isdirfsW ==4==( ( )< =4??2244 4III Irc||jvSrrrs rrbzReferenceFileSystem.isfilepst&&rc2K|j||fi|Swr)r)rr2rrs r_lszReferenceFileSystem._lssstwwtV.v..rc P|rt|f|||d|Sr0|jtfd|jD}nt|j}|r?|j s|j |Dcic]}||j|dc}S|Scc}w)N)maxdepthwithdirsrc3FK|]}|js|ywrr)rr=r2s rrz+ReferenceFileSystem.find..}sHQQ\\$5GqHs!!r)rfindrwrr3rrr) rr2rrrrrr=rs ` rrzReferenceFileSystem.findvs 7<'(6MS  ''-DH$//HHAt'A ==))+:;GD!"hhtn11$//$2G2JKDGFOAw9s 8 C3C3c,K|j|Swr)rdrs r_infozReferenceFileSystem._infosyyscxK|jj|d|jjywr)r3r0rr/rs r_rm_filezReferenceFileSystem._rm_files/  $  s8:cK|dk(r|j|rt||j|<|jj yw)Nr)rFileExistsErrorr3rr/)rr2rrrs r _pipe_filezReferenceFileSystem._pipe_files= 8  D 1! ! $ sAA cK|dk(r|j|rtt|d5}|j|j|<ddd|j j y#1swY$xYww)NrrX)rrr readr3rr/)rrrrrrs r _put_filezReferenceFileSystem._put_filesc 8  E 2! ! %  .!%&VVXDOOE " .  . .s)A8A, #A8,A51A8c i}|jjD]0\}}t|tr |j d||<,|||<2tj|dfi|5}|jtjd|djdddy#t $r,dt j|zj ||<YwxYw#1swYyxYw)z+Write modified references into new locationasciirurrW)rrN)r3rr.rrUnicodeDecodeErrorry b64encoderrr rrCrDrE)rrqrr:r=rrs r save_jsonzReferenceFileSystem.save_jsonsOO))+ DAq!U#IXXg.CFA [[d 6o 6 F! GGDJJ1c:;BBD E F F *I(6+;+;A+>>FFHCFI F FsB&'6C&2CCC') NNNNNNNNTii@BrOrQ)F)Frr)rXNN)T)NFF) overwrite)#r r!r"rRr;rrrrrrrrr-rrfrrrrrrrrrrbrrrdr r rrrr#r$s@rrVrVPs H  Z8x(!T BB&/? , 2`DH %MB &#JT< ("5J'/"*  FrrV)/ry collectionsrrKloggingrr functoolsrrtypingrr fsspec.corerrujsonrCru fsspec.asynrfsspec.callbacksrr r r #fsspec.implementations.asyn_wrapperr fsspec.utilsr rr getLoggerrc RuntimeErrorrr*r5r<abc ValuesViewr> ItemsViewrRr^MutableMappingr`rVrPrrr*s   ) (-88FEE   - .TLT"?  B[__// BAKOO--A w@+//88w@tp F/p FA sC22 D?D