a h3@sddlZddlZddlZddlmZddlmZmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZddlmZmZddlZdd lmZmZmZdd lmZdd l m!Z!d d l"m#Z#d dl$m%Z%m&Z&d dl'm(Z(m)Z)m*Z*d dl+m,Z,m-Z-d dl.m/Z/m0Z0m1Z1d dl2m3Z3m4Z4m5Z5e6dej7Z8eGdddZ9Gdddej:Z;GdddejGdddejZ)d?d@Z*Z+S)K HfFileSystema8 Access a remote Hugging Face Hub repository as if were a local file system. [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. Args: token (`str` or `bool`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . Usage: ```python >>> from huggingface_hub import HfFileSystem >>> fs = HfFileSystem() >>> # List files >>> fs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> fs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with fs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` r0ZhfNendpointtokencs<tj|i||ptj|_||_t||d|_i|_dS)NrD) super__init__rZENDPOINTrErFr _api_repo_and_revision_exists_cache)r8rErFargsZstorage_options __class__r:r;rHfs  zHfFileSystem.__init__)r'r(r)r/c Cs|||f|jvrz|jj|||tjdWnttfyz}z4d|f|j|||f<d|f|j||df<WYd}~npd}~0ty}z0d|f|j|||f<d|j||df<WYd}~n*d}~00d|j|||f<d|j||df<|j|||fS)N)r)r'timeoutF)TN)rJrIZ repo_inforZHF_HUB_ETAG_TIMEOUTrr#r)r8r'r(r)er:r:r;_repo_and_revision_existxs (&z%HfFileSystem._repo_and_revision_exist)pathr)r/cCs|ttttttddd}||}|s6tdnN|dddtjvr~d|vrbtd|dd\}}tj|}ntj }| ddkrd|vr>|dd\}}d|vrt |}|d ur|d | fvrt d |d}| }n|dd\}}nd }|t||}||||\} } | st|| nd }d|dd d } d|dd d } |dd} d|ddd }| }| }||||\} } | sVt| ttfr| }|}||||\} }| st|| n t|| nV|}d }d|vr2|dd\}}|t||}nd }||||\} }| sVtd|d urd|ntj}t|||||d S) a Resolve a Hugging Face file system path into its components. Args: path (`str`): Path to resolve. revision (`str`, *optional*): The revision of the repo to resolve. Defaults to the revision specified in the path. Returns: [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. Raises: `ValueError`: If path contains conflicting revision information. `NotImplementedError`: If trying to list repositories. )revision_in_pathr)r/cSs8|dur0|dur4||kr4td|d|dn|}|S)NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.) ValueError)rRr)r:r:r;%_align_revision_in_path_with_revisionszHHfFileSystem.resolve_path.._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r2rrr1Nr0)r-)r rAZ_strip_protocolNotImplementedErrorsplitrr3valuesZREPO_TYPES_MAPPINGZREPO_TYPE_MODELcountSPECIAL_REFS_REVISION_REGEXsearchgroupsublstriprrP_raise_file_not_foundjoin isinstancerr#r6r&)r8rQr)rTr'r(rRmatchr*Zrepo_and_revision_existerrZrepo_id_with_namespaceZpath_in_repo_with_namespaceZrepo_id_without_namespaceZpath_in_repo_without_namespace_r:r:r; resolve_pathsj            zHfFileSystem.resolve_path)rQr/cCs|s|j|jnh||}|}|rJ|j|d||}q,|js|j|j|j dfd|j|j|j |j fddS)ac Clear the cache for a given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). Args: path (`str`, *optional*): Path to clear from cache. If not provided, clear the entire cache. N) dircacheclearrJrer<pop_parentr*r'r(r))r8rQ resolved_pathr:r:r;invalidate_caches     zHfFileSystem.invalidate_cacherbHfFileSystemFile)rQmoder) block_sizer/cKsPd|vrtd|dkr2t||f|||d|St||f|||d|SdS)Naz/Appending to remote files is not yet supported.r)rnr)ro)rVHfFileSystemStreamFilerm)r8rQrnr)rokwargsr:r:r;_opens zHfFileSystem._openc KsR|j||d}|jj|j|j|j|j|j|d|dd|j | ddS)Nr)commit_messagecommit_description)r*r(rFr'r)rurvrQ) rerIZ delete_filer*r(rFr'r)r4rkr<)r8rQr)rrrjr:r:r;_rms zHfFileSystem._rmF)rQ recursivemaxdepthr)r/c sj||d}j||||d}fdd|D}dd|D} d|d} | |rVdnd 7} | |d urrd |dnd 7} jj|j|jj| |j|d | |d dj | dd S)a Delete files from a repository. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. rt)ryrzr)cs"g|]}|s|jqSr:)isdirrer*).0rQr8r:r; Bz#HfFileSystem.rm..cSsg|]}t|dqS))r*)r)r|r*r:r:r;r~CrzDelete  z recursively r0Nz up to depth rurv)r(r'rF operationsr)rurvrw) reZ expand_pathrI create_commitr(r'rFr)r4rkr<) r8rQryrzr)rrrjpathsZ paths_in_reporrur:r}r;rm"s"   zHfFileSystem.rmT)rQdetailrefreshr)r/c s|j|d}|z|jf||d|}Wnhty|jsPtd|j|f||d|}fdd|D}t|dkrtdYn0|r|Sdd|DS)a List the contents of a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `List[Union[str, Dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). rt)rr)Ncsg|]}|dkr|qSnamer:r|orwr:r;r~yrz#HfFileSystem.ls..rcSsg|] }|dqSrr:rr:r:r;r~|r)rer<_ls_treerr*r_rilen)r8rQrrr)rrrjoutr:rwr;lsSs   zHfFileSystem.ls)rQryrr) expand_infoc s`j||d}|}t|j|j|jd|jd}g}|jvr|sj|} || g} |rt dd| D} | r| } | djvr| | dqxj| d} || | dd| Dqxg} |rfdd|D} |r| s|r\| r\t j | | }|d s4||ks4|t| | vr>|d n|fd d|D}jD]$}|d r`j|dq`jd|j|d ||d njj|j|j|||j|jd }|D]}t|tr|d |j |jd|j|j|j|jd}n|d |j dd|j |jd}|d}j!|g || |q|S)Nrtr0r*r-cSsg|]}|ddkr|qStype directoryr:r| path_infor:r:r;r~rz)HfFileSystem._ls_tree..rcSsg|]}|ddkr|qSrr:rr:r:r;r~rcs&g|]}|ddur|dqS) last_commitNr)rirr}r:r;r~rr2cs"g|]}|dds|qS)rr2) startswithr) common_pathr:r;r~rT)ryrr)r)ryexpandr)r'filersizerblob_idlfsrsecurityrrrrrtree_idr)"rer<r&r'r(r)r-rfextendrpopleftappendosrQ commonprefixendswithrr5rirrhrrIZlist_repo_treer*rar"rrrrrr setdefault)r8rQryrr)rrj root_pathrZcached_path_infosZdirs_not_in_dircacheZ dirs_to_visitdir_infoZdirs_not_expanded common_prefixZ cached_pathtreerZcache_path_info parent_pathr:)rr8r;r~s            zHfFileSystem._ls_treec/s<|j||dd}tj|g|Ri|EdHdS)a Return all files below the given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). Args: path (`str`): Root path to list files from. Returns: `Iterator[Tuple[str, List[str], List[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. r)rtN)rer4r<rGwalk)r8rQrKrrrLr:r;rs zHfFileSystem.walkc s,|j||dd}tj|fi|S)ah Find files by glob-matching. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). Args: path (`str`): Path pattern to match. Returns: `List[str]`: List of paths matching the pattern. r)rt)rer4r<rGglobr8rQrrrLr:r;rs zHfFileSystem.glob)rQrzwithdirsrrr)r/c  s|r$tj|f|||||d|S|j||d}|}z |j|fd||jd|Wn>ty|j|fd|i|ddkr|iiniYnV0|sdd Dn2|j|fd|ji|} | dd kr| gnd d Dt} |s| Sfd d | DSdS)a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[List[str], Dict[str, Dict[str, Any]]]`: List of paths or dict of file information. )rzrrrr)rtT)ryrr)r)rrcSsg|]}|ddkr|qSrr:rr:r:r;r~6rz%HfFileSystem.find..rcSsi|]}|d|qSrr:rr:r:r; ;rz%HfFileSystem.find..csi|]}||qSr:r:)r|rrr:r;r@rN) rGfindrer<rr)rinfosorted) r8rQrzrrrr)rrrjrnamesrLrr;rs2      zHfFileSystem.find)path1path2r)r/c Ks,|j||d}|j||d}|j|jko2|j|jk}|rd|d|}|jj|j|j|j|d||ddt|j|j|jdgdn~|j |d |jd} | } Wd n1s0Yd|d|}|jj | |j|j|j |j|j|d||dd |j |d |j |d d S) a Copy a file within or between repositories. Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rtzCopy z to rurvr0)Zsrc_path_in_repor*Z src_revision)r(r'r)rurvrrlNZpath_or_fileobjr*r(rFr'r)rurvrw)rer'r(rIrr)r4rr*openread upload_filerFrkr<) r8rrr)rrZresolved_path1Zresolved_path2Z same_reporufcontentr:r:r;cp_fileBsD  &  zHfFileSystem.cp_filecKs*|j|fii|ddi}|ddS)ai Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last commit date of the file. rTrdater)r8rQrrrr:r:r;modified}s zHfFileSystem.modified)rQrr)r/c  s|j|d}||dd}|js|dddd|rx|jj|j|j|jdd }idt |j |j |j d d nHd| }|s||jvr||||jvrڇfd d |j|D} | std| d|sdus|rĈrĈddur|jj|j|j||j|jd} | s2td| d} t|j|j|jd|jd} t| tr| d| j| jd| j| j| j| jdn| d| jdd| j| jd|sćfdddDdusJS)a Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance. Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `Dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rtrFrrN)rrrr)r'r))oidtitler)rrcsg|]}|dkr|qSrr:rrwr:r;r~rz%HfFileSystem.info..r)rr)r'r0rr2rrrcsi|]}||qSr:r:)r|krr:r;rrz%HfFileSystem.info..)rrr)rer<r4r*rIZlist_repo_commitsr(r'r)r! commit_idrZ created_atrirfrr_Zget_paths_infor&r-rar"rQrrrrrr) r8rQrr)rrrjrrrZout1Z paths_inforrr:)rrQr;rs       *     zHfFileSystem.infocKsBz.|ddr|||j|fi|WdSYdS0dS)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFTN)r4rkrrr:r:r;existss  zHfFileSystem.existscCs.z||ddkWSty(YdS0dS)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. rrFN)rOSErrorr8rQr:r:r;r{s  zHfFileSystem.isdircCs(z||ddkWSYdS0dS)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. rrFNrrr:r:r;isfile s zHfFileSystem.isfilecCsB||}t|j|j|j|j|jd}||r>|ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. )r'r)rEz /resolve/z/tree/r) rerr(r*r'r)rEr{replace)r8rQrjurlr:r:r;r2s  zHfFileSystem.urlr.c sd|d}t|dh}t|ttfr6t|dkrRtj||f||d|St |r`|}n| |r|t j |dddSt|t tfrt j t j|ddd}|durt|d}d}|} |j||d } |j||d d } || zbtt| j| j| j| j|jd ||| d|jt|tr,|jndd | | W|r`|!n|r^|!0dS) a^ Copy single remote file to local. Note: When possible, use `HfApi.hf_hub_download()` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. r)r)callbackoutfileT)exist_okNFwbrtrr(r)filenamer'rE)r temp_fileZdisplayed_filename expected_sizeZ resume_sizeheadersZ _tqdm_bar)"r4setkeysrarrrrGget_filerr{rmakedirsrArrQdirnamertellrerZset_sizerrr(r)r*r'rErI_build_hf_headersZtqdmseekclose) r8ZrpathZlpathrrrrr)Zunhandled_kwargsZ close_fileZ initial_posZresolve_remote_pathrrLr:r;rIsP      zHfFileSystem.get_filecCs tddS)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.NrVr}r:r:r; transactions zHfFileSystem.transactioncCs tddS)z@Begin write transaction for deferring files, non-context versionrNrr}r:r:r;start_transactionszHfFileSystem.start_transaction)N)N)rlNN)N)FNN)TFN)FFNF)NFFFN)N)FN),r=r>r?r@Z root_markerprotocolr rArboolrHr ExceptionrPr&rerkintrsrxrr r rrrr rrrrrrrrr{rrrrpropertyrr __classcell__r:r:rLr;rC:s(   _  2  . h*<;"eG rCcseZdZdeeeedfdd ZfddZeee ddd Z dd d d Z de ddddZ dfdd Zed ddZZS)rmN)fsrQr)c sz|j||d|_WnFtyZ}z.d|ddvrDt|d|WYd}~n d}~00tj||jfi||dS)Nrtwrnr0B. Make sure the repository and revision exist before writing data.)rerjFileNotFoundErrorr4rGrHr<)r8rrQr)rrrOrLr:r;rHszHfFileSystemFile.__init__cst|dsdStSNrjhasattrrG__del__r}rLr:r;rs zHfFileSystemFile.__del__)startendr/cCsldd|d|di|jj}t|jj|jj|jj|jj|jj d}t d||dt j d}t ||jS) Nrangezbytes=-rrGETiiii)rretry_on_status_codesrN)rrIrrrjr(r)r*r'rEr%rHF_HUB_DOWNLOAD_TIMEOUTr$r)r8rrrrrr:r:r; _fetch_ranges( zHfFileSystemFile._fetch_ranger.cCstjddd|_dS)Nzhffs-F)prefixdelete)tempfileNamedTemporaryFilerr}r:r:r;_initiate_uploadsz!HfFileSystemFile._initiate_uploadF)finalr/c Cs|jd|j}|j||r|j|jjj|jj |j j |j j |jj |j j|j j|jd|jddt|jj |jj|j ddS)Nrrurvrrw)bufferrrrwriterrrIrrrjr*r(rFr'r)rrr4rremoverkr<)r8rblockr:r:r; _upload_chunks&       zHfFileSystemFile._upload_chunkrcs|jdkrv|dus|dkrv|jdkrv|jj|jddd.}|}|jt|7_|WdS1sl0Yt|S)a)Read remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems and if `hf_transfer` is not enabled, the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rlNrr)ro)rnlocrrrQrrrG)r8lengthrrrLr:r;rs $"zHfFileSystemFile.readcCs|j|jSNrrrQr}r:r:r;rszHfFileSystemFile.url)N)F)r)r=r>r?rCrAr rHrrbytesrrrrrrrr:r:rLr;rms rmcsveZdZdeeeeeeedfdd Zdeedd d Zded d dZ edddZ fddZ ddZ Z S)rqrlNrnone)rrQrnr)ro cache_typec s|dkrtd||dkr,td|d|vrDtd|dz|j||d|_WnDty}z,d|d d vrt|d |WYd}~n d}~00|jdd |_tj||jf|||d |d|_ |dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got r z?HfFileSystemStreamFile only supports cache_type='none' but got rz;HfFileSystemStreamFile only supports reading but got mode=''rtrnr0r)rrrnror) rSrerjrr4r<detailsrGrHresponse) r8rrQrnr)rorrrrOrLr:r;rHs2  zHfFileSystemStreamFile.__init__)rwhencecCs6|dkr|dkrdS||jkr*|dkr*dStddS)NrrzCannot seek streaming HF file)rrS)r8rrr:r:r;rs zHfFileSystemStreamFile.seekr)r c CsF|dkr|fnd}|jdurlt|jj|jj|jj|jj|jjd}t d||jj ddt j d|_t|jz|jjj|}Wnty.|jt|jj|jj|jj|jj|jjd}t d|dd |ji|jj ddt j d|_t|jz|jjj|}Wn ty(|jYn0Yn0|jt|7_|S) Nrr:rrrT)rrstreamrNRangez bytes=%d-)rrrjr(r)r*r'rrEr%rIrrrr$rawrrrrr)r8r Z read_argsrrr:r:r;rsZ      zHfFileSystemStreamFile.readr.cCs|j|jSr r r}r:r:r;rMszHfFileSystemStreamFile.urlcst|dsdStSrrr}rLr:r;rPs zHfFileSystemStreamFile.__del__cCst|j|j|j|j|jjffSr )reopenrrQrn blocksizecacherr}r:r:r; __reduce__Vsz!HfFileSystemStreamFile.__reduce__)rlNrr )r)r)r=r>r?rCrAr rrHrrrrrrr:r:rLr;rqs"1 rq)r)r/cCst|r|St|Sr )rZrb safe_quotertr:r:r;r7Zsr7)sr/cCs t|ddS)Nr0)safe)r)rr:r:r;r^sr)rQrcr/cCsR|}t|tr|d}n*t|tr0|d}nt|trD|d}t||dS)Nz (repository not found)z (revision not found)z (invalid repository id))rarrr#r)rQrcmsgr:r:r;r_bs      r_rrQrnrorcCs|j||||dS)Nr)rrr:r:r;rmsr)Grrer collectionsr dataclassesrrr itertoolsrpathlibrtypingrr r r r r rr urllib.parserrZfsspecZfsspec.callbacksrrrZ fsspec.utilsrrequestsrr0rZ _commit_apirrerrorsrrrZ file_downloadrrZhf_apir r!r"utilsr#r$r%compileVERBOSErZr&ZAbstractFileSystemrCspecZAbstractBufferedFilermrqrAr7rrr_rrr:r:r:r;sL    (    nSe