
    rh                         d dl Z d dlZd dlmZmZ d dlmZ d dlmZ d dl	m
Z
mZmZ d dlmZ d dlmZ d dlmZmZmZmZ erd d	lmZ d
dgZ G d de      Z G d d
e      Z G d de      Zy)    N)	GeneratorSequence)contextmanager)Path)OptionalTYPE_CHECKINGUnion)	url_to_fs)StreamTransformExtension)FileSystemBaseFileSystemReaderFileSystemWriterSerializationFormat)AbstractFileSystemFsspecWriterFsspecReaderc            
       R   e Zd ZddZedeeej                  f   dede	e
j                  ddf   fd       Zdeeej                  f   dedeeej                  f   fdZdeeej                  f   deeej                  f   fd	Zdeeej                  f   d
eeej                  f   ddfdZdeeej                  f   ddfdZedeeej                  f   defd       Zdeeej                  f   defdZdeeej                  f   ddfdZdeeej                  f   dee   fdZy)
FileSystemreturnNc                     d | _         y N)fs)selfs    /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/_fsspec_filesystem.py__init__zFileSystem.__init__!   s	    04    pathmodec              #   *  K   | j                   J t        j                  |      }| j                   j                  |      5 }	 | 	 d d d        y #  t	        fddD              r	 | j                  |        #  Y  xY w xY w# 1 sw Y   y xY ww)Nc              3   &   K   | ]  }|v  
 y wr    ).0chr   s     r   	<genexpr>z+FileSystem.create_stream.<locals>.<genexpr>2   s     2brTz2s   zw+a)r   osfspathopenanyrm_file)r   r   r   streams     ` r   create_streamzFileSystem.create_stream$   s      ww"""yy
 WW\\$% 				 		2E22T* 		 		sH   A BBA
	BB*A<;B<B >BBBBsuffixc                 B    t         j                  j                  ||      S r   )r%   r   join)r   r   r,   s      r   concat_pathzFileSystem.concat_path9   s     ww||D&))r   c                 .    t        |fi |\  | _        }|S r   )r
   r   )r   r   kwargs_s       r   	init_pathzFileSystem.init_path>   s     t.v.
r   new_pathc                 <    | j                   j                  ||       y r   )r   rename)r   r   r4   s      r   r6   zFileSystem.renameD   s     	tX&r   c                 >    | j                   j                  |d       y )NT)exist_ok)r   makedirsr   r   s     r   mkdirzFileSystem.mkdirI   s    -r   checkpoint_idc                 \    t        |t              ry	 t        |       y# t        $ r Y yw xY w)NFT)
isinstancer   r
   
ValueErrorclsr<   s     r   validate_checkpoint_idz!FileSystem.validate_checkpoint_idL   s5    mT*	m$   		s    	++c                 8    | j                   j                  |      S r   )r   existsr:   s     r   rD   zFileSystem.existsX   s    ww~~d##r   c                 :    | j                   j                  |       y r   )r   rmr:   s     r   r)   zFileSystem.rm_file[   s    

4r   c                 <    | j                   j                  |d      S )NF)detail)r   lsr:   s     r   rI   zFileSystem.ls^   s     wwzz$uz--r   )r   N)__name__
__module____qualname__r   r   r	   strr%   PathLiker   ioIOBaser+   r/   r3   r6   r;   classmethodboolrB   rD   r)   listrI   r!   r   r   r   r       s   5 #r{{*+36	299dD(	) (*#r{{*+*58*	sBKK	 *
#r{{*+	sBKK	 '#r{{*+'7<S"++=M7N'	'
.%R[[ 01 .d . 	5bkk9I3J 	t 	 	$5bkk!12 $t $E#r{{"23  .uS"++-. .49 .r   r   c                        e Zd ZdZddddddej
                  fdeeej                  f   de
de
d	ed
ede
deee      deddf fdZedeeej                  f   de
fd       Z xZS )r   a`  
    Basic implementation of StorageWriter using FFspec.

    This implementation makes the following assumptions and simplifications:

    * The checkpoint path is an empty or non-existing directory.
    * File creation is atomic

    The checkpoint consist of one file per write request plus
    a `.metadata` file with the serialized metadata.

    T   i Nr   single_file_per_rank
sync_filesthread_countper_thread_copy_ahead	overwrite_extensionsserialization_formatr   c	           
          t         
|   ||||||||       t               | _         | j                  j                  |fi |	| _        y)a=  
        Initialize the writer pointing to `path`.

        Args:
            path: directory where the checkpoint will be written to.
            single_file_per_rank: Produce one file per rank instead of one file per tensor/blob. Default to True.
            sync_files : force files to be synced to permanent storage. Default to True.
            thread_count: Number of IO threads to use to write. Default to 1.
            per_thread_copy_ahead: How many bytes to copy from the GPU ahead of saving then. Default 10Mb.
            overwrite: Whether to allow overwriting existing checkpoints. Defaults to True.
            _extensions: Extensions to apply to output streams (EXPERIMENTAL)

        N. B. If sync_files is disabled, there's no guarantee that the checkpoint will be consistent in the case of a failure.
        )rZ   r[   r\   Nsuperr   r   r   r3   r   )r   r   rV   rW   rX   rY   rZ   r[   r\   r1   	__class__s             r   r   zFsspecWriter.__init__s   sW    4 	 !#!5 	 		
 ,%DGG%%d5f5	r   r<   c                 ,    t         j                  |      S r   r   rB   r@   s     r   rB   z#FsspecWriter.validate_checkpoint_id       00??r   )rJ   rK   rL   __doc__r   
TORCH_SAVEr	   rM   r%   rN   rR   intr   r   r   r   rQ   rB   __classcell__r`   s   @r   r   r   e   s      &*%/DH4G4R4R%6C$%%6 #%6 	%6
 %6  #%6 %6 h'?@A%6 2%6 
%6N @5bkk9I3J @t @ @r   c                   ~     e Zd Zdeeej                  f   ddf fdZedeeej                  f   de	fd       Z
 xZS )r   r   r   Nc                     t         |   |       t               | _         | j                  j                  |fi || _        y r   r^   )r   r   r1   r`   s      r   r   zFsspecReader.__init__   s6    ,%DGG%%d5f5	r   r<   c                 ,    t         j                  |      S r   rb   r@   s     r   rB   z#FsspecReader.validate_checkpoint_id   rc   r   )rJ   rK   rL   r	   rM   r%   rN   r   rQ   rR   rB   rg   rh   s   @r   r   r      sX    6U3#34 64 6
 @5bkk9I3J @t @ @r   )rO   r%   collections.abcr   r   
contextlibr   pathlibr   typingr   r   r	   fsspec.corer
   'torch.distributed.checkpoint._extensionr   'torch.distributed.checkpoint.filesystemr   r   r   r   fsspecr   __all__r   r   r   r!   r   r   <module>ru      sq    
 	 / %  1 1 ! L  ) A. A.J7@# 7@t@# @r   