rh2$ddlZddlZddlZddlmZmZddlZddlmZddl m Z m Z ddl m Z mZmZmZmZmZmZmZmZmZmZmZmZddlmZddlmZmZmZm Z m!Z!m"Z"ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+dd l,m-Z-d d gZ.Gd d e Z/Gdd e Z0y)N)AnyOptional)narrow_tensor_by_index) FsspecReader FsspecWriter) _gen_file_name _get_dtype_get_safetensors_file_metadata_HFStorageInfo _metadata_fnCUSTOM_METADATA_KEYDATA_KEYDATA_OFFSETS_KEYDEFAULT_EXTRA_METADATA_KEY DTYPE_KEYSAVED_OFFSETS_KEY SHAPE_KEYSUFFIX)SerializationFormat)ChunkStorageMetadataMetadata MetadataIndex StorageMetaTensorPropertiesTensorStorageMetadata)LoadPlan LoadPlannerReadItemSavePlan SavePlanner WriteItem) WriteResult)FutureHuggingFaceStorageWriterHuggingFaceStorageReaderc eZdZdZ ddedeeeefdeededdf fd Z d e e de e fd Z d e d e dee effd Zdede e eddfdZdeeeefde edeee effdZedefdZxZS)r$z A writer that writes to a huggingface repository in the huggingface format. Uses Fsspec back-end to communicate with back-end storage. Fsspec registration of the storage solution is required. Npathfqn_to_index_mappingtoken save_shardedreturnc|!t|||tjnt||tj||_||_y)a Initialize the huggingface writer pointing to path. Args: path: hf directory where the checkpoint will be read from. Needs to have .safetensors files, but can be from any fsspec supported storage, including localFS and hf://. fqn_to_index_mapping: A mapping from tensor FQN to the index of the file that the tensor should be written to. Indices are from 1 to N, where N is the number of files. If not provided, the tensors will be written to a single file. If none, then all the tensors on the same rank will be written to the same file. token: The token to use to authenticate with huggingface hub. save_sharded: If True, save the checkpoint as a sharded checkpoint where every rank saves its own shard. Default is False which assumes full tensors are being saved. N)r'r)serialization_format)r'r-)super__init__r SAFETENSORS_fqn_to_index_mapping _save_sharded)selfr'r(r)r* __class__s z/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/hf_storage.pyr/z!HuggingFaceStorageWriter.__init__8s`0   G %8%D%D   G %8%D%D  @T")planscg}t|dD]Y\}}i}|j|j|d<|jr||d<|jt j ||[|S)N)startr( shard_index) storage_data) enumerater1r2append dataclassesreplace)r3r7 new_plansiplanr<s r5prepare_global_planz,HuggingFaceStorageWriter.prepare_global_plan^s{  a0 SGAt+-L))57;7Q7Q 34!!./ ]+   [00LQ R Sr6rCplannerc&t|jdk(rt}|jg|S|j}d}d}d|vr|d}d|vr|d}|j ||j}|t |jnd}tj} |jD]J\} } t| ||} | j|jj|j| | | fLt |A|| S)Nrr(r;r9)lenitemsr# set_resultr<_split_by_storage_planmaxvaluesqueueQueuerputfs concat_pathr'r. _write_data)r3rCrEfutr< storage_planr;buckets highest_index file_queue file_index write_items file_namer4s r5 write_dataz#HuggingFaceStorageWriter.write_dataks  tzz?a  (C NN2 J(,'8'8 15 %) !\ 1'(>?L L (&}5K--lDJJG6B6NL//12TU "'++- '.}}  #J &z=+NI NN$$TYY :I{S   w"7J77r6metadataresultsc<|jryi}i}d}|D]z}|j|Dcic]-}|jj|jj /c}|t |Dcgc]}|jjc}z }|d|i|d<||d<|jj|jt}|jj|d5} tj|| ddddycc}wcc}w#1swYyxYw)Nr total_sizer\ weight_mapw)indent)r2updateindexfqnr< relative_pathsumlengthrPrQr'r create_streamjsondump) r3r\r]metadata_to_write storage_mdr_wr_listwr metadata_path metadata_files r5finishzHuggingFaceStorageWriter.finishs       JG   GNOr<<<O  #H"r55HI IJ  J *6z(B*%*4,'++DII,I WW " "=# 6 B- II'q A B BPH  B Bs2D +D &DDrTrHc|d|iSi}|D]<}|jj}||}||vr|g||<)||j|>|S)Nr9)rerfr>)r3rTrHrUitemkeyidxs r5rJz/HuggingFaceStorageWriter._split_by_storage_plansk  u:  *D**..Cs#C'! $v  ##D) *r6ctSN)r )r3s r5rqz&HuggingFaceStorageWriter.metadata_pathsr6)NNF)__name__ __module__ __qualname____doc__strrdictintboolr/listrrDr r#r"r[rrsr!rJpropertyrq __classcell__r4s@r5r$r$1s:>#" $*$*'tCH~6$*} $*  $*  $*L h DN 888 [! " 8>BxB$tK7H2IBdB&$T#s(^4=A)_ c4 ?" #&sr6c^eZdZdZd dedeeddffd Zdedede dfd Z de fd Z xZ S) r%z A reader that reads from a huggingface repository in the huggingface format. Uses in Fsspec back-end to communicate with storage. Fsspec registration of the storage solution is required. Nr'r)r+cN|t|||yt||y)ai Initialize the huggingface reader pointing to path. Args: path: hf directory where the checkpoint will be read from. Needs to have .safetensors file, but can be from any fsspec supported storage, including localFS and hf://. token: The token to use to authenticate with huggingface hub. N)r'r))r')r.r/)r3r'r)r4s r5r/z!HuggingFaceStorageReader.__init__s-   G $e  4 G $  'r6rCrEc fddlm}i}|jD]H}|j|j}|j }|j |gj|J|jD]\}}|jj|d5} || j} | D cic] } | d| d } } |D]2} |j| j}| | jjt}tj||j }|j#|j$}t'|| j(| j*}|j-| j/}|j1|j1k(s6Jd| jd|j1d|j1|j3||j5| |5 dddt7}|j9d|Scc} w#1swYxYw) Nr) deserializerbr9dtypezreq z mismatch sizes z vs ) safetensorsrrHr< storage_indexrg setdefaultr>rPrjread dest_indexrfrtorch frombufferrreshapeshaperstorage_offsetslengthsresolve_tensordetachsizecopy_ commit_tensorr#rI)r3rCrErper_file read_itemitem_mdrZreqsstream deserialized tensor_infodeserialized_dictreq tensor_bytestensor target_tensorrSs r5 read_dataz"HuggingFaceStorageReader.read_datas +.0 AI&*&7&7 8O8O&PG--I    2 . 5 5i @ A (~~/ >OIt&&y$7 >6 +6;;=9 FR@7BKNKN2@!@ >C"//0A0ABG#4S^^5G5G#H#RL"--$%mmF$^^GMM:F3 3 3S[[F%,$:$:3$?$F$F$HM(--/6;;=@s0011A-BTBTBVAWW[\b\g\g\i[jk@"''/))#}=)> > > >>h t 9@  > >sH&.H!>D:H&!H&&H0 ci}i}g}|jj|jD])}|jts|j |+|D]}|jj |d5}t|\}}|jt} d} | r=| jtr(tj| jt} |jD]7\} } | tk(r| | | t} ndgt| t z} | |vrt#t%t'| t(t+j,t/| t | D cgc] \}} || z c} }t1t+j, t+j,| t g|| <n|| j2j t1t+j,| t+j,| t t5|| j6}t9t|D]$}t;||| t || |z||<&t+j,||| _| t=| | | t}n!t=| dgt| t z}t?|| t@d| t@d| t@dz t+j,| t t'| t( ||<: dddtC|| }tE|d dtG|_$|jJ|jH_%|Scc} }w#1swY2xYw) Nrrr)offsetssizes) propertiesrchunks)r)rfoffsetr9)rgrrirr)state_dict_metadatar< storage_meta)&rPlsr'endswithrr>rjr getrr rkloadsrHrrGrrrr rrSizeziprrrrrangerKrr rrgetattrrrload_id)r3rr<safetensors_filesfilesafetensor_filefsafetensors_metadata_custom_metadatadcp_sharding_inforvvalrsavedrrBmetadata_indexr\s r5 read_metadataz&HuggingFaceStorageReader.read_metadatasn@B<> GGJJtyy) /D}}V$!((. / 1A O&&=@ *H*K'$a"6":":;U"V$(!"':':;N'O(, '++,?@)%!5 : : <6HC88 )4!23!78I!J"#s3y>':!:"553H'7&0Y&@("':=S^V9T!"(5v%*FN!""!5,1JJv,>*/**S^*D!"$4+C0$,C077>>0 % 6 2%**S^:T $$7$<$A$AB!&s4y!1RA&)$q'3y>!3Dvay3P&QDGR8= 48H+C05)4)6 #,=c,BCT,U**7 #QC#c)n2E,E*4B&5"#34Q7"#34Q7#>N:OPQ:RR#jjY8(Y8 4L0a6@ @ A F 3%  8^T 2 :$/MH !(, %c!"5@ @ s :C4N.M<=F-N<NN ry)rzr{r|r}r~rr/rrr#rrrrrs@r5r%r%sS (S(#($( +h+++ZUxUr6)1r?rkrMtypingrrrtorch.distributed._shard._utilsr/torch.distributed.checkpoint._fsspec_filesystemrr&torch.distributed.checkpoint._hf_utilsrr r r r r rrrrrrr'torch.distributed.checkpoint.filesystemr%torch.distributed.checkpoint.metadatarrrrrr$torch.distributed.checkpoint.plannerrrrrr r!$torch.distributed.checkpoint.storager" torch.futuresr#__all__r$r%r6r5rsz BVH=  &'A BA|AHY|Yr6