
    rhdf              	          U d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZ d dlZd dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZmZmZmZmZmZm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5 ddl6m7Z7  ejp                  e9      Z:ejv                  e<d<   g dZ= G d de'      Z> G d de$      Z? G d de?      Z@	 d-deAeBe	f   dedeCde#fdZDdeEe#   deEe#   fdZFdeAeBe	f   d eCde&fd!ZG	 d-deEe&   d"eCdeHeEe&   ef   fd#ZIdedefd$ZJd%ed&edeCfd'ZKd(ej                  d)edeCfd*ZMd+eEe&   dedeCfd,ZNy).    N)ChainMap)reduce)AnycastOptionalUnion)narrow_tensor_by_index)dedup_save_plans)FLATTEN_MAPPINGflatten_state_dict)_flatten_sharded_tensors)set_element)BytesStorageMetadataChunkStorageMetadataMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESStorageMetaTensorStorageMetadata)LoadPlanLoadPlannerReadItemSavePlanSavePlanner	WriteItemWriteItemType)_compare_save_plans_contains_usable_plan"_create_default_metadata_only_plan_create_read_items_create_write_items_init_state_dict_merge_delta_local_plans)find_state_dict_object)DTensor   )_versionlogger)DefaultSavePlannerDefaultLoadPlannercreate_default_local_load_plancreate_default_global_load_plancreate_default_local_save_plancreate_default_global_save_planc                      e Zd ZU eed<   	 	 	 	 	 ddededee   dededdfd	Z	 	 dd
edee	   deddfdZ
defdZdee   dee   fdZdee   deee   ef   fdZdee   deee   ee   ef   fdZdee   deee   ef   fdZdedefdZdedefdZdedeej2                  ej6                  f   fdZdedefdZdedefdZ y)r*   mappingsNr   flatten_sharded_tensorsdedup_replicated_tensorsdedup_save_to_lowest_rankenable_plan_cachingreturnc                     || _         || _        i | _        || _        |t        j                  d       | j                  j                  | _        || _	        y )NzDefaultSavePlanner's `dedup_replicated_tensors` argument is being deprecated, and no longer has any effect. Please remove this argument from your call.)
r   r2   r1   r4   r)   warning	__class____name___cached_plans_key_enable_plan_caching)selfr   r2   r3   r4   r5   s         /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/default_planner.py__init__zDefaultSavePlanner.__init__H   sV     #5'>$)B&#/NN"
 '+nn&=&=$7!    
state_dictstorage_metais_coordinatorc                     | j                   rt        |      \  }| _        | j                  rt        |      }|| _        || _        y N)r   r1   r2   r   rA   rC   )r=   rA   rB   rC   s       r>   set_up_plannerz!DefaultSavePlanner.set_up_planner]   s@     ""(::(F%J''1*=J$,r@   c                    t        | j                  | j                        }| j                  r!t	        j
                  || j                        }|| _        | j                  r| j                  t        j                  v rIt        |t        j                  | j                           r"t        j                  d       t        g d      S |t        j                  | j                  <   | j                  S )Nplanner_datazINo change in the local plan. Skipping sending the plan to the coordinatorFusable)r.   rA   rC   r   dataclassesreplacer1   planr<   r;   r   _cached_save_planr   r)   infor   )r=   rN   s     r>   create_local_planz$DefaultSavePlanner.create_local_planj   s    -doot?R?RS""&&t$--HD	$$ &&+*G*GG'+778N8NO _  511HL--d.D.DEyyr@   	all_plansc                 .    t        || j                        S rE   )r
   r4   )r=   rR   s     r>   _dedup_save_plansz$DefaultSavePlanner._dedup_save_plans   s    	4+I+IJJr@   c                    | j                  |      }t        |      \  }}| j                  rA|D cg c]  }|j                   }}t	        t        |       }t        j                  ||      }t        ||      st        d      ||fS c c}w )NrH   zFailed to validate global plan)
rT   r/   r   rI   dictr   rL   rM   _validate_global_plan
ValueError)r=   rR   deduped_plansglobal_planmetadatapplanner_data_dictmerged_mappingss           r>   _create_global_planz&DefaultSavePlanner._create_global_plan   s     ..y9 ? NX""
 :E EA E E"8->#?@O"**8/RH$[(;=>>H$$ !Fs   Bc                    g }| j                   t        j                  vrp|t        j                  | j                   <   | j                  |      \  }}|t        j                  | j                   <   |t        j
                  | j                   <   |||fS t        |      sVt        g d      gt        |      z  }t        j                  | j                      }t        j
                  | j                      }nt        t        j                  | j                      |      }|t        j                  | j                   <   | j                  |      \  }}| j                   | j                  v rit        t        j                  | j                      |      D ]?  \  }}t        ||      r|j                  t        g d             /|j                  |       A |t        j                  | j                   <   |t        j
                  | j                   <   |||fS )zw
        Create global plan with caching.
        Returns a tuple of global_plan_delta, global_plan, metadata.
        FrJ   )r;   r   _cached_all_plansr_   _cached_global_plan_cached_metadatar   r   lenr$   zipr   append)r=   rR   global_plan_deltarZ   r[   merged_planscached_plannew_plans           r>    _create_global_plan_with_cachingz3DefaultSavePlanner._create_global_plan_with_caching   s    -/!!)F)FF
 ENK))$*@*@A$($<$<Y$G!KFQK++D,B,BCCKK(()?)?@X55 %Y/ "*"U!; <s9~ M%99$:P:PQK"33D4J4JKH 4--d.D.DEyL EQK))$*@*@A$($<$<\$J!K%%)A)AA-033D4J4JK[. ;)K +;A)00"U1KL)00:; GRK++D,B,BCCKK(()?)?@ +x77r@   c                     g }| j                   r| j                  |      \  }}}n| j                  |      \  }}|}|| _        || _        || j                  fS rE   )r<   rk   r_   rZ   r[   )r=   rR   rg   rZ   r[   s        r>   create_global_planz%DefaultSavePlanner.create_global_plan   sl     -/$$ 55i@	! %)$<$<Y$G!K +&  $--//r@   rj   c                     |}|j                   st        j                  | j                     }|S |}|t        j                  | j                  <   |S rE   )rK   r   _cached_final_save_planr;   r=   rj   finished_plans      r>   _finish_plan_with_cachingz,DefaultSavePlanner._finish_plan_with_caching   sO    "*'??@V@VWM  %MJRK//0F0FGr@   c                 f    |}| j                   r| j                  |      }|| _        | j                  S rE   )r<   rr   rN   rp   s      r>   finish_planzDefaultSavePlanner.finish_plan   s1    "*$$ ::8DM!	yyr@   
write_itemc                 \    | j                  |j                        }| j                  ||      S rE   )lookup_objectindextransform_object)r=   ru   objects      r>   resolve_datazDefaultSavePlanner.resolve_data   s+    ##J$4$45$$Z88r@   rx   c                 .    t        | j                  |      S zSExtension from the planner interface to make it easy to extend the default planner.r%   rA   r=   rx   s     r>   rw   z DefaultSavePlanner.lookup_object      %doou==r@   rz   c                     |j                   t        j                  k(  r,t        j                         }t        j                  ||       |}|S r}   )typer   BYTE_IOioBytesIOtorchsave)r=   ru   rz   bytess       r>   ry   z#DefaultSavePlanner.transform_object  s7    ??m333JJLEJJvu%Fr@   )TTNFFNF)!r:   
__module____qualname__r   __annotations__boolr   r?   r   r   rF   r   rQ   listrT   tupler   r_   rk   rm   rr   rt   r   r   r   Tensorr   r   r{   r   r   rw   ry    r@   r>   r*   r*   E   s    $((,37*/$)8 8 "&8 #+4.	8
 $(8 "8 
80 /3$	-#- {+- 	-
 
-8 .K4> Kd8n K%h%	tH~x'	(%*98h98	tH~tH~x7	898v0h0	tH~x'	(0,( x H  9y 9U5<<;S5T 9>= >S >9 c r@   r*   c            	       F   e Zd ZU dZeed<   eed<   	 	 	 ddedededdfd	Z	 	 dd
ede	e
   deddfdZdefdZdee   dee   fdZdedefdZdedej&                  ddfdZdefdZdedej.                  ddfdZdedej.                  fdZdedej.                  fdZy)r+   ak  
    DefaultLoadPlanner that adds multiple features on top of LoadPlanner.

    In particular it adds the following:

    flatten_state_dict: Handle state_dict with nested dicts
    flatten_sharded_tensors: For FSDP in 2D parallel mode
    allow_partial_load: If False, will raise a runtime error if a key is present in state_dict, but not in the checkpoint.
    original_state_dictr1   r   r2   allow_partial_loadr6   Nc                 J    || _         || _        i | _        i | _        || _        y rE   )r   r2   r   r1   r   )r=   r   r2   r   s       r>   r?   zDefaultLoadPlanner.__init__  s+     #5'>$#% "4r@   rA   r[   rC   c                     t        |       || _        | j                  rt        |      }| j                  rt	        |      \  }| _        || _        || _        || _        y rE   )	r#   r   r2   r   r   r1   rA   r[   rC   )r=   rA   r[   rC   s       r>   rF   z!DefaultLoadPlanner.set_up_planner(  sW     	$#- ''1*=J""(::(F%J$ ,r@   c                    | j                   J | j                  rt        | j                  j	                               }t        | j                   j
                  j	                               }||z
  }|r[dt        _        t        | j                        \  }}t        |j	                               }||z  r||c| _        | _	        d t        _        t        | j                  | j                   | j                         S )N2_3)r[   r   setrA   keysstate_dict_metadatar(   _derived_versionr   r1   r,   r   )r=   current_keys	load_keysmissing_keysold_state_dictold_mappingsold_keyss          r>   rQ   z$DefaultLoadPlanner.create_local_plan;  s    }}(((""$ t3356LDMM==BBDEI$|3L,1)/A,,0, ~2245l*5C\2DOT] -1)-OOT]]0G0G,G
 	
r@   rZ   c                     t        |      S rE   )r-   )r=   rZ   s     r>   rm   z%DefaultLoadPlanner.create_global_planb  s    .{;;r@   rj   c                     |S rE   r   )r=   rj   s     r>   rt   zDefaultLoadPlanner.finish_plane  s    r@   	read_itemvaluec                 (   | j                   rNt        | j                  | j                  |j                  j
                     t        j                  |d             y t        j                  |d      | j                  |j                  j
                  <   y )NF)weights_only)	r   r   r   r1   
dest_indexfqnr   loadrA   )r=   r   r   s      r>   
load_byteszDefaultLoadPlanner.load_bytesh  sl    ""((i22667

5u5 9>

E9DOOI00445r@   c                 \    | j                  |j                        }| j                  ||      S rE   )lookup_tensorr   transform_tensorr=   r   tensors      r>   resolve_tensorz!DefaultLoadPlanner.resolve_tensort  s+    ##I$8$89$$Y77r@   r   c                      y rE   r   r   s      r>   commit_tensorz DefaultLoadPlanner.commit_tensorx  s    r@   rx   c                 .    t        | j                  |      S r}   r~   r   s     r>   r   z DefaultLoadPlanner.lookup_tensor{  r   r@   c                 D    t        ||j                  |j                        S r}   )r	   dest_offsetslengthsr   s      r>   r   z#DefaultLoadPlanner.transform_tensor  s    %fi.D.DiFWFWXXr@   )TTFr   )r:   r   r   __doc__r   r   r   r   r?   r   r   rF   r   rQ   r   rm   rt   r   r   r   r   r   r   r   r   r   r   r   r   r@   r>   r+   r+     s7    )( $((,#(	
5 
5 "&
5 !	
5
 

5 (,$	-#- 8$- 	-
 
-&%
8 %
N<d8n <h <H  
H 
RZZ 
D 
8 8x  $ >= >U\\ >Y( YELL Yr@   r+   c            	       `     e Zd ZdZd fd	ZdededefdZ	 	 dde	de
e   d	eddf fd
Z xZS )_EmptyStateDictLoadPlannera  
    Extension of DefaultLoadPlanner, which rebuilds state_dict from the saved metadata.
    Useful for loading in state_dict without first initializing a model, such as
    when converting a DCP checkpoint into a Torch save file.

    . N.B. `state_dict` must be an empty dictionary when used with this LoadPlanner

    .. warning::
        Because the entire state dict is initialized, It's recommended to only utilize
        this LoadPlanner on a single rank or process to avoid OOM.

    Nc                 2    || _         t        |   |i | y rE   )r   superr?   )r=   r   argskwargsr9   s       r>   r?   z#_EmptyStateDictLoadPlanner.__init__  s    	$)&)r@   keyr[   r6   c           	      4     j                   y| j                   v r	 g }|j                  j                  |      }|D ]D  }|r/|j                  dj	                  |d   t        |      g             4|j                  |       F t         fd|D              ryy)NT.c              3   :   K   | ]  }|j                   v   y wrE   )r   ).0unflattened_keyr=   s     r>   	<genexpr>zA_EmptyStateDictLoadPlanner._should_include_key.<locals>.<genexpr>  s     T$))+Ts   F)r   rI   getrf   joinstrany)r=   r   r[   unflattened_keysrI   r   s   `     r>   _should_include_keyz._EmptyStateDictLoadPlanner._should_include_key  s    99$))&(,,005+ 	9O ''HH.r2C4HIJ
 !''8	9 TCSTTr@   rA   rC   c                    |rJ |J |j                   j                         D ]  \  }}| j                  ||      st        |t              r5t        j                  |j                  |j                  j                        }|j                  )||j                  v rt        ||j                  |   |       |||<    t        | 5  |||       y )N)dtype)r   itemsr   
isinstancer   r   emptysize
propertiesr   rI   r   r   rF   )r=   rA   r[   rC   kvr9   s         r>   rF   z)_EmptyStateDictLoadPlanner.set_up_planner  s     ~### 00668 		"DAq++Ax8!23KKall.@.@A$$0Q(:O:O5OJ(=(=a(@!D !
1		" 	z8^Dr@   rE   r   )r:   r   r   r   r?   r   r   r   r   r   r   rF   __classcell__)r9   s   @r>   r   r     sj    *s h 4 4 (,$	E#E 8$E 	E
 
E Er@   r   rA   r[   strictr6   c           	         g }	 | j                         D ]  \  }}||j                  vr|rt        d| d      &|j                  |   }t        |t              rVt        |dd       I|j                  |j                         k7  r,t        d|j                   d|j                          d|       t        |t              r,|j                  j                         |t        |||      z  }|t        |||      z  } t        |      S )Nz&Missing key in checkpoint state_dict: r   r   zSize mismatch between saved z and current: z for )r   r   RuntimeErrorr   r   getattrr   rX   r&   device_meshget_coordinater!   r   )rA   r[   r   requestsr   objmds          r>   r,   r,     s    H $$& 9Sh222"%KC5PQ#RSS))#.r01VT*6388:%.rwwi~chhj\QVWZV[\ 
 c7#--/;.sB<<*3C88H/92 Hr@   rR   c                     | S )z
    Create global load plan used by DefaultLoadPlanner.

    The default load behavior involved no global coordination and this function
    currently doesn't change the local plans.
    r   )rR   s    r>   r-   r-     s
     r@   rC   c                     g }| j                         D ]O  \  }}t        |t              r+|j                  j	                         1|t        ||      z  }A|t        ||      z  }Q t        |      S )a  
    Create the ``SavePlan`` used by DefaultSavePlanner.

    On non-coordinator ranks, this function ignores tensors and non-tensor objects,
    only producing writes for ShardedTensor objects.

    On the coordinator rank, produce writes for all values.
    )r   r   r&   r   r   r"   r   )rA   rC   r   r   r   s        r>   r.   r.     sw     H$$& 
6S c7#--/;/S99
 +C55H
6 Hr@   rewrite_index_hintsc                 .   i }g }| D ]  }g }|j                   D ]  }|j                  t        j                  k(  s|j                  j
                  |vsJ |j                  t        j                  k(  r3t               ||j                  j
                  <   |j                  |       |j                  J t        t        |j                  |j                  j
                  t        |j                  j                  |j                  j                  g                   }|}|rKt        j                   |j                  t#        |j$                              }	t        j                   ||	      }|j                  |       |j                  j&                  J d|j                  j
                   d       |j$                  j                  |j                  j&                          |j                  t        j                   ||              |t)        |      fS )a6  
    Create the global plan and metadata used by DefaultSavePlanner.

    Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans.

    The only global planning change is to update index hints in all ``MetadataIndex`` objects if
    ``rewrite_index_hints`` is True.
    )r   r   chunks)rx   zZ
                    Cannot create MD for tensor without bounds.
                    FQN: z
                )r   )r   r   r   SHARDrx   r   r   r   rf   tensor_datar   r   
setdefaultr   r   rL   rM   rd   r   chunkr   )
rR   r   r   	new_plansrN   	new_itemsitem	tensor_mdnew_item	new_indexs
             r>   r/   r/     s    $&BI #E	JJ  	@D99 3 33zz~~R///yyM111%9%;4::>>"  &''333 )MM

-'+'7'7'B'B!%!1!1!6!6#%
	  & + 3 3

#i.>.>*?!I  +224yIH  *''--9  @**..) *< 9   ''(8(8(>(>?A 	@B 	,,TCDG#EH x|$$r@   c                 :    t        |       }t        |g      \  }}|S )zTReturn the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``.)r    r/   )rA   rN   _r   s       r>   _create_default_local_metadatar   K  s!    -j9D+TF3EArIr@   box0box1c                    t        | j                        }t        |      D ]d  }| j                  |   |j                  |   |j                  |   z   k\  r y|j                  |   | j                  |   | j                  |   z   k\  sd y y)z9Check if two boxes overlap. Tuples are (offset, lengths).FT)rd   offsetsrangesizes)r   r   ndimsis       r>   _check_box_overlapr   R  sz     E5\ <<?dll1o

1==<<?dll1o

1==	 r@   outer_box_size	inner_boxc                     t        t        |             D ]Q  }|j                  |   dk  r y|j                  |   dk  r y|j                  |   |j                  |   z   | |   kD  sQ y y)Nr   FT)r   rd   r   r   )r   r   r   s      r>   _check_box_boundsr   b  su     3~&' Q!#??1!Q)//!"44~a7HH r@   rZ   c           	         d}|j                   j                         D ]5  \  }}t        |t              rt	        |j
                        dk(  r1d}t        |j                        D ]  \  }}t        |j
                  |      s$t        j                  d||j
                  |       d}|t        t        j                  |j                  d      z  }|j                  |dz   d  D ])  }t        ||      st        j                  d|||       d}+  t        t        j                  |j
                  d      }	||	k7  st        j                  d||	|       d}8 |S )NTr   z~
                        key:%s has out of bounds chunk:
                        tensor-size:%s chunk: %s
                    Fr'   z$key:%s has overlapping chunks: %s %szq
                    key:%s invalid fill tensor-volume:
                    %s chunks-volume: %s
                )r   r   r   r   rd   r   	enumerater   r   r)   r8   r   operatormulr   r   )
rZ   r[   all_goodr   r   chunks_volume	chunk_idxchunk0chunk1tensor_volumes
             r>   rW   rW   p  sN   H2288: )
Ue12uzz?a!*5<<!8 	%Iv$UZZ8 JJ !VHLL&,,BBM  ,,y1}7 %%ff5NN>VV  %H%!	%0 x||UZZ;M)NN  HS)V Or@   )T)OrL   r   loggingr  collectionsr   	functoolsr   typingr   r   r   r   r   torch.distributed._shard._utilsr	   .torch.distributed.checkpoint._dedup_save_plansr
   )torch.distributed.checkpoint._nested_dictr   r   2torch.distributed.checkpoint._sharded_tensor_utilsr   &torch.distributed.checkpoint._traverser   %torch.distributed.checkpoint.metadatar   r   r   r   r   r   r   r   $torch.distributed.checkpoint.plannerr   r   r   r   r   r   r   ,torch.distributed.checkpoint.planner_helpersr   r   r    r!   r"   r#   r$   "torch.distributed.checkpoint.utilsr%   torch.distributed.tensorr&    r(   	getLoggerr:   r)   Loggerr   __all__r*   r+   r   rV   r   r   r,   r   r-   r.   r   r/   r   r   Sizer   rW   r   r@   r>   <module>r     s    	      - -  B K X >	 	 	     F ,  +**84 4F FRsY sYl>E!3 >ED DH'S#X'*2'<@''T	H~		(^	S#X04: !%2%H~2%2% 4>8#$2%j 8 1 9M RV  JJ+?	-tH~ - -d -r@   