
    rh*                        d Z ddlmZmZ ddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZmZmZmZ ddlmZ  ej:                  e      ZdZ dZ! G d dejD                        Z#d"dZ$ G d dejD                        Z% G d de      Z& G d de      Z' G d dee'      Z( G d de      Z) G d d e      Z*g d!Z+y)#zPyTorch Phi-3 model.    )CallableOptionalN)nn   )ACT2FN)Cache)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONS)Unpack)logging   )MistralDecoderLayerMistralForCausalLM MistralForSequenceClassificationMistralForTokenClassificationMistralPreTrainedModeleager_attention_forwardrotate_half   )
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Phi3MLPc                 *   t         |           || _        t        j                  |j
                  d|j                  z  d      | _        t        j                  |j                  |j
                  d      | _        t        |j                     | _        y )Nr   Fbias)super__init__configr   Linearhidden_sizeintermediate_sizegate_up_proj	down_projr   
hidden_actactivation_fn)selfr   	__class__s     x/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/phi3/modular_phi3.pyr   zPhi3MLP.__init__1   sp    IIf&8&8!f>V>V:V]bc6#;#;V=O=OV[\#F$5$56    hidden_statesreturnc                     | j                  |      }|j                  dd      \  }}|| j                  |      z  }| j                  |      S )Nr   dim)r"   chunkr%   r#   )r&   r*   	up_statesgates       r(   forwardzPhi3MLP.forward9   sL    %%m4	#//!/4i 2 24 88	~~i((r)   )__name__
__module____qualname__r   torchFloatTensorr3   __classcell__r'   s   @r(   r   r   0   s'    7)U%6%6 )5;L;L )r)   r   c                 `   |j                  |      }|j                  |      }|j                  d   }| dd|f   | d|df   }}|dd|f   |d|df   }
}	t        j                  ||z  t	        |      |z  z   |gd      }t        j                  |	|z  t	        |	      |z  z   |
gd      }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r-   .Nr.   )	unsqueezeshaper7   catr   )qkcossinposition_idsunsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds                r(   apply_rotary_pos_embrL   B   s    ( --
&C
--
&C2Jc;J;&'3
+;)<6Ec;J;&'3
+;)<6Eii%#++e*<s*BCVLRTUGii%#++e*<s*BCVLRTUGGr)   c                   >    e Zd ZdZddedee   f fdZ	 	 ddej                  de
ej                  ej                  f   deej                     dee   d	eej                     d
ee   de
ej                  eej                     ee
ej                        f   fdZ xZS )Phi3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperr   	layer_idxc                 |   t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        |j                  | _        | j                  dz  | _
        |j                  | _        d| _        |j                  | j                  z  d|j                  | j                  z  z  z   }t        j                  |j                  | j                  z  |j
                  d      | _        t        j                  |j
                  |d      | _        y )Nhead_dimg      Tr   Fr   )r   r   r   rO   getattrr    num_attention_headsrQ   num_key_value_headsnum_key_value_groupsscalingattention_dropout	is_causalr   r   o_projqkv_proj)r&   r   rO   op_sizer'   s       r(   r   zPhi3Attention.__init__e   s    "
F4F4F&JdJd4de$*$>$>&B\B\$\!#)#=#= }}d*!'!9!9,,t}}<qFD^D^aeananDn?ooii : :T]] JFL^L^ejk		&"4"4gEJr)   r*   position_embeddingsattention_maskpast_key_valuecache_positionkwargsr+   c           
         |j                   d d }g |d| j                  }| j                  |      }	| j                  j                  | j                  z  }
|	dd |
f   }|	d|
|
| j
                  | j                  z  z   f   }|	d|
| j
                  | j                  z  z   d f   }|j                  |      j                  dd      }|j                  |      j                  dd      }|j                  |      j                  dd      }|\  }}t        ||||      \  }}|'|||d}|j                  ||| j                  |      \  }}t        }| j                  j                  dk7  rt        | j                  j                     } || ||||f| j                  sdn| j                  | j                   t#        | j                  dd       d	|\  }} |j$                  g |d j'                         }| j)                  |      }||fS )
Nr-   .r   r   )rB   rA   r_   eagerg        sliding_window)dropoutrV   rc   )r=   rQ   rZ   r   rS   rT   view	transposerL   updaterO   r   _attn_implementationr
   trainingrW   rV   rR   reshape
contiguousrY   )r&   r*   r\   r]   r^   r_   r`   input_shapehidden_shapeqkv	query_posquery_states
key_statesvalue_statesrA   rB   cache_kwargsattention_interfaceattn_outputattn_weightss                       r(   r3   zPhi3Attention.forwardt   s$    $))#2.88b8$--8mmM*KK33dmmC	3

?+i)d6N6NQUQ^Q^6^*^^^_
3	D,D,Dt}},T T VVW#((6@@AF__\2<<QB
#((6@@AF&S#7jRUWZ#[ j%#&snUL'5'<'<ZW[WeWegs't$J(?;;++w6"9$++:Z:Z"[$7
%
  $}}C$2H2HLL"4;;0@$G
%
 
%
!\ *k));;;;FFHkk+.L((r)   )N)NN)r4   r5   r6   __doc__r   r   intr   r7   Tensortupler   
LongTensorr   r	   r3   r9   r:   s   @r(   rN   rN   b   s    GKz Khsm K( +/590)||0) #5<<#=>0) !.	0)
 !0) !!1!120) -.0) 
u||Xell3XeELL>Q5RR	S0)r)   rN   c                   d    e Zd Zdedef fdZ	 	 	 	 	 	 ddej                  deej                     deej                     dee
   dee   d	eej                     d
eeej                  ej                  f      dee   deej                  eeej                  ej                  f      f   fdZ xZS )Phi3DecoderLayerr   rO   c                    t         |   ||       || _        t        ||      | _        t        |      | _        t        j                  |j                        | _
        t        j                  |j                        | _        y )N)r   rO   )r   r   r   rN   	self_attnr   mlpr   Dropoutresid_pdropresid_attn_dropoutresid_mlp_dropout)r&   r   rO   r'   s      r(   r   zPhi3DecoderLayer.__init__   s`    +&f	J6?"$**V-?-?"@!#F,>,>!?r)   r*   r]   rC   r^   	use_cacher_   r\   r`   r+   c                     |}	| j                  |      } | j                  d|||||||d|\  }}
|	| j                  |      z   }|}	| j                  |      }| j	                  |      }|	| j                  |      z   }|S )N)r*   r]   rC   r^   r   r_   r\    )input_layernormr   r   post_attention_layernormr   r   )r&   r*   r]   rC   r^   r   r_   r\   r`   residualself_attn_weightss              r(   r3   zPhi3DecoderLayer.forward   s     !,,];+94>> 	,
')%)) 3	,
 	,
(( !4#:#:=#II 55mD/ 4#9#9-#HHr)   )NNNFNN)r4   r5   r6   r   rx   r   r7   ry   r   r{   r   boolrz   r   r	   r8   r3   r9   r:   s   @r(   r}   r}      s    @z @c @ 2637*.$)59KO|| !. u//0	
 ! D> !!1!12 &eELL%,,,F&GH -. 
u  (51B1BEDUDU1U+V"WW	Xr)   r}   c                       e Zd ZdZy)Phi3PreTrainedModelz0.0.5N)r4   r5   r6   _versionr   r)   r(   r   r      s    Hr)   r   c                   "    e Zd Z	 	 	 	 	 	 	 ddZy)Phi3ForCausalLMNc	                    |r_| j                   j                  rI|j                  d   | j                   j                  dz   k\  r |d   }
|
| j                   j                  k  rd } t	               j
                  d||||||||d|	}|S )Nr   r   )	input_idspast_key_valuesr]   inputs_embedsr_   rC   r   logits_to_keepr   )r   rope_scalingr=    original_max_position_embeddingsr   prepare_inputs_for_generation)r&   r   r   r]   r   r_   rC   r   r   r`   past_lengthmodel_inputss               r(   r   z-Phi3ForCausalLM.prepare_inputs_for_generation   s    $ (("dkk&R&RUV&VV(+KdkkJJJ"&J*,JJ 

+)')%)

 

 r)   )NNNNNTN)r4   r5   r6   r   r   r)   r(   r   r      s     %r)   r   c                       e Zd Zy)Phi3ForSequenceClassificationNr4   r5   r6   r   r)   r(   r   r          r)   r   c                       e Zd Zy)Phi3ForTokenClassificationNr   r   r)   r(   r   r     r   r)   r   )r   	Phi3Modelr   r   r   )Nr   ),rw   typingr   r   r7   torch.utils.checkpointr   activationsr   cache_utilsr   modeling_flash_attention_utilsr	   modeling_utilsr
   processing_utilsr   utilsr   mistral.modeling_mistralr   r   r   r   r   r   r   configuration_phi3r   
get_loggerr4   logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCModuler   rL   rN   r}   r   r   r   r   __all__r   r)   r(   <module>r      s      %    !   B 5 &    + 
		H	%8 )bii )$@B)BII B)J'* 'T0 &(*= &R	$D 		!> 	r)   