
    rh                        d Z ddlZddlmZmZ ddlZddlZddlmZ ddlm	Z	 ddl
mZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZmZ ddlmZmZ ddlmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&  e#       rddl'm(Z( ddl)m*Z*  e       rddlm+Z+  e$jX                  e-      Z. G d dej^                        Z0d Z1d5dZ2 G d dej^                        Z3 G d dej^                        Z4dejj                  de6dejj                  fd Z7 G d! d"ej^                        Z8 G d# d$e8      Z9 G d% d&e8      Z:e8e9e:d'Z; G d( d)e      Z<e! G d* d+e             Z=e! G d, d-e=             Z> G d. d/e=e      Z? G d0 d1ee=      Z@ G d2 d3ee=      ZAg d4ZBy)6zPyTorch StableLM model.    N)OptionalUnion)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)AttentionMaskConverter)!flash_attn_supports_top_left_maskis_flash_attn_available) GenericForSequenceClassificationGenericForTokenClassificationGradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)PreTrainedModel)auto_docstringcan_return_tupleis_torch_flex_attn_availablelogging   )StableLmConfig)	BlockMask)make_flex_block_causal_mask)_flash_attention_forwardc                   ^     e Zd Zddef fdZ ej                         ed               Z xZ	S )StableLmRotaryEmbeddingconfigc                    t         |           t        |d      rUt        |j                  t
              r;|j                  j                  d|j                  j                  d            | _        nd| _        |j                  | _	        |j                  | _
        || _        t        | j                     | _        | j                  | j                  |      \  }| _        | j                  d|d       | j                   | _        y )Nrope_scaling	rope_typetypedefaultinv_freqF)
persistent)super__init__hasattr
isinstancer#   dictgetr$   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr!   r   rope_init_fnattention_scalingregister_bufferr'   original_inv_freq)selfr!   devicer'   	__class__s       /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/stablelm/modeling_stablelm.pyr*   z StableLmRotaryEmbedding.__init__@   s    6>*z&:M:Mt/T#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B!/?+/+<+<T[[&+Q($(ZeD!%    c                 b   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        j                  |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   r   mpscpuF)device_typeenabled   dim)dtype)r'   floatexpandshapetor7   r,   r%   strtorchautocast	transposecatcosr3   sinrD   )
r6   xposition_idsinv_freq_expandedposition_ids_expandedr?   freqsembrN   rO   s
             r9   forwardzStableLmRotaryEmbedding.forwardQ   sV    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk^^UC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s    BF%%F.N)
__name__
__module____qualname__r   r*   rJ   no_gradr   rV   __classcell__r8   s   @r9   r    r    ?   s3    /~ /" U]]_<  <r:   r    c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr<   rA   rB   )rG   rJ   rM   )rP   x1x2s      r9   rotate_halfra   b   sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r:   c                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezera   )qkrN   rO   rQ   unsqueeze_dimq_embedk_embeds           r9   apply_rotary_pos_embri   j   sY    ( --
&C
--
&C3w;q>C/0G3w;q>C/0GGr:   c                   $     e Zd Z fdZd Z xZS )StableLmMLPc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        t        |j                     | _        y NFbias)r)   r*   r!   hidden_sizeintermediate_sizer   Linear	gate_projup_proj	down_projr   
hidden_actact_fnr6   r!   r8   s     r9   r*   zStableLmMLP.__init__   s    !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXV../r:   c                     | j                  | j                  | j                  |            | j                  |      z        }|S rW   )ru   rw   rs   rt   )r6   rP   ru   s      r9   rV   zStableLmMLP.forward   s6    NN4;;t~~a/@#ADLLQRO#ST	r:   )rX   rY   rZ   r*   rV   r\   r]   s   @r9   rk   rk      s    0r:   rk   c                   @     e Zd Zd fd	Zdej
                  fdZ xZS )StableLmLayerNormPerHeadc           
          t         |           || _        || _        t	        j
                  t        | j                        D cg c]  }t	        j                  |||       c}      | _        y c c}w )N)epsro   )	r)   r*   rC   	num_headsr   
ModuleListrange	LayerNormnorms)r6   rC   r~   r}   ro   _r8   s         r9   r*   z!StableLmLayerNormPerHead.__init__   sT    "]]SXY]YgYgSh#iaBLL#D$I#ij
#is   A0hidden_statesc           	          t        j                  |dd      }t        j                  t        | j                  |      D cg c]  \  }} ||       c}}d      S c c}}w )Nr   rB   )rJ   splitrM   zipr   )r6   r   states_per_headsnorms       r9   rV   z StableLmLayerNormPerHead.forward   sM     !;;}aQ?yyTZZYiIjk2E$$}-kqrssks   A
)gh㈵>F)rX   rY   rZ   r*   rJ   TensorrV   r\   r]   s   @r9   r{   r{      s    ktU\\ tr:   r{   r   n_repreturnc                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rG   rF   reshape)r   r   batchnum_key_value_headsslenhead_dims         r9   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr:   c                   j    e Zd ZdZddedee   f fdZ	 	 	 	 	 	 	 ddej                  deej                     deej                     dee   d	ed
edeej                     deeej                  ej                  f      deej                  eej                     eeej                        f   fdZ xZS )StableLmAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr!   	layer_idxc                    t         |           || _        || _        |-t        j                  d| j                  j                   d       |j                  | _        |j                  | _
        | j                  | j                  z  | _        |j                  | _        | j                  | j                  z  | _        |j                  | _        t        | j                  |j                   z        | _        d| _        | j                  | j                  z  | j                  k7  r&t'        d| j                   d| j                   d      t)        j*                  | j                  | j                  | j                  z  |j,                        | _        t)        j*                  | j                  | j                  | j                  z  |j,                        | _        t)        j*                  | j                  | j                  | j                  z  |j,                        | _        t)        j*                  | j                  | j                  d      | _        |j6                  | _        | j6                  rbt9        | j                  | j                  |j:                  	      | _        t9        | j                  | j                  |j:                  	      | _        t)        j@                  |jB                        | _!        tE        | j                  
      | _#        y )NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).rn   Fr}   r!   )$r)   r*   r!   r   loggerwarning_oncer8   rX   rp   num_attention_headsr~   r   r   num_key_value_groups
rope_thetaintpartial_rotary_factorrotary_ndims	is_causal
ValueErrorr   rr   use_qkv_biasq_projk_projv_projo_projqk_layernormr{   layer_norm_epsq_layernormk_layernormDropoutattention_dropoutr    
rotary_embr6   r!   r   r8   s      r9   r*   zStableLmAttention.__init__   sR   " !8!8 9 :, , "--33((DNN:#)#=#= $(NNd6N6N$N! ++0L0L LMMMDNN*t/?/??QRVRbRbQc$T^^$4B8  ii 0 0$..4==2PW]WjWjkii 0 0$2J2JT]]2Zagatatuii 0 0$2J2JT]]2Zagatatuii 0 0$2B2BO"//7t~~[a[p[pqD7t77V=R=R D "$F,D,D!E1Er:   r   attention_maskrQ   past_key_valueoutput_attentions	use_cachecache_positionposition_embeddingsr   c	                 J   |j                         \  }	}
}| j                  |      }| j                  |      }| j                  |      }|j	                  |	|
| j
                  | j                        j                  dd      }|j	                  |	|
| j                  | j                        j                  dd      }|j	                  |	|
| j                  | j                        j                  dd      }| j                  r"| j                  |      }| j                  |      }|\  }}|dd | j                  f   |d| j                  d f   }}|dd | j                  f   |d| j                  d f   }}t        ||||      \  }}t        j                  ||fd      }t        j                  ||fd      }|2||| j                  |d}|j!                  ||| j"                  |      \  }}t%        || j&                        }t%        || j&                        }t        j(                  ||j                  dd            t+        j,                  | j                        z  }|#|d d d d d d d |j.                  d   f   }||z  }t0        j2                  j5                  |t        j6                  d	      j9                  |j:                        }| j=                  |      }t        j(                  ||      }|j                         |	| j
                  |
| j                  fk7  r7t?        d
|	| j
                  |
| j                  f d|j                                |j                  dd      jA                         }|jC                  |	|
| jD                        }| jG                  |      }|sd }||fS )Nr   rA   .r<   rB   rO   rN   partial_rotation_sizer   r   )rD   rC   z `attn_output` should be of size z	, but is )$sizer   r   r   viewr~   r   rL   r   r   r   r   r   ri   rJ   rM   updater   r   r   matmulmathsqrtrG   r   
functionalsoftmaxfloat32rH   rD   r   r   
contiguousr   rp   r   )r6   r   r   rQ   r   r   r   r   r   bszq_lenr   query_states
key_statesvalue_statesrN   rO   	query_rot
query_passkey_rotkey_passcache_kwargsattn_weightscausal_maskattn_outputs                            r9   rV   zStableLmAttention.forward   s    &**,UA{{=1[[/
{{=1#((eT^^T]]S]]^_abc__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm++L9L))*5J&S 1 1 1112d//112 	
 s/d////0sD--//0 
 2)Wc3O	7 yy)Z!8bAYY2;
% )-):):"0	L (6'<'<ZW[WeWegs't$J z4+D+DE
 t/H/HI||L*2F2Fq!2LMPTPYPYZ^ZgZgPhh%(Aq2HJ4D4DR4H2H)HIKK'L }},,\TV,WZZ[g[m[mn--l;ll<>#t~~udmm!LL2CPTP]P]3^2_ `$$&') 
 "++Aq1<<>!))#ud6F6FGkk+. LL((r:   rW   NNNFFNN)rX   rY   rZ   __doc__r   r   r   r*   rJ   r   
LongTensorr   booltuplerV   r\   r]   s   @r9   r   r      s   G&F~ &F(3- &FV 2637*."'59KOS)||S) !.S) u//0	S)
 !S)  S) S) !!1!12S) &eELL%,,,F&GHS) 
u||Xell3XeELL>Q5RR	SS)r:   r   c                   N    e Zd Z	 	 	 	 	 	 	 ddej                  deej                     deej                     dee   dededeej                     dee	ej                  ej                  f      d	e	ej                  eej                     ee	ej                        f   f fd
Z
 xZS )StableLmSdpaAttentionr   r   rQ   r   r   r   r   r   r   c	           
         |r,t         j                  d       t        |   ||||||||      S |j	                         \  }	}
}| j                  |      }| j                  |      }| j                  |      }|j                  |	|
| j                  | j                        j                  dd      }|j                  |	|
| j                  | j                        j                  dd      }|j                  |	|
| j                  | j                        j                  dd      }| j                  r"| j                  |      }| j                  |      }|\  }}|dd | j                   f   |d| j                   d f   }}|dd | j                   f   |d| j                   d f   }}t#        ||||      \  }}t%        j&                  ||fd      }t%        j&                  ||fd      }|2||| j                   |d}|j)                  ||| j*                  |      \  }}t-        || j.                        }t-        || j.                        }|}||d d d d d d d |j0                  d	   f   }|j2                  j4                  d
k(  r2|0|j7                         }|j7                         }|j7                         }t9        |d u xr |
dkD        }t$        j:                  j<                  j?                  ||||| j@                  r| jB                  jD                  nd|      }|j                  dd      j7                         }|j                  |	|
| jF                        }| jI                  |      }|d fS )Na  StableLmModel is using StableLmSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r   r   rQ   r   r   r   r   r   r   rA   .r<   rB   r   r   cuda        )	attn_mask	dropout_pr   )%r   r   r)   rV   r   r   r   r   r   r~   r   rL   r   r   r   r   r   ri   rJ   rM   r   r   r   r   rG   r7   r%   r   r   r   r   scaled_dot_product_attentiontrainingr   prp   r   )r6   r   r   rQ   r   r   r   r   r   r   r   r   r   r   r   rN   rO   r   r   r   r   r   r   r   r   r8   s                            r9   rV   zStableLmSdpaAttention.forward4  s    [ 7?+-)-"3#-$7 # 	 	 &**,UA{{=1[[/
{{=1#((eT^^T]]S]]^_abc__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm++L9L))*5J&S 1 1 1112d//112 	
 s/d////0sD--//0 
 2)Wc3O	7 yy)Z!8bAYY2;
% )-):):"0	L (6'<'<ZW[WeWegs't$J z4+D+DE
 t/H/HI$%(Aq2HJ4D4DR4H2H)HIK ##v-.2L'224L#..0J'224L
 ,:;	hh))FF!26--d,,..S G 
 "++Aq1<<>!&&sE43C3CDkk+.D  r:   r   )rX   rY   rZ   rJ   r   r   r   r   r   r   rV   r\   r]   s   @r9   r   r   3  s     2637*."'59KOh!||h! !.h! u//0	h!
 !h!  h! h! !!1!12h! &eELL%,,,F&GHh! 
u||Xell3XeELL>Q5RR	Sh! h!r:   r   c                   X    e Zd ZdZ fdZ	 	 	 	 	 	 	 ddej                  deej                     deej                     dee	   de
de
d	eej                     d
eeej                  ej                  f      deej                  eej                     eeej                        f   fdZ xZS )StableLmFlashAttention2aL  
    StableLM flash attention module. This module inherits from `StableLmAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                 B    t        |   |i | t               | _        y rW   )r)   r*   r   _flash_attn_uses_top_left_mask)r6   argskwargsr8   s      r9   r*   z StableLmFlashAttention2.__init__  s#    $)&)
 /P.Q+r:   r   r   rQ   r   r   r   r   r   r   c	                 \   d}|j                         \  }
}}| j                  |      }| j                  |      }| j                  |      }|j	                  |
|| j
                  | j                        j                  dd      }|j	                  |
|| j                  | j                        j                  dd      }|j	                  |
|| j                  | j                        j                  dd      }| j                  r"| j                  |      }| j                  |      }|\  }}|dd | j                  f   |d| j                  d f   }}|dd | j                  f   |d| j                  d f   }}t        ||||      \  }}t        j                  ||fd      }t        j                  ||fd      }|2||| j                  |d}|j!                  ||| j"                  |      \  }}|j                  dd      }|j                  dd      }|j                  dd      }| j$                  r| j&                  j(                  nd}t+        |||||||| j,                  | j.                  		      }|j1                  |
|| j2                        j5                         }| j7                  |      }|sd }|fS )
NFr   rA   .r<   rB   r   r   )rQ   dropoutuse_top_left_maskr   )r   r   r   r   r   r~   r   rL   r   r   r   r   r   ri   rJ   rM   r   r   r   r   r   r   r   r   r   rp   r   r   )r6   r   r   rQ   r   r   r   r   r   r   r   r   r   r   r   r   rN   rO   r   r   r   r   r   dropout_rater   r   s                             r9   rV   zStableLmFlashAttention2.forward  s    "%**,UA{{=1[[/
{{=1
 $((eT^^T]]S]]^_abc__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm++L9L))*5J&S 1 1 1112d//112 	
 s/d////0sD--//0  2)Wc3O	7 yy)Z!8bAYY2;
%)-):):"0	L (6'<'<ZW[WeWegs't$J $--a3))!Q/
#--a337==t--//c.% "AAnn

 "))#ud6F6FGRRTkk+. LL((r:   r   )rX   rY   rZ   r   r*   rJ   r   r   r   r   r   r   rV   r\   r]   s   @r9   r   r     s    R 6:37*."'59KOU)||U) !!1!12U) u//0	U)
 !U)  U) U) !!1!12U) &eELL%,,,F&GHU) 
u||Xell3XeELL>Q5RR	SU)r:   r   )eagersdpaflash_attention_2c                       e Zd Zdedef fdZ	 	 	 	 	 	 	 ddej                  deej                     deej                     dee
ej                        dee   d	ee   d
eej                     dee
ej                  ej                  f      de
ej                  ee
ej                  ej                  f      f   fdZ xZS )StableLmDecoderLayerr!   r   c                    t         |           |j                  | _        |j                  | _        t	        |j
                     ||      | _        t        |      | _        t        j                  |j                  |j                        | _        d | _        | j                  s0t        j                  |j                  |j                        | _        t        j                  |j                        | _        y )N)r   r   )r)   r*   use_parallel_residualrp   ATTENTION_CLASSES_attn_implementation	self_attnrk   mlpr   r   r   input_layernormpost_attention_layernormr   hidden_dropoutr   r   s      r9   r*   zStableLmDecoderLayer.__init__  s    %+%A%A"!--*6+F+FGZcdv&!||F,>,>FDYDYZ(,%)),.LL9K9KQWQfQf,gD)zz&"7"78r:   r   r   rQ   r   r   r   r   r   r   c	           
      ^   |}	| j                  |      }| j                  ||||||||      \  }
}| j                  r+| j                  |      }| j	                  |      }|	|
z   |z   }n;|	|
z   }	| j                  | j                  |	            }| j	                  |      }|	|z   }|f}|r||fz  }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
                Indices of positions of each input sequence tokens in the position embeddings. Selected in the range
                `[0, config.n_positions - 1]`.

                [What are position IDs?](../glossary#position-ids)
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*):
                cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        r   )r   r   r   r   r   r   )r6   r   r   rQ   r   r   r   r   r   residualself_attn_outputself_attn_weights
mlp_outputoutputss                 r9   rV   zStableLmDecoderLayer.forward  s    H !,,]; /3nn')%)/) 3 /= 	/
++ %% -0Jj1J$'77*DM  "22H$"?"?"IJJj1J$z1M ")++Gr:   r   )rX   rY   rZ   r   r   r*   rJ   r   r   r   r   r   FloatTensorrV   r\   r]   s   @r9   r   r     s   
9~ 
9# 
9 26378<,1$)59KOI||I !.I u//0	I
 !u||!45I $D>I D>I !!1!12I &eELL%,,,F&GHI 
u  (51B1BEDUDU1U+V"WW	XIr:   r   c                   <    e Zd ZU eed<   dZdZdgZdZdZ	dZ
dZd Zy)StableLmPreTrainedModelr!   modelTr   past_key_valuesc                    | j                   j                  }t        |t        j                        rY|j
                  j                  j                  d|       |j                  %|j                  j                  j                          y y t        |t        j                        rf|j
                  j                  j                  d|       |j                  2|j
                  j                  |j                     j                          y y t        |t        j                        rJ|j
                  j                  j                  d       |j                  j                  j                          y y )Nr   )meanstdg      ?)r!   initializer_ranger,   r   rr   weightdatanormal_ro   zero_	Embeddingpadding_idxr   fill_)r6   moduler  s      r9   _init_weightsz%StableLmPreTrainedModel._init_weightsr  s    kk++fbii(MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> .-MM$$S)KK""$ .r:   N)rX   rY   rZ   r   __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraphr   r:   r9   r  r  f  s9    &*#/0"3N!%r:   r  c                       e Zd ZdZdef fdZee	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     dee   dee	j                     d	ee   d
ee   dee   dee	j                     defd              Z	 ddee	j                  df   de	j                  de	j                  ded
ef
dZede	j                  dedede	j*                  de	j                  defd       Z xZS )StableLmModelz
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`StableLmDecoderLayer`]

    Args:
        config: StableLmConfig
    r!   c           	      B   t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        j                  |j                  |j                        | _        t#        |      | _        |j&                  | _        d| _        | j+                          y c c}w )Nr   r   F)r)   r*   pad_token_idr  
vocab_sizer   r  rp   embed_tokensr   r   num_hidden_layersr   layersr   r   r   r    r   r   gradient_checkpointing	post_initr   s      r9   r*   zStableLmModel.__init__  s     !.. ++LL):):F<N<NPTP`P`ammFKFLdLdFef!&)4f
 LL!3!39N9NO	1@$*$?$?!&+# gs   D	input_idsr   rQ   r  inputs_embedsr   r   output_hidden_statesr   r   c
                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|d u |d uz  rt	        d      | j
                  r%| j                  r|rt        j                  d       d}t        |t        d       t        f      st	        d      |r|
t               }|| j                  |      }|	F||j                         nd}
t        j                   |
|
|j"                  d   z   |j$                        }	||	j'                  d      }| j)                  |||	||      }|}| j+                  ||      }|rdnd }|rdnd }| j,                  D ]+  }|r||fz  } ||||||||	|	      }|d   }|s#||d   fz  }- | j/                  |      }|r||fz  }t1        ||||
      S )Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FzBThe `past_key_values` should be either a `Cache` object or `None`.r   r   r7   r  )r   rQ   r   r   r   r   r   )last_hidden_stater  r   
attentions)r!   r   r'  r   r   r#  r   r   r   r,   r%   r   r	   r   get_seq_lengthrJ   arangerG   r7   rc   _update_causal_maskr   r"  r   r   )r6   r%  r   rQ   r  r&  r   r   r'  r   past_seen_tokensr   r   r   all_hidden_statesall_self_attnsdecoder_layerlayer_outputss                     r9   rV   zStableLmModel.forward  s0    2C1N-TXT_T_TqTq$8$D $++JjJj 	 "+!6IDKK<Q<Q	-t";<YZZ&&4==##p "	 /DJ+>?abb0*nO  --i8M!CRC^==?de"\\ "2]5H5H5K"KTaThThN )33A6L..M>?L]
 & #oom\J #7BD0d![[ 	6M#!m%55!)*)."3#-$7	M *!,M =#3"55%	6( 		-0  -!11&+++%	
 	
r:   r   input_tensorc           	         | j                   j                  dk(  r||dk(  j                         r|S y | j                   j                  dk(  r't        |t        j
                        rt        |      }|S ||j                         nd}||j                  nd}| j                   j                  dk(  r(|s&|s$t        j                  |||| j                        ry |j                  }|j                  d   }	|r|j                         }
n1t        |t        j
                        r|j                  d	   n||	z   dz   }
| j                  ||	|
|||j                  d   
      }| j                   j                  dk(  rQ|O|j                   j"                  dv r7|s5t	        j$                  |      j&                  }t        j(                  ||      }|S )Nr   r   flex_attentionr   Fr   )r&  past_key_values_lengthis_trainingr   r<   )sequence_lengthtarget_lengthrD   r   
batch_size)r   xpunpu)r!   r   anyr,   rJ   r   r   r,  is_compileabler   _ignore_causal_mask_sdpar   rD   rG   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionr7   r%   finfomin_unmask_unattended)r6   r   r4  r   r  r   r/  using_compilable_cacherD   r9  r:  r   	min_dtypes                r9   r.  z!StableLmModel._update_causal_mask  s    ;;++/BB)~/D.I.I.K%%;;++/??.%,,7!<^!L!!
 @O?Z?99;`aCRC^!?!?di ;;++v5>T]n%>>*'7 MM	 ""&,,Q/!+??AM nell; $$R(%7!;  PP+')#))!, Q 
 KK,,6*%%**.DD%
 E*..I0CCKQZ[Kr:   r9  r:  rD   r;  c                    | | j                         dk(  r| }|S t        j                  |      j                  }t        j                  ||f|||j
                        }|dk7  rt        j                  |d      }|t        j                  ||j
                        |j                  dd      kD  z  }|ddddddf   j                  |ddd      }| |j                         }| j                  d   }	|ddddddd|	f   | ddddddf   j                  |j
                        z   }
|
dk(  }
|ddddddd|	f   j                  |
|      |ddddddd|	f<   |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        N   )
fill_valuerD   r7   r   )diagonalr)  r<   r   )rC   rJ   rC  rD  fullr7   triur-  r   rF   clonerG   rH   masked_fill)r   r9  r:  rD   r   r;  r   r   rG  mask_lengthpadding_masks              r9   rB  zCStableLmModel._prepare_4d_causal_attention_mask_with_cache_position>  s   > %.*<*<*>!*C(K* ' E*..I** -0Ye\j\q\qK !##jjqA5<<n>S>STWeWmWmnprsWtttK%dD!Q&67>>z1bRTUK))//1,2226*1aL[L+@ANSTVZ\`bcScDdDgDg&&E    ,q05@Aq,;,AV5W5c5c )6Aq!\k\12 r:   )	NNNNNNNNN)F)rX   rY   rZ   r   r   r*   r   r   r   rJ   r   r   r   r   r   r   rV   r   r.  staticmethodr   rD   rB  r\   r]   s   @r9   r  r    s   ~ "  151537+/59$(,0/359Z
E,,-Z
 !.Z
 u//0	Z

 "%Z
   1 12Z
 D>Z
 $D>Z
 'tnZ
 !!1!12Z
 
!Z
  Z
F #(BellK78B llB 	B
 B  BH 444 4 {{	4
 4 4 4r:   r  c                   f    e Zd ZdgZ fdZd Zd Zee	 	 	 	 	 	 	 	 	 	 	 dde	e
j                     de	e
j                     de	e
j                     de	e   d	e	e
j                     d
e	e
j                     de	e   de	e   de	e   de	e
j                     deee
j                  f   defd              Z xZS )StableLmForCausalLMzlm_head.weightc                     t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        | j                          y rm   )
r)   r*   r  r  r  r   rr   rp   lm_headr$  rx   s     r9   r*   zStableLmForCausalLM.__init__|  sU     "6*
 ++yy!3!3V5F5FUS 	r:   c                     || _         y rW   r  )r6   decoders     r9   set_decoderzStableLmForCausalLM.set_decoder  s	    
r:   c                     | j                   S rW   rX  )r6   s    r9   get_decoderzStableLmForCausalLM.get_decoder  s    zzr:   r%  r   rQ   r  r&  labelsr   r   r'  r   logits_to_keepr   c                    ||n| j                   j                  }|	|	n| j                   j                  }	| j                  ||||||||	|
	      }|j                  }t        |t              rt        | d      n|}| j                  |dd|ddf         }d}|* | j                  ||fd| j                   j                  i|}t        |||j                  |j                  |j                        S )ui  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, StableLmForCausalLM

        >>> model = StableLmForCausalLM.from_pretrained("adept/persimmon-8b-base")
        >>> tokenizer = AutoTokenizer.from_pretrained("adept/persimmon-8b-base")

        >>> prompt = "human: Hey, what should I eat for dinner?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        'human: Hey, what should I eat for dinner?\n\ncat: 🐱\n\nhuman: 😐\n\n'
        ```N)	r%  r   rQ   r  r&  r   r   r'  r   r  )losslogitsr  r   r+  )r!   r   r'  r  r*  r,   r   slicerV  loss_functionr  r   r  r   r+  )r6   r%  r   rQ   r  r&  r]  r   r   r'  r   r^  r   r   r   slice_indicesra  r`  s                     r9   rV   zStableLmForCausalLM.forward  s   R 2C1N-TXT_T_TqTq$8$D $++JjJj 	
 ,0::)%+'/!5) ,6 
,
  118B>SV8W~ot4]kmA}a,?@A%4%%  ;;11 	D &#33!//))
 	
r:   )NNNNNNNNNNr   )rX   rY   rZ   _tied_weights_keysr*   rZ  r\  r   r   r   rJ   r   r   r   r   r   r   r   r   rV   r\   r]   s   @r9   rT  rT  x  s<   *+  151537+/59-1$(,0/35934L
E,,-L
 !.L
 u//0	L

 "%L
   1 12L
 ))*L
 D>L
 $D>L
 'tnL
 !!1!12L
 c5<</0L
 
 L
  L
r:   rT  c                       e Zd Zy)!StableLmForSequenceClassificationNrX   rY   rZ   r  r:   r9   rg  rg        r:   rg  c                       e Zd Zy)StableLmForTokenClassificationNrh  r  r:   r9   rk  rk    ri  r:   rk  )rT  r  r  rg  rk  )Nr   )Cr   r   typingr   r   rJ   torch.utils.checkpointr   activationsr   cache_utilsr   r	   
generationr
   modeling_attn_mask_utilsr   modeling_flash_attention_utilsr   r   modeling_layersr   r   r   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   utilsr   r   r   r   configuration_stablelmr   !torch.nn.attention.flex_attentionr   integrations.flex_attentionr   r   
get_loggerrX   r   Moduler    ra   ri   rk   r{   r   r   r   r   r   r   r   r   r  r  rT  rg  rk  __all__r  r:   r9   <module>r~     s  (   "    ! . ) > h 
 L - \ \ 2  !;J J 
		H	%<bii <F(8"))  tryy t 	UU\\ 	U# 	U%,, 	U~)		 ~)Bi!- i!Xd)/ d)P !0 V5 Vr %o % %4 r+ r rld
1? d
N h(HJa g b%BD[ ar:   