
    rh=                    b   d Z ddlZddlmZmZmZ ddlZddlmc m	Z
 ddlZddlmZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z,  e       rddlm-Z-  e*       rddl.m/Z/m0Z0 ddl1m2Z2 nd\  Z2Z0Z/ e)       r	ddl3m4Z4m5Z5 nd\  Z5Z4 e6e2e0e4e5e/f      Z7 e'jp                  e9      Z:	 	 	 d@deejv                  e<ejv                     df   dee=   deejv                     deejv                  e=f   fdZ> G d dej~                        Z@d ejv                  d!e=dejv                  fd"ZA G d# d$e      ZB G d% d&ej~                        ZC G d' d(eC      ZD G d) d*eC      ZEeCeDeEd+ZF G d, d-ej~                        ZG G d. d/ej~                        ZH G d0 d1ej~                        ZI G d2 d3e      ZJ G d4 d5e      ZKe% G d6 d7e              ZLeJeKd8ZMe% G d9 d:eL             ZN G d; d<eLe      ZO G d= d>eeL      ZPg d?ZQy)AzPyTorch Jamba model.    N)AnyOptionalUnion)nn   )ACT2FN)CacheDynamicCacheDynamicLayer)GenerationMixin)AttentionMaskConverter)!flash_attn_supports_top_left_maskis_flash_attn_available) GenericForSequenceClassificationGradientCheckpointingLayer)MoeCausalLMOutputWithPastMoeModelOutputWithPast)PreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)is_causal_conv1d_availableis_mamba_ssm_available   )JambaConfig)_flash_attention_forward)mamba_inner_fnselective_scan_fn)selective_state_update)NNN)causal_conv1d_fncausal_conv1d_updateNNrouter_logitsnum_expertsattention_maskreturnc                    | t        | t              syt        | t              rC| d   j                  }t        j                  | D cg c]  }|j                  |       c}d      }t        j                  j                  j                  d      }t        j                  ||d      \  }}	t        j                  j                  j                  |	|      }
|>t        j                  |
j                         d      }t        j                  |d      }n1|j                  \  }}|j                  d   ||z  z  }|dddddddf   j                  |||||f      j                  d||      j                        }t        j                   |
j                         |z  d      t        j                   |d      z  }|ddddddf   j                  ||||j                  d   f      j                  d|j                  d         j                  |      }t        j                   ||z  d      t        j                   |d      z  }|j                  d   t#        |j                  j$                        z  }t        j                   |dd|||j                  d   z   f   |j'                  d      z        }||z  S c c}w )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        router_logits:
            Logits from the `router`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   dimr   )
isinstancetupledevicetorchcattor   
functionalsoftmaxtopkone_hotmeanfloatshapeexpandreshapesumintindex	unsqueeze)r%   r&   top_kr'   compute_devicelayer_routerconcatenated_router_logitsrouting_weights_selected_expertsexpert_masktokens_per_expertrouter_prob_per_expert
batch_sizesequence_lengthnum_hidden_layersexpert_attention_mask router_per_expert_attention_maskrankoverall_losss                       {/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/jamba/modeling_jamba.pyload_balancing_loss_funcrR   G   s   : J}e$D-'&q)00%*YYANO\__^,OUV&
" hh))112LRT1UO**_eDA((%%--.>LK!JJ{'8'8':B "'O!C&4&:&:#
O6<<Q?JQ`D`a 4AtT12V&
OUKXYWR,R	 	 "IIk&7&7&9<Q&QWXY\a\e\e!q]
 
 4At+,V&
O_EZEZ[\E]^_WR..q12R	 	) "'?=]+]cd!ehmhqhq,!i
 "
   #c/*@*@*F*F&GGD99!TD?+@+@+C$CCCDG]GgGghiGjjL +%%c Ps   J<c                   ,     e Zd Zd fd	Zd Zd Z xZS )JambaRMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z;
        JambaRMSNorm is equivalent to T5LayerNorm
        N)super__init__r   	Parameterr0   onesweightvariance_epsilon)selfhidden_sizeeps	__class__s      rQ   rW   zJambaRMSNorm.__init__   s1     	ll5::k#:; #    c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S )N   r,   T)keepdim)	dtyper2   r0   float32powr7   rsqrtr[   rZ   )r\   hidden_statesinput_dtypevariances       rQ   forwardzJambaRMSNorm.forward   sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r`   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)r.   rZ   r9   r[   r\   s    rQ   
extra_reprzJambaRMSNorm.extra_repr   s*    ))*+6$2G2G1HIIr`   )gư>)__name__
__module____qualname__rW   rk   rn   __classcell__r_   s   @rQ   rT   rT      s    $;Jr`   rT   rh   n_repc                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r9   r:   r;   )rh   rt   batchnum_key_value_headsslenhead_dims         rQ   	repeat_kvrz      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr`   c                       e Zd ZdZdZdZdZej                  df fd	Z		 ddej                  dej                  dedeeeef      d	eej                  ej                  f   f
d
Zdej$                  fdZddee   d	efdZd	eeej                     eej                     f   fdZeddeeeej.                           d	dfd       Z xZS ) HybridMambaAttentionDynamicCachea  
    A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache
    (which has a constant shape regardless of seq_len).

    This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states`
    and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor
    For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`,
    while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors).
    For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors),
    while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`,
    and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`.
    NFc           
         t         
|   t               || _        |j                  | _        d| _        |j                  |j                  z  }|j                  }|j                  }g | _
        g | _        g | _        t        |j                        D ]  }| j                  |   dk(  r]| xj                  t        j                   |||||      gz  c_
        | xj                  t        j                   |||||      gz  c_        r| xj                  t        j"                  g g|z  |      gz  c_
        | xj                  t        j"                  g g|z  |      gz  c_        | j                  j%                  |        t        |j                        D 	cg c]  }	t        j"                  g g|z  |       c}	| _        t        |j                        D 	cg c]  }	t        j"                  g g|z  |       c}	| _        y c c}	w c c}	w )N)layer_classesFmambar/   rd   r/   )rV   rW   r   rd   layers_block_typehas_previous_statemamba_expandr]   mamba_d_statemamba_d_convconv_states
ssm_statestransformer_layersrangerL   r0   zerostensorappend	key_cachevalue_cache)r\   configrJ   rd   r/   intermediate_sizessm_state_sizeconv_kernel_sizeirE   r_   s             rQ   rW   z)HybridMambaAttentionDynamicCache.__init__   s   |4
!'!9!9"'"//&2D2DD--!.."$v//0 	2A%%a(G3  KK
,=?OX^fkl%   KK
,=~V\dij$    U\\2$2CF%S$TT ELL"
1B6$R#SS''..q1	2 SXX^XpXpRqrQ%,,tj'8HrTYZ`ZrZrTstqELL"
):6Jt sts   "H "H
key_statesvalue_states	layer_idxcache_kwargsr(   c                    | j                   |   j                  d   dk(  r|| j                   |<   || j                  |<   nft        j                  | j                   |   |gd      | j                   |<   t        j                  | j                  |   |gd      | j                  |<   | j                   |   | j                  |   fS )Nr,   r   rb   r*   )r   r9   r   r0   r1   )r\   r   r   r   r   s        rQ   updatez'HybridMambaAttentionDynamicCache.update   s     >>)$**2.!3(2DNN9%*6DY'(-		4>>)3Lj2Y_`(aDNN9%*/))T5E5Ei5PR^4_ef*gDY'~~i($*:*:9*EEEr`   beam_idxc                    t        t        | j                              D ]S  }| j                  |   j                  }| j                  |   j	                  d|j                  |            | j                  |<   | j                  |   j                  }| j                  |   j	                  d|j                  |            | j                  |<   | j                  |   j                  }| j                  |   j	                  d|j                  |            | j                  |<   | j                  |   j                  }| j                  |   j	                  d|j                  |            | j                  |<   V y)zDReorders the cache for beam search, given the selected beam indices.r   N)	r   lenr   r/   index_selectr2   r   r   r   )r\   r   r   r/   s       rQ   reorder_cachez.HybridMambaAttentionDynamicCache.reorder_cache   sD   s4>>23 		iI^^I.55F(,y(A(N(NqRZR]R]^dRe(fDNN9%%%i077F*.*:*:9*E*R*RSTV^VaVabhVi*jDY'%%i077F*.*:*:9*E*R*RSTV^VaVabhVi*jDY'__Y/66F)-)C)P)PQRT\T_T_`fTg)hDOOI&		ir`   c                     || j                   vr| j                   d   n|}t        | j                        |k  ry| j                  |   j                  d   S )zYReturns the sequence length of the cached states. A layer index can be optionally passed.r   )r   r   r   r9   )r\   r   s     rQ   get_seq_lengthz/HybridMambaAttentionDynamicCache.get_seq_length  sR     3<4CZCZ2ZD++A.`i	t~~)+~~i(..r22r`   c                     t        d      NzIHybridMambaAttentionDynamicCache does not have a legacy cache equivalent.NotImplementedErrorrm   s    rQ   to_legacy_cachez0HybridMambaAttentionDynamicCache.to_legacy_cache  s    !"mnnr`   past_key_valuesr
   c                     t        d      r   r   )clsr   s     rQ   from_legacy_cachez2HybridMambaAttentionDynamicCache.from_legacy_cache  s    !"mnnr`   N)r   )ro   rp   rq   __doc__r   r   is_compileabler0   float16rW   Tensorr=   r   dictstrr   r.   r   
LongTensorr   r   r   classmethodFloatTensorr   rr   rs   s   @rQ   r|   r|      s!    IKN16t u@ 26FLLF llF 	F
 tCH~.F 
u||U\\)	*F"ie&6&6 i3 3c 3ouU\\':E%,,<O'O!P o ouUEVEV?W9X0Y oes o or`   r|   c                   ,    e Zd ZdZddedee   f fdZ	 	 	 	 	 	 ddej                  deej                     deej                     dee   d	ed
edeej                     deej                  eej                     eeej                        f   fdZ xZS )JambaAttentionz
    Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer
    and "Generating Long Sequences with Sparse Transformers".
    r   r   c                 2   t         |           || _        || _        |-t        j                  d| j                  j                   d       |j                  | _        |j                  | _
        | j                  | j                  z  | _        |j                  | _        | j                  | j                  z  | _        d| _        |j                  | _        | j                  | j                  z  | j                  k7  r&t!        d| j                   d| j                   d      t#        j$                  | j                  | j                  | j                  z  d      | _        t#        j$                  | j                  | j                  | j                  z  d      | _        t#        j$                  | j                  | j                  | j                  z  d      | _        t#        j$                  | j                  | j                  z  | j                  d      | _        y )	NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).Fbias)rV   rW   r   r   loggerwarning_oncer_   ro   r]   num_attention_heads	num_headsry   rw   num_key_value_groups	is_causalattention_dropout
ValueErrorr   Linearq_projk_projv_projo_proj)r\   r   r   r_   s      rQ   rW   zJambaAttention.__init__"  s   " !8!8 9 :, , "--33((DNN:#)#=#= $(NNd6N6N$N!!'!9!9MMDNN*t/?/??QRVRbRbQc$T^^$4B8  ii 0 0$..4==2PW\]ii 0 0$2J2JT]]2Zafgii 0 0$2J2JT]]2Zafgii >@P@PW\]r`   rh   r'   position_idspast_key_valueoutput_attentions	use_cachecache_positionr(   c                    |j                         \  }}	}
| j                  |      }| j                  |      }| j                  |      }|j	                  ||	| j
                  | j                        j                  dd      }|j	                  ||	| j                  | j                        j                  dd      }|j	                  ||	| j                  | j                        j                  dd      }| |j                  ||| j                        \  }}t        || j                        }t        || j                        }t        j                  ||j                  dd            t        j                   | j                        z  }|#|d d d d d d d |j"                  d   f   }||z   }t$        j&                  j)                  |dt        j*                        j-                  |j.                        }t$        j&                  j1                  || j2                  | j4                        }t        j                  ||      }|j                         || j
                  |	| j                  fk7  r7t7        d|| j
                  |	| j                  f d	|j                                |j                  dd      j9                         }|j;                  ||	| j<                        }| j?                  |      }|sd }|||fS )
Nr   rb   r   r   r,   r+   rd   )ptrainingz `attn_output` should be of size z	, but is ) sizer   r   r   viewr   ry   	transposerw   r   r   rz   r   r0   matmulmathsqrtr9   r   r3   r4   re   r2   rd   dropoutr   r   r   
contiguousr;   r]   r   )r\   rh   r'   r   r   r   r   r   bszq_lenrE   query_statesr   r   attn_weightscausal_maskattn_outputs                    rQ   rk   zJambaAttention.forward?  s    &**,UA{{=1[[/
{{=1#((eT^^T]]S]]^_abc__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm%'5'<'<ZW[WeWe'f$J z4+D+DE
 t/H/HI||L*2F2Fq!2LMPTPYPYZ^ZgZgPhh%(Aq2HJ4D4DR4H2H)HIK'+5L }},,\r,WZZ[g[m[mn}},,\T=S=S^b^k^k,lll<>#t~~udmm!LL2CPTP]P]3^2_ `$$&') 
 "++Aq1<<>!))#ud6F6FGkk+. LL.88r`   r   NNNFFN)ro   rp   rq   r   r   r   r=   rW   r0   r   r   r|   boolr.   rk   rr   rs   s   @rQ   r   r     s    
^{ ^x} ^@ 2637EI"'5949||49 !.49 u//0	49
 !!AB49  49 49 !!1!1249 
u||Xell3XeELL>Q5RR	S49r`   r   c                        e Zd ZdZ fdZ	 	 	 	 	 	 ddej                  deej                     deej                     dee	   de
de
d	eej                     fd
Z xZS )JambaFlashAttention2aF  
    Jamba flash attention module. This module inherits from `JambaAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                 B    t        |   |i | t               | _        y r   )rV   rW   r   _flash_attn_uses_top_left_mask)r\   argskwargsr_   s      rQ   rW   zJambaFlashAttention2.__init__~  s#    $)&)
 /P.Q+r`   rh   r'   r   r   r   r   r   c                 J   |j                         \  }	}
}| j                  |      }| j                  |      }| j                  |      }|j	                  |	|
| j
                  | j                        }|j	                  |	|
| j                  | j                        j                  dd      }|j	                  |	|
| j                  | j                        j                  dd      }| |j                  ||| j                        \  }}t        || j                        }t        || j                        }| j                  sdn| j                  }|j                  }|j                   j"                  dk7  r|j                   j"                  nd}|t$        j&                  k(  rt%        j(                         r:t+        t$        d      rt%        j,                  |      nt%        j.                         }nMt+        | j0                  d      r| j0                  j2                  }n | j                  j4                  j                  }t6        j9                  d| d	       |j;                  |      }|j;                  |      }|j;                  |      }|j                  dd      }|j                  dd      }t=        |||||
|t?        | j0                  d
d       | j@                  | jB                  	      }|jE                  |	|
| jF                        jI                         }| jK                  |      }|sd }||fS )Nr   rb           mpscpuget_autocast_dtype_pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .sliding_window)r   r   r   use_top_left_mask)&r   r   r   r   r   r   ry   rw   r   r   r   rz   r   r   r   rd   r/   typer0   re   is_autocast_enabledhasattrr   get_autocast_gpu_dtyper   r   rZ   r   r   r2   r   getattrr   r   r;   r]   r   r   )r\   rh   r'   r   r   r   r   r   r   r   r   rE   r   r   r   dropout_rateri   device_typetarget_dtyper   r   s                        rQ   rk   zJambaFlashAttention2.forward  s    &**,UA{{=1[[/
{{=1
 $((eT^^T]]S__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm%'5'<'<ZW[WeWe'f$J z4+D+DE
 t/H/HI"&--sT5K5K
 #((2>2E2E2J2Je2Sl))..Y^%--'((* u&:; ,,[9557  &?@#{{BB#{{1177 >$ (??<8L#|4J'??<8L  ))!Q/
#--a3. "4;;0@$Gnn"AA

 "))#ud6F6FGRRTkk+. LL.88r`   r   )ro   rp   rq   r   rW   r0   r   r   r   r|   r   rk   rr   rs   s   @rQ   r   r   w  s    R 2637EI"'59R9||R9 !.R9 u//0	R9
 !!ABR9  R9 R9 !!1!12R9r`   r   c                       e Zd ZdZ	 	 	 	 	 	 ddej
                  deej
                     deej                     dee   de	de	deej                     d	e
ej
                  eej
                     ee
ej
                        f   f fd
Z xZS )JambaSdpaAttentionz
    Jamba attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
    `JambaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
    SDPA API.
    rh   r'   r   r   r   r   r   r(   c                    |r*t         j                  d       t        |   ||||||      S |j	                         \  }}	}
| j                  |      }| j                  |      }| j                  |      }|j                  ||	| j                  | j                        j                  dd      }|j                  ||	| j                  | j                        j                  dd      }|j                  ||	| j                  | j                        j                  dd      }| |j                  ||| j                        \  }}t        || j                         }t        || j                         }|}||d d d d d d d |j"                  d   f   }|j$                  j&                  dk(  r2|0|j)                         }|j)                         }|j)                         }| j*                  xr |d u xr |	dkD  }t,        j.                  j0                  j3                  ||||| j4                  r| j6                  nd|      }|j                  dd      j)                         }|j                  ||	| j8                        }| j;                  |      }|d |fS )	Na  JambaModel is using JambaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rh   r'   r   r   r   r   r   rb   r   cudar   )	attn_mask	dropout_pr   )r   r   rV   rk   r   r   r   r   r   r   ry   r   rw   r   r   rz   r   r9   r/   r   r   r   r0   r   r3   scaled_dot_product_attentionr   r   r]   r   )r\   rh   r'   r   r   r   r   r   r   r   rE   r   r   r   r   r   r   r_   s                    rQ   rk   zJambaSdpaAttention.forward  s}    [ 7?+-)-"3# #   &**,UA{{=1[[/
{{=1#((eT^^T]]S]]^_abc__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm%'5'<'<ZW[WeWe'f$Jz4+D+DE
 t/H/HI$%%aA/E1A1A"1E/E&EFK ##v-.2L'224L#..0J'224L
 NNH{d':Huqy	hh))FF!04d,,3 G 
 "++Aq1<<>!&&sE43C3CDkk+.D.00r`   r   )ro   rp   rq   r   r0   r   r   r   r|   r   r.   rk   rr   rs   s   @rQ   r   r     s     2637EI"'59G1||G1 !.G1 u//0	G1
 !!ABG1  G1 G1 !!1!12G1 
u||Xell3XeELL>Q5RR	SG1 G1r`   r   )eagerflash_attention_2sdpac                        e Zd ZdZdef fdZ	 	 d
dej                  dede	ej                     fdZd
dede	ej                     fdZ	 	 d
dede	ej                     fd	Z xZS )JambaMambaMixeru  
    Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    r   c           	         t         |           || _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _
        |j                  | _        |j                  | _        |j                  | _        t#        j$                  | j                  | j                  | j                  | j                  | j                  | j                  dz
        | _        |j(                  | _        t,        |j(                     | _        |j0                  | _        t#        j4                  | j                  | j                  dz  | j                         | _        t#        j4                  | j                  | j                  | j                  dz  z   d      | _        t#        j4                  | j                  | j                  d      | _        t=        j>                  d| j                  dz         d d d f   }|jA                  | j                  d      jC                         }t#        jD                  t=        jF                  |            | _$        t#        jD                  t=        jJ                  | j                              | _&        t#        j4                  | j                  | j                  | j                         | _'        tQ        | j                  |jR                        | _*        tQ        | j                  |jR                        | _+        tQ        | j                  |jR                        | _,        tZ        st\        j_                  d	       y y )
Nr   )in_channelsout_channelsr   kernel_sizegroupspaddingrb   r   FTr,   r^   ap  The fast path is not available because on of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d. If you want to use the naive implementation, set `use_mamba_kernels=False` in the model config)0rV   rW   r   r   r]   r   r   r   r   r   r   mamba_dt_ranktime_step_rankmamba_conv_biasuse_conv_biasmamba_proj_biasuse_biasr   Conv1dconv1d
hidden_act
activationr   actuse_mamba_kernelsuse_fast_kernelsr   in_projx_projdt_projr0   aranger:   r   rX   logA_logrY   Dout_projrT   rms_norm_epsdt_layernormb_layernormc_layernormis_fast_path_availabler   r   )r\   r   r   Ar_   s       rQ   rW   zJambaMambaMixer.__init__>  s   "!--$22 & 3 3!'!4!4v7I7I!I$22#33..ii..//##--))))A-
 !++&++, & 8 8 yy!1!143I3IA3MTXTaTabii 6 68K8KdNaNadeNe8elqryy!4!4d6L6LSWX LLD//!34T1W=HHT++R0;;=\\%))A,/
ejj)?)?@A		$"8"8$:J:JQUQ^Q^_()<)<&BUBUV'(;(;ATATU'(;(;ATATU%^ &r`   rh   cache_paramsr'   c                 4
   |j                   \  }}}|d uxrm |j                  xr_ |dk(  xrX |j                  | j                     j                   d   |j                  | j                     j                   d   cxk(  xr |k(  nc }| j                  |      j                  dd      }|j                  dd      \  }}	|||j                  d      z  }| j                  j                  j                  | j                  j                  j                  d      | j                  j                  j                  d            }
|ret        |j                  d      |j                  | j                     |
| j                  j                  | j                         }|j                  d      }n|dt"        j$                  j'                  || j(                  |j                   d   z
  df      }|j                  | j                     j+                  |       t-        ||
| j                  j                  | j                         }|||j                  d      z  }| j/                  |j                  dd            }t1        j2                  || j4                  | j6                  | j6                  gd      \  }}}| j9                  |      }| j;                  |      }| j=                  |      }| j>                  j                  j@                  }t1        jB                         5  t1        jD                  | j>                  j                  j@                        | j>                  j                  _         d d d        | j?                  |      j                  dd      }t1        jB                         5  || j>                  j                  _         d d d        t1        jF                  | jH                  jK                                }||jK                         nd }|r]tM        |j                  | j                     |d   |d   ||d d df   |d d df   | jN                  |	d   |d	
      j                  d      }n|tQ        ||||j                  dd      |j                  dd      | jN                  jK                         |	|dd

      \  }}|*|(|j                  | j                     j+                  |       | jS                  |j                  dd            }|S # 1 sw Y   xY w# 1 sw Y   WxY w)Nr   r   rb   r*   r,   )r  ).r   T)dt_softplus)delta_softplusreturn_last_state)*r9   r   r   r   r   r  r   chunkr?   r  rZ   r   r   r#   squeezer   r  r   r3   padr   copy_r"   r  r0   splitr  r   r!  r"  r#  r  datano_grad
zeros_likeexpr  r8   r!   r  r    r  )r\   rh   r&  r'   rJ   seq_lenrE   use_precomputed_statesprojected_statesgateconv_weightsr   ssm_parameters	time_stepBCtime_proj_biasdiscrete_time_stepr%  scan_outputs	ssm_statecontextualized_statess                         rQ   cuda_kernels_forwardz$JambaMambaMixer.cuda_kernels_forwardr  s    "/!4!4
GQ$ //1 ((8>>qA&&t~~6<<Q?	 	  <<6@@AF /44QA4>t%)N,D,DQ,GGM {{))..t{{/A/A/F/Fq/I4;;K]K]KbKbcdKef!0%%b)((8  M *33B7M' mm//@U@UXeXkXklnXo@oqr?st((8>>{K,]L$++JZJZgkgvgvwM%)N,D,DQ,GGM ]%<%<Q%BC++T00$2E2EtGZGZ[ac
	1a %%i0	QQ **//]]_ 	N%*%5%5dll6G6G6L6L%MDLL"	N!\\)4>>q!D]]_ 	4%3DLL"	4 YYtzz'')**3A3M--/SW!1''7f%"6*!Q$!Q$V  im  '8"Aq!Aq!#"&'#L) $)A''7==iH !%l.D.DQ.J K$$S	N 	N	4 	4s   AT T T
Tc           	      n   |j                   \  }}}|j                  }| j                  |      j                  dd      }|j	                  dd      \  }	}
||	|j                  d      z  }	t        |t              }|r8|j                  | j                     j                   d   |k(  r| j                  r(|j                  | j                     j                         }n|j                  | j                     }|j                  |	j                        }|j                  r|dk(  r|j                  | j                     j                   d   |k(  r|j                  | j                     }t!        j"                  |dd      }|	d d d d df   |d d d d df<   ||j                  | j                  <   t!        j$                  || j&                  j(                  d d dd d f   z  d      }	| j*                  r|	| j&                  j,                  z  }	| j/                  |	      j                  |      j                  d      }	nt0        j2                  j5                  |	| j6                  |	j                   d   z
  df      }||j                  | j                  <   | j/                  | j'                  |	      dd |f         }	n`t!        j8                  || j:                  | j<                  f|	j                  |      }| j/                  | j'                  |	      dd |f         }	||	|j                  d      z  }	| j?                  |	j                  dd            }t!        j@                  || jB                  | j<                  | j<                  gd      \  }}}| jE                  |      }| jG                  |      }| jI                  |      }| jK                  |      }t0        j2                  jM                  |      j                  dd      }t!        jN                  | jP                  jS                                }t!        jN                  |d d d d d d f   |d d d d d d d f   z        }|d d d d d d d f   |d d d d d d d f   jS                         z  }||	d d d d d d d f   jS                         z  }g }tU        |      D ]}  }|d d d d |d d f   |z  |d d d d |d d f   z   }t!        jV                  |j                  |      |d d |d d f   j                  d            }|jY                  |d d d d df           t!        jZ                  |d      }||	| j\                  d d d d f   z  z   }|| j/                  |
      z  }|r||j                  | j                  <   | j_                  |j                  dd            }|S )	Nr   rb   r*   r   r,   )shiftsdims.r   )0r9   rd   r  r   r+  r?   r-   r|   r   r   r   cloner2   r/   r   r   r0   rollr<   r  rZ   r  r   r  r   r3   r-  r   r   r   r   r  r/  r  r!  r"  r#  r  softplusr3  r  r8   r   r   r   stackr  r  )r\   input_statesr&  r'   rJ   r4  rE   rd   r6  rh   r7  r   r@  
conv_stater9  r:  r;  r<  r>  r%  
discrete_A
discrete_BdeltaB_ur?  r   scan_outputrA  s                              rQ   slow_forwardzJambaMambaMixer.slow_forward  s   !-!3!3
GQ""<<5??1E.44QA4>t%)N,D,DQ,GGM|-MN	00@FFqIZW}}(33DNNCIIK	(33DNNC	!]%9%9:I..7a< ,,T^^<BB1ES)55dnnE
"ZZ
2BG
'4Q1W'=
1a8$;E((8 %		*t{{7I7I!QPQ'7R*RXZ [%%!T[[%5%55M $ 7 : :5 A K KB O]]..!**]-@-@-DDaH
 <F((8 $])CC'M)R ST33T5H5HI$++5I !HHT[[%?XgX%NOM%)N,D,DQ,GGM ]%<%<Q%BC++T00$2E2EtGZGZ[ac
	1a %%i0	QQ!\\)4]]334FGQQRSUVW YYtzz'')**YYqq$!125G1aQU5VVW
'1a61dAq=9I9O9O9QQ
aAtm < B B DDw 	6A"1aA:.:XaAqj=QQI,,y||E':AaAgJ<P<PQS<TUKAq!G 45	6 kk,B7!]TVVD!TM5J%JK"TXXd^36?L##DNN3 !%k.C.CAq.I J$$r`   c                     | j                   rPt        r,d| j                  j                  j                  j
                  vrt        d      | j                  |||      S | j                  |||      S )Nr   zsFast Mamba kernels are not available. Make sure to they are installed and that the mamba module is on a CUDA device)	r  r$  r  rZ   r/   r   r   rB  rP  )r\   rh   r&  r'   s       rQ   rk   zJambaMambaMixer.forward2  sl       )V4;;;M;M;T;T;Y;Y-Y  J  ,,]L.YY  nMMr`   r$   )ro   rp   rq   r   r   rW   r0   r   r|   r   r   rB  rP  rk   rr   rs   s   @rQ   r  r  6  s    2{ 2n :>59	h%||h% 7h% !!1!12	h%VR%7W R%pxy~  zJ  zJ  qK R%p :>59	N 7N !!1!12	Nr`   r  c                   $     e Zd Z fdZd Z xZS )JambaMLPc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        t        |j                     | _        y NFr   )rV   rW   r   r]   r   r   r   	gate_projup_proj	down_projr   r  act_fnr\   r   r_   s     rQ   rW   zJambaMLP.__init__C  s    !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXV../r`   c                     | j                  | j                  | j                  |            | j                  |      z        }|S r   )rX  rY  rV  rW  )r\   xrX  s      rQ   rk   zJambaMLP.forwardM  s6    NN4;;t~~a/@#ADLLQRO#ST	r`   )ro   rp   rq   rW   rk   rr   rs   s   @rQ   rS  rS  B  s    0r`   rS  c                   ~     e Zd ZdZdef fdZdej                  deej                  ej                  f   fdZ	 xZ
S )JambaSparseMoeBlocka  
    This implementation is
    strictly equivalent to standard MoE with full capacity (no
    dropped tokens). It's faster since it formulates MoE operations
    in terms of block-sparse operations to accommodate imbalanced
    assignments of tokens to experts, whereas standard MoE either
    (1) drop tokens at the cost of reduced performance or (2) set
    capacity factor to number of experts and thus waste computation
    and memory on padding.
    r   c                    t         |           |j                  | _        |j                  | _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d      | _        t        j                  t        | j                        D cg c]  }t        |       c}      | _        y c c}w rU  )rV   rW   r]   
hidden_dimr   ffn_dimr&   num_experts_per_tokr@   r   r   router
ModuleListr   rS  experts)r\   r   rE   r_   s      rQ   rW   zJambaSparseMoeBlock.__init___  s     ,,//!--//
ii1A1AN}}dFVFV@W%X1hv&6%XY%Xs   +C
rh   r(   c                    |j                   \  }}}|j                  d|      }| j                  |      }t        j                  |dt
        j                        }t        j                  || j                  d      \  }}|j                  |j                        }t        j                  ||z  |f|j                  |j                        }t
        j                  j                  j                  || j                         j#                  ddd      }	t%        | j                         D ]  }
| j&                  |
   }t        j(                  |	|
         \  }}|j                   d   dk(  r@|d	|f   j+                  d|      } ||      |||d	f   z  }|j-                  d||j                  |j                                |j+                  |||      }||fS )
 r,   r   r   r*   )rd   r/   )num_classesrb   r   N)r9   r   rc  Fr4   r0   r8   r5   r@   r2   rd   r   r/   r   r3   r6   r&   permuter   re  wherer;   
index_add_)r\   rh   rJ   rK   r`  r%   rD   rF   final_hidden_statesrG   
expert_idxexpert_layeridxtop_xcurrent_statecurrent_hidden_statess                   rQ   rk   zJambaSparseMoeBlock.forwardi  s   2?2E2E/
OZ%**2z:M2))MqL,1JJ

XZ,[))),,]-@-@A#kk/):6m>Q>QZgZnZn
 hh))112BPTP`P`1aiijkmnpqr   0 01 	dJ<<
3L[%<=JC{{1~"
 *$+6>>r:NM$0$?/RWY\^bRbBc$c!  **1e5J5M5MmNaNa5bc	d  299*oWab"M11r`   )ro   rp   rq   r   r   rW   r0   r   r.   rk   rr   rs   s   @rQ   r^  r^  S  s?    	Z{ Z&2U\\ &2eELL%,,<V6W &2r`   r^  c                   4    e Zd Zdedef fdZ	 	 	 	 	 	 	 ddej                  deej                     deej                     dee
   dee   d	ee   d
ee   deej                     deej                  eeej                  ej                  f      f   fdZ xZS )JambaAttentionDecoderLayerr   r   c                 P   t         |           |j                  |   }t        |j                     ||      | _        |dkD  rt        nt        } ||      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        y )Nr   r
  )rV   rW   layers_num_expertsJAMBA_ATTENTION_CLASSES_attn_implementation	self_attnr^  rS  feed_forwardrT   r]   r   input_layernormpre_ff_layernormr\   r   r   r&   ffn_layer_classr_   s        rQ   rW   z#JambaAttentionDecoderLayer.__init__  s    //	:01L1LMfV_`1<q-h+F3+F,>,>FDWDWX ,V-?-?VEXEX Yr`   rh   r'   r   r   r   output_router_logitsr   r   r(   c	           	      *   |}	| j                  |      }| j                  |||||||      \  }}
}|	|z   }|}	| j                  |      }| j                  |      }t	        |t
              r|\  }}n|d}}|	|z   }|f}|r||
fz  }|r||fz  }|r||fz  }|S )  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`HybridMambaAttentionDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_router_logits (`bool`, *optional*):
                Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
                should not be returned during inference.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence.
        )rh   r'   r   r   r   r   r   N)r|  rz  r}  r{  r-   r.   )r\   rh   r'   r   r   r   r  r   r   residualself_attn_weightspresent_key_value
ff_outputsr%   outputss                  rQ   rk   z"JambaAttentionDecoderLayer.forward  s    < !,,];>Bnn')%)/) ?M ?
;(*; !=0 !--m<&&}5
j%(+5(M=+5t=M =0 ")++G)++G''Gr`   NNNFFFNro   rp   rq   r   r=   rW   r0   r   r   r   r|   r   r.   r   rk   rr   rs   s   @rQ   ru  ru    s    Z{ Zs Z 2637EI,1/4$)59D||D !.D u//0	D
 !!ABD $D>D 'tnD D>D !!1!12D 
u  (51B1BEDUDU1U+V"WW	XDr`   ru  c                   4    e Zd Zdedef fdZ	 	 	 	 	 	 	 ddej                  deej                     deej                     dee
   dee   d	ee   d
ee   deej                     deej                  eeej                  ej                  f      f   fdZ xZS )JambaMambaDecoderLayerr   r   c                 8   t         |           |j                  |   }t        ||      | _        |dkD  rt
        nt        } ||      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        y )N)r   r   r   r
  )rV   rW   rw  r  r   r^  rS  r{  rT   r]   r   r|  r}  r~  s        rQ   rW   zJambaMambaDecoderLayer.__init__  s}    //	:$FiH
1<q-h+F3+F,>,>FDWDWX ,V-?-?VEXEX Yr`   rh   r'   r   r   r   r  r   r   r(   c	                    |}	| j                  |      }| j                  |||      }d}
|	|z   }|}	| j                  |      }| j                  |      }t	        |t
              r|\  }}n|d}}|	|z   }|f}|r||
fz  }|r||fz  }|r||fz  }|S )r  )rh   r&  r'   N)r|  r   r}  r{  r-   r.   )r\   rh   r'   r   r   r   r  r   r   r  r  r  r%   r  s                 rQ   rk   zJambaMambaDecoderLayer.forward  s    < !,,];

'') # 

 ! !=0 !--m<&&}5
j%(+5(M=+5t=M =0 ")++G((G''Gr`   r  r  rs   s   @rQ   r  r    s    Z{ Zs Z 2637EI,1/4$)59A||A !.A u//0	A
 !!ABA $D>A 'tnA D>A !!1!12A 
u  (51B1BEDUDU1U+V"WW	XAr`   r  c                   >    e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZd Zy)	JambaPreTrainedModelr   modelTru  r  r   c                 &   | j                   j                  }t        |t        j                  t        j
                  f      rY|j                  j                  j                  d|       |j                  %|j                  j                  j                          y y t        |t        j                        rf|j                  j                  j                  d|       |j                  2|j                  j                  |j                     j                          y y t        |t              r&|j                  j                  j                  d       y t        |t              rt!        j"                  d|j$                  dz         d d d f   }|j'                  |j(                  d      j+                         }|j,                  j                  j/                  t!        j0                  |             |j2                  j                  j                  d       y y )Nr   )r7   stdg      ?r   r,   )r   initializer_ranger-   r   r   r  rZ   r0  normal_r   zero_	Embeddingpadding_idxrT   fill_r  r0   r  r   r:   r   r   r  r.  r  r  )r\   moduler  r%  s       rQ   _init_weightsz"JambaPreTrainedModel._init_weights?  sq   kk++fryy"))45MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> .-MM$$S)0Q 5 5 9:47CA1126AACALL##EIIaL1HHMM$	 1r`   N)ro   rp   rq   r   __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_is_statefulr   r`   rQ   r  r  3  s;    &*#57OP"3NL%r`   r  )	attentionr   c                   <    e Zd ZdZdef fdZee	 	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     dee   dee	j                     d	ee   d
ee   dee   dee   dee	j                     dee   defd              Zd Zd Z xZS )
JambaModelz
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`JambaDecoderLayer`]

    Args:
        config: JambaConfig
    r   c                 B   t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        g }t        |j                        D ]1  }t        |j                  |      }|j                   |||             3 t        j                  |      | _        |j                   | _        t#        |j                  |j$                        | _        d| _        | j+                          y )N)r   r
  F)rV   rW   pad_token_idr  
vocab_sizer   r  r]   embed_tokensr   rL   ALL_DECODER_LAYER_TYPESr   r   rd  layersry  rT   r   final_layernormgradient_checkpointing	post_init)r\   r   decoder_layersr   layer_classr_   s        rQ   rW   zJambaModel.__init___  s     !.. ++LL):):F<N<NPTP`P`av//0 	DA1&2J2J12MNK!!+f"BC	D mmN3$*$?$?!+F,>,>FDWDWX&+#r`   	input_idsr'   r   r   inputs_embedsr   r   output_hidden_statesr  r   r   r(   c                    ||n| j                   j                  }|	|	n| j                   j                  }	||n| j                   j                  }||n| j                   j                  }|d u |d uz  rt        d      | j                  r%| j                  r|rt        j                  d       d}|| j                  |      }|}|r|t        j                  d       |
.t        j                  |j                  d   |j                        }
||
j                  d      }| j!                  |||
      }| j#                  ||
      }|rdnd }|rdnd }|	rdnd }| j$                  D ]U  }t'        |t(              r|n|}|r||fz  } |||||||	||
	      }|d   }|r|d   	||d   fz  }|	sG|d
   M||d
   fz  }W | j+                  |      }|r||fz  }|r|j,                  sd|_        |sd n|}t/        |||||      S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzJamba requires an initialized `HybridMambaAttentionDynamicCache` to return a cache. None was provided, so no cache will be returned.r   r   r   r  )r'   r   r   r   r  r   r   r,   T)last_hidden_stater   rh   
attentionsr%   )r   r   r  r  r   r   r  r   r   r   r  r0   r  r9   r/   r?   _update_causal_mask_update_mamba_maskr  r-   r  r  r   r   )r\   r  r'   r   r   r  r   r   r  r  r   r   rh   r   
mamba_maskall_hidden_statesall_self_attnsall_router_logitsdecoder_layer
layer_masklayer_outputs
next_caches                         rQ   rk   zJambaModel.forwardr  so     2C1N-TXT_T_TqTq$8$D $++JjJj 	 %9$D $++JjJj 	 "+!6IDKK<Q<Q	-t";<YZZ&&4==Yj I  --i8M%0:
 !"\\-*=*=a*@I]I]^N)33A6L..~}n],,^^L
"6BD0d"6BD![[ 	>M'1-AW'X^iJ#!m%55!)))."3%9#-	M *!,M  #/"}Q'7&99N# $0%-*;)==%9	>< ,,];  -!11?#E#E15O.!*T
%+&+%+
 	
r`   c                    | j                   j                  dk(  r	|d|v r|S y |j                  |j                  }}t	        j
                  |      j                  }|j                  d   }|d   dz   }t	        j                  ||f|||      }	|dk7  rt	        j                  |	d      }	|	t	        j                  ||      |j                  dd      kD  z  }	|	d d d d d d f   j                  |j                  d   ddd      }	||	j                         }	|j                         d	k(  rd|j                  d   }
|	d
d |
f   j                  d      |d d d d d d f   j                  d      z  }|	d
d |
f   j!                  ||      |	d
d |
f<   | j                   j                  dk(  r0|.|j                  j"                  dv rt%        j&                  |	|      }	|	S )Nr   r   r   r,   )
fill_valuerd   r/   )diagonalr   r   rb   .r  )r   xpunpu)r   ry  rd   r/   r0   finfominr9   fulltriur  r;   r:   rF  r+   eqmasked_fillr   r   _unmask_unattended)r\   r'   input_tensorr   rd   r/   	min_dtyperK   target_lengthr   mask_lengthpadding_masks               rQ   r  zJambaModel._update_causal_mask  s   ;;++/BB)c^.C%%$**L,?,?vKK&**	&,,Q/&r*Q.jj/=!Ai_dmsta**[1=Ku||M&ANDZDZ[]_`Daaa!$a"23::<;M;Ma;PRSUWY[\%%++-K!!#q(,2226*3+<=@@EWXZ^`dfgWgHhHkHkloHpp1<S,;,=N1O1[1[\hjs1tC+-. KK,,6*%%**.DD
 1CCKQZ[Kr`   c                 R    |}|d   dkD  s|t        j                  |dk(        rd}|S )zv
        No need for zeroing states when
            1. Cached forward
            2. Attending to all inputs
        r   Nr   )r0   all)r\   r'   r   r  s       rQ   r  zJambaModel._update_mamba_mask  s7     $
!q ^%?EIIn`aNaDbJr`   )
NNNNNNNNNN)ro   rp   rq   r   r   rW   r   r   r   r0   r   r   r|   r   r   r   r   r   rk   r  r  rr   rs   s   @rQ   r  r  V  s1   { &  151537FJ59$(,0/3/359e
E,,-e
 !.e
 u//0	e

 ""BCe
   1 12e
 D>e
 $D>e
 'tne
 'tne
 !!1!12e
 +,e
 
 e
  e
N!F	r`   r  c                       e Zd ZdgZdef fdZd Zd Zee		 	 	 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     de
ej                     d	e
e   d
e
ej                     de
ej                     de
e   de
e   de
e   de
e   de
ej                     deeej                  f   dee   defd              Z	 	 	 	 	 	 	 ddZ xZS )JambaForCausalLMzlm_head.weightr   c                 N   t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        |j                  | _	        |j                  | _
        |j                  | _        | j                          y rU  )rV   rW   r  r  r  r   r   r]   lm_headrouter_aux_loss_coefr&   rb  r  rZ  s     rQ   rW   zJambaForCausalLM.__init__  s}     '
 ++yy!3!3V5F5FUS$*$?$?!!--#)#=#= r`   c                     || _         y r   r  )r\   decoders     rQ   set_decoderzJambaForCausalLM.set_decoder  s	    
r`   c                     | j                   S r   r  rm   s    rQ   get_decoderzJambaForCausalLM.get_decoder  s    zzr`   r  r'   r   r   r  labelsr   r   r  r  r   logits_to_keepr   r(   c                    ||n| j                   j                  }|
|
n| j                   j                  }
|	|	n| j                   j                  }	| j	                  ||||||||	|
|
      }|j
                  }t        |t              rt        | d      n|}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |}d}|
rYt        |j                  | j                  | j                  |      }|+|| j                   |j#                  |j$                        z  z  }t'        ||||j(                  |j*                  |j,                  |j                        S )aj  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, JambaForCausalLM

        >>> model = JambaForCausalLM.from_pretrained("ai21labs/Jamba-v0.1")
        >>> tokenizer = AutoTokenizer.from_pretrained("ai21labs/Jamba-v0.1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)
r  r'   r   r   r  r   r   r  r  r   )lossaux_losslogitsr   rh   r  r%   )r   r   r  r  r  r  r-   r=   slicer  loss_functionr  rR   r%   r&   rb  r  r2   r/   r   r   rh   r  )r\   r  r'   r   r   r  r  r   r   r  r  r   r  r   r  rh   slice_indicesr  r  r  s                       rQ   rk   zJambaForCausalLM.forward  s   R 2C1N-TXT_T_TqTq$8$D $++JjJj 	
 %9$D $++JjJj 	
 +/**)%+'/!5!5) +5 +
  118B>SV8W~ot4]kmA}a,?@A%4%%ffdooPPD/%%  ((	H !11HKK4LLL(#33!//))!//
 	
r`   c	           
      v   |d u }
|
sZ||d   |j                   d   k\  r|d d |j                   d    d f   }nc|j                   d   |j                   d   k7  rD|d d |f   }n:t        | j                  |j                   d   | j                  | j                        }|T|R|j                         j                  d      dz
  }|j                  |dk(  d       |
s|d d |j                   d    d f   }||
rd|i}nd|j                         i}|j                  |||||| j                  j                  |d       |S )Nr,   r   r   r   r  r  )r   r   r   r'   r  r  r   )r9   r|   r   rd   r/   longcumsummasked_fill_r   r   num_logits_to_keep)r\   r  r   r'   r  r  r   r   r   r   empty_past_kvmodel_inputss               rQ   prepare_inputs_for_generationz.JambaForCausalLM.prepare_inputs_for_generation|  sd    (4/ )!"%);;%a.*>*>q*A)A)C&CD	#~';';A'>>%a&78	>Y__Q/DKKO %,*>)..077;a?L%%n&91= +A	0B/B/D,DE $+];L')=)=)?@L ,#2&"0(<"&++"@"@"0
	
 r`   )NNNNNNNNNNNr   )NNNFNNT)ro   rp   rq   _tied_weights_keysr   rW   r  r  r   r   r   r0   r   r   r|   r   r   r   r=   r   r   r   rk   r  rr   rs   s   @rQ   r  r    s   *+	{ 	  151537FJ59-1$(,0/3/35934Y
E,,-Y
 !.Y
 u//0	Y

 ""BCY
   1 12Y
 ))*Y
 D>Y
 $D>Y
 'tnY
 'tnY
 !!1!12Y
 c5<</0Y
 +,Y
 
#Y
  Y
| ":r`   r  c                       e Zd Zy)JambaForSequenceClassificationN)ro   rp   rq   r  r`   rQ   r  r    s    r`   r  )r  r  r  r  )Nrb   N)Rr   r   typingr   r   r   r0   torch.nn.functionalr   r3   ri  torch.utils.checkpointactivationsr   cache_utilsr	   r
   r   
generationr   modeling_attn_mask_utilsr   modeling_flash_attention_utilsr   r   modeling_layersr   r   modeling_outputsr   r   modeling_utilsr   processing_utilsr   utilsr   r   r   r   utils.import_utilsr   r   configuration_jambar   r   &mamba_ssm.ops.selective_scan_interfacer   r    +mamba_ssm.ops.triton.selective_state_updater!   causal_conv1dr"   r#   r  r$  
get_loggerro   r   r   r.   r=   rR   ModulerT   rz   r|   r   r   r   rx  r  rS  r^  ru  r  r  r  r  r  r  __all__r  r`   rQ   <module>r	     s  (   ' '      ! < < ) > h R - & R R T , J XR@P=-~DD-7**.0@BVXfg 
 
		H	% "&
-1	T&uU\\':D@AT&#T& U\\*	T&
 5<<T&pJ299 J*	UU\\ 	U# 	U%,, 	UXou XoxW9RYY W9va9> a9JO1 O1f - HNbii HNXryy "<2")) <2~O!; OdL7 L^ %? % %< )CMcd  p% p phk+_ k\ b%EG[ a gr`   