
    rh                        d Z ddlZddlmZmZmZ ddlZddlmc m	Z
 ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZ ddlmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-m.Z.  e        rddl/m0Z0  G d de      Z1 G d de'      Z2 G d de(      Z3	 	 	 d=dejh                  dejj                  dejj                  dejj                  deejj                  d f   d!ee6   d"ee6   d#eejj                     d$e7ejj                  ejj                  f   fd%Z8 e       Z9e8e9d&<    G d' d(ejh                        Z: G d) d*e%      Z; G d+ d,ejh                        Z< G d- d.e      Z= G d/ d0e&      Z> G d1 d2e.      Z?	 	 	 	 d>d3eejj                  e7ejj                     df   d4ee@   d5ee@   d6e@deejj                     d$eejj                  e@f   fd7ZA G d8 d9e-      ZB G d: d;e$      ZCg d<ZDy)?zPyTorch Doge model.    N)CallableOptionalUnion)nn   )ACT2FN)Cache)PretrainedConfig)compile_friendly_flex_attention)GradientCheckpointingLayer)MoeCausalLMOutputWithPastMoeModelOutputWithPast)rope_config_validation)AttentionInterface)Unpack)TransformersKwargsis_torch_flex_attn_available)OutputRecorder   )LlamaForSequenceClassificationLlamaMLPLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbeddingapply_rotary_pos_embeager_attention_forward	repeat_kv)MixtralForCausalLMMixtralModel)	BlockMaskc                        e Zd ZdZdZdgZi dddddddd	d
d	dddddddddddddddd	ddddddZdgdgfddgdgfdgdgfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z xZ	S ) 
DogeConfiga   
    This is the configuration class to store the configuration of a [`DogeModel`]. It is used to instantiate an Doge
    model according to the specified arguments, defining the model architecture like [SmallDoge/Doge-320M](https://huggingface.co/SmallDoge/Doge-320M).

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 32768):
            Vocabulary size of the Doge2 model. Defines the number of different tokens that can be represented by the `inputs_ids` passed when calling [`DogeModel`]
        hidden_size (`int`, *optional*, defaults to 1024):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 2048):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        hidden_dropout (`float`, *optional*, defaults to 0.0):
            Dropout probability for each sequence transformation and state transformation module.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether the model's input and output word embeddings should be tied.
        max_position_embeddings (`int`, *optional*, defaults to 2048):
            The maximum sequence length that this model might ever be used with.
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        rope_scaling (`Dict`, *optional*):
            Dictionary containing the scaling configuration for the RoPE embeddings.
            NOTE: if you apply new rope type and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value accordingly.
            Doge family of small models use `{ 'rope_type': 'dynamic', 'factor': 4.0, 'original_max_position_embeddings': 2048 }` as the default value.
            Expected contents:
                `rope_type` (`str`):
                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', 'llama3'], with 'default' being the original RoPE implementation.
                `factor` (`float`, *optional*):
                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings.
                    In most scaling types, a `factor` of x will enable the model to handle sequences of length x * original maximum pre-trained length.
                `original_max_position_embeddings` (`int`, *optional*):
                    Used with 'dynamic', 'longrope' and 'llama3'.
                    The original max position embeddings used during pretraining.
                `attention_factor` (`float`, *optional*):
                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
                    computation.
                    If unspecified, it defaults to value recommended by the implementation, using the `factor` field to infer the suggested value.
                `beta_fast` (`float`, *optional*):
                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
                    ramp function. If unspecified, it defaults to 32.
                `beta_slow` (`float`, *optional*):
                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
                    ramp function. If unspecified, it defaults to 1.
                `short_factor` (`List[float]`, *optional*):
                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<`original_max_position_embeddings`).
                    Must be a list of numbers with the same length as the hidden size divided by the number of attention heads divided by 2
                `long_factor` (`List[float]`, *optional*):
                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<`original_max_position_embeddings`).
                    Must be a list of numbers with the same length as the hidden size divided by the number of attention heads divided by 2
                `low_freq_factor` (`float`, *optional*):
                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
                `high_freq_factor` (`float`, *optional*):
                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
        num_attention_heads (`int`, *optional*, defaults to 8):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention.
            If `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
            When converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed by meanpooling all the original heads within that group.
            For more details checkout [this paper](https://arxiv.org/pdf/2305.13245.pdf).
            If it is not specified, will default to `num_attention_heads`.
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        mlp_bias (`bool`, *optional*, defaults to `False`):
            Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.
        sliding_window (`int`, *optional*):
            Sliding window attention window size. If not specified, will default to `None`.
        keep_window_size (`int`, *optional*, defaults to 2048):
            The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
        is_moe (`bool`, *optional*, defaults to `False`):
            Whether to use the Cross Domain Mixture of Experts, if `True`, the MoE will inherit the MLP to initialize.
        num_experts (`int`, *optional*, defaults to 16384):
            Number of routed experts in the model. This is only used when `is_moe=True`.
        num_experts_per_tok (`int`, *optional*, defaults to 64):
            Number of selected experts to route per-token.
        norm_topk_prob (`bool`, *optional*, defaults to `False`):
            Whether to normalize the topk probabilities.
        output_router_logits (`bool`, *optional*, defaults to `False`):
            Whether or not the router logits should be returned by the model. Enabling this will also
            allow the model to output the auxiliary loss, including load balancing loss and router z-loss.
        router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
            The aux loss factor for the total loss.

    ```python
    >>> from transformers import DogeConfig, DogeModel

    >>> # Initializing a Doge-320M style configuration
    >>> configuration = DogeConfig()

    >>> # Initializing a model from the Doge-320M style configuration
    >>> model = DogeModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```dogepast_key_valueszlayers.*.self_attn.q_projcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.dt_projrowwisezlayers.*.self_attn.o_projzlayers.*.input_layernorm.weightsequence_parallelzlayers.*.input_residual.weightz(layers.*.post_attention_layernorm.weightz'layers.*.post_attention_residual.weightznorm.weightzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projzlayers.*.mlp.router_gatecolwise_repzlayers.*.mlp.down_embedrowwise_repzlayers.*.mlp.up_embed	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormc                    || _         || _        || _        || _        || _        || _        || _        || _        |	| _        || _	        || _
        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        | j                  *d| j                  v r| j                  d   | j                  d<   t3        |        ||| _        t5        | l  dd|
i| y )Ntype	rope_typetie_word_embeddings )
vocab_sizehidden_sizeintermediate_sizenum_hidden_layershidden_dropout
hidden_actinitializer_rangerms_norm_eps	use_cachemax_position_embeddings
rope_thetarope_scalingnum_attention_headsnum_key_value_headsattention_biasattention_dropoutmlp_biassliding_windowkeep_window_sizeis_moenum_expertsnum_experts_per_toknorm_topk_proboutput_router_logitsrouter_aux_loss_coefr   super__init__)selfr6   r7   r8   r9   r:   r;   r<   r=   r>   r4   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   kwargs	__class__s                               x/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/doge/modular_doge.pyrP   zDogeConfig.__init__   s2   < %&!2!2,$!2("'>$$(#6 #6 ,!2 , 0&#6 ,$8!$8! (Vt7H7H-H-1->->v-FDk*t$ &':D$ 	
 3	
	
    )i   i                  silug{Gz?gư>TFrV   g     @N   NFrX   FNrV   Fi @  @   FFgMbP?)
__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planrP   __classcell__rS   s   @rT   r"   r"   6   su   n` J#4"5#Y#Y 	$Y 	%i	
 	$Y 	*+> 	)*= 	34G 	23F 	* 	!) 		 	!) 	#M 	"=  	 !& &(9:#%568IJ!"_$56 ! $ ""7G
 G
rU   r"   c                       e Zd Zy)DogeRMSNormNr\   r]   r^   r5   rU   rT   rg   rg         rU   rg   c                       e Zd Zy)DogeRotaryEmbeddingNrh   r5   rU   rT   rk   rk     ri   rU   rk   modulequerykeyvaluer-   r    scalingsoftcap	head_maskreturnc           
      2   d }	d t        |t              r|}	n|d d d d d d d |j                  d   f   fd}
t        ||||
|	d|d      \  }}|j	                  |j
                        }|j                  dd      j                         }||fS )Nc                     t        j                  | z        z  } | |   |   |   |   z   } | |   |   d   d   z   } | S )Nr   )torchtanh)score	batch_idxhead_idxq_idxkv_idxcausal_maskrr   rq   s        rT   	score_modz)flex_attention_forward.<locals>.score_mod)  sm    ejj99E"K	28<UCFKKE Ii0:1=a@@ErU   T)r   
block_mask
enable_gqascale
return_lse   r   )
isinstancer    shaper   todtype	transpose
contiguous)rl   rm   rn   ro   r-   rp   rq   rr   rR   r   r   attn_outputattention_weightsr~   s         ``     @rT   flex_attention_forwardr     s     JK.),#
$!!Q?SYYr]?":; &E &"K" *,,U[[9''1-88:K)))rU   doge_flex_attentionc                       e Zd Zddedee   f fdZ	 	 	 ddej                  de	ej                  ej                  f   deej                     dee
   deej                     d	e	ej                  eej                     ee	ej                        f   fd
Z	 	 ddej                  dej                  dedeej                     fdZ xZS )DogeAttentionconfig	layer_idxc                    t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        |j                  | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j&                  t)        j*                  |j                              | _        t        j                  |j                  | j                  z  |j                  |j                        | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        t3        | j                  |j4                        | _        t3        | j                  |j4                        | _        y )Nhead_dimg      ࿩biaseps)rO   rP   r   r   getattrr7   rB   r   rC   num_key_value_groupsrp   rE   rH   r   LinearrD   q_projk_projv_proj	Parameterrw   zerosAdt_projo_projrg   r=   q_normk_normrQ   r   r   rS   s      rT   rP   zDogeAttention.__init__J  s   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9 & 7 7ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ekk&*D*DEFyy&&68R8RY_YnYn
 ii&&68J8JQWQfQf
 "$--V5H5HI!$--V5H5HIrU   r,   position_embeddingsr-   past_key_valuecache_positionrs   c                    |j                   d d }g |d| j                  }| j                  | j                  |      j	                  |            j                  dd      }	| j                  | j                  |      j	                  |            j                  dd      }
| j                  |      j	                  |      j                  dd      }|\  }}t        |	|
||      \  }	}
|'|||d}|j                  |
|| j                  |      \  }
}| j                  |j                  dd      j                  |j                   d   |j                   d   d            }t        j                  | j                   t#        j$                  |      z        j                  dd      }| j'                  ||| j(                  |      }t+        || j,                        }t.        }| j0                  j2                  dk7  rt4        | j0                  j2                     } || |	|
|f|| j6                  sd	n| j8                  | j:                  d
|\  }} |j                  g |d j=                         }| j?                  |      }||fS )Nr   r   )sincosr   r   ru   )r,   	dt_statesrH   r-   eagerrX   )r-   dropoutrp   ) r   r   r   r   viewr   r   r   r   r   updater   r   reshaperw   expr   Fsoftplusprepare_dynamic_maskrH   r   r   r   r   _attn_implementationALL_ATTENTION_FUNCTIONStrainingrE   rp   r   r   )rQ   r,   r   r-   r   r   rR   input_shapehidden_shapequery_states
key_statesvalue_statesr   r   cache_kwargsr   	attn_maskattention_interfacer   attn_weightss                       rT   forwardzDogeAttention.forwardh  so    $))#2.88b8$--8{{4;;}#=#B#B<#PQ[[\]_`a[[]!;!@!@!NOYYZ[]^_
{{=166|DNNqRST&S#7jRUWZ#[ j%#&snUL'5'<'<ZW[WeWegs't$J LL""1a(001C1CA1FHZHZ[]H^`bc
	 IIdffqzz)'<<=GGBO	--'!22)	 . 
	 i)B)BC	(?;;++w6"9$++:Z:Z"[$7		%

 %#}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((rU   r   rH   c           	         t        j                  |j                        j                  }|j                  }|dddddddf   j	                  dd|j
                  d   d      }|t        |t              s|j                  t         j                  k(  rC|j                  }t        j                  |t        j                  d|j                  |      |      }|j                  |ddddddd|j
                  d   f   dk7  |      }|j
                  d   |kD  rnt        j                  |||j                        }t        j                  ||ddd	
      j                  }	|j!                  d|	d      }|j                  |dk(  |      }|S )a8  
        The core idea of DMA is to calculate the dynamic attention mask to mask the tokens that should be masked, so as to form sparse attention.

        Combine `dt_states` with `attention_mask` to generate the final `attn_mask`.

        Args:
            hidden_states (`torch.Tensor`): The input hidden_states, used to determine the minimum value of the current input precision.
            dt_states (`torch.Tensor`): dt_states of shape `(batch_size, num_heads, key_sequence_length)`.
            keep_window_size (`int`): The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
            attention_mask (`torch.Tensor`, *optional*): attention mask of shape `(batch_size, 1, query_sequence_length, key_sequence_length)`.
        Nr   r   rX   )devicer   r   r   r   TF)dimlargestsorted      ?)rw   finfor   minexpandr   r   r    boolwheretensorr   masked_fill
zeros_liketopkindicesscatter)
rQ   r,   r   rH   r-   	min_dtyper   r   active_masktopk_indicess
             rT   r   z"DogeAttention.prepare_dynamic_mask  sg   $ KK 3 3488	##aD!m,33M''*B
	 %j.S##uzz1%++!&"ELL^=R=RZ_$`bk" "--nQ1F[	XZH[F[=[.\`a.aclmI??2!11**9E)JZJZ[K ::i1ArSW`efnnL%--b,DK!--kS.@)LIrU   NNNN)rV   N)r\   r]   r^   r"   r   intrP   rw   Tensortupler	   
LongTensorr   r   rd   re   s   @rT   r   r   I  s   Jz Jhsm JD 26*.596)||6) #5<<#=>6) !.	6)
 !6) !!1!126) 
u||Xell3XeELL>Q5RR	S6)x !%15#||# <<# 	#
 !.#rU   r   c                       e Zd Zy)DogeMLPNrh   r5   rU   rT   r   r     ri   rU   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )	DogeCDMoEr   c                    t         |           |j                  | _        |j                  | _        t        |j
                     | _        |j                  | _        t        j                  t        j                  | j                              | _        |j                  | _        |j                  | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  dz  d      | _        t        j,                  | j                  | j                        | _        t        j,                  | j                  | j                        | _        y )Nr   r   F)rO   rP   r7   r8   r   r;   act_fnrJ   mathfloorsqrtnum_keysrK   top_krL   r   r   rF   	gate_projup_proj	down_projrouter_gate	Embedding
down_embedup_embedrQ   r   rS   s     rT   rP   zDogeCDMoE.__init__  s_   !--!'!9!9V../!--

499T-=-=#>?//
$33 4#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRab 99T%5%5t}}q7HuU ,,t'7'79I9IJT%5%5t7G7GHrU   r,   rs   c                    |j                   \  }}}| j                  |      j                  d||z  d      }|j                  | j                  d      \  \  }}\  }	}
|j                  d      |j                  d      z   }|	j                  d      | j                  z  |
j                  d      z   } |j                  g |j                   d d d } |j                  g |j                   d d d }|j                  | j                  d      \  }}|j                  d|      }t        j                  |d      }| j                  r||j                  dd      z  }| j                  |      }| j                  |      }t        j                  ||j                  ||z  dd            j                  ||z  d      }| j!                  |      |z  }t        j                  |j                  ||z  dd      |      j                  ||d      }| j#                  | j!                  | j%                  |            | j'                  |      z        }||z   }||fS )Nr   r   r   ru   T)r   keepdimr   )r   r   r   r   r   	unsqueezer   gatherr   softmaxrL   sumr   r   rw   matmulr   r   r   r   )rQ   r,   rR   bszseq_len_router_logitsscores_xscores_y	indices_x	indices_y
all_scoresall_indicesscoresposition_indicesr   routing_weightsr   r   experts_weightsexperts_statess                        rT   r   zDogeCDMoE.forward  sF   
 (--Wa ((7<<QgrR 8E7I7I$--]_7I7`484y)''+h.@.@.DD
))"-=	@S@STV@WW$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC#-??4::2?#F  $$R)9:))F322r42HHO __W-
==),,z=3E3EcGmUWYZ3[\aabehoboqst++o6Ho&:&:3=!R&PRZ[``adfmoqrt{{4>>-3P'QTXT`T`anTo'op%6m++rU   )	r\   r]   r^   r"   rP   rw   r   r   rd   re   s   @rT   r   r     s0    Iz I.,||, 
	,rU   r   c                   ~    e Zd Zddedee   f fdZ	 	 	 	 	 ddej                  de	ej                  ej                  f   deej                     deej                     dee	ej                        d	ee   d
eej                     dee   de	ej                  ee	ej                  ej                  f      f   fdZ xZS )DogeDecoderLayerr   r   c                 *   t         |           |j                  | _        t        |j                  |j
                        | _        t        ||      | _        t        j                  t        j                  |j                              | _        t        |j                  |j
                        | _        |j                  st!        |      n
t#        |      | _        t        j                  t        j                  |j                              | _        y )Nr   )r   r   )rO   rP   r:   rg   r7   r=   input_layernormr   	self_attnr   r   rw   onesinput_residualpost_attention_layernormrI   r   r   mlppost_attention_residualr   s      rT   rP   zDogeDecoderLayer.__init__  s    $33*6+=+=6CVCVW&f	J ll5::f6H6H+IJ(3F4F4FFL_L_(`%*0--76?Yv=N')||EJJv?Q?Q4R'S$rU   r,   r   r-   position_idsr   r>   r   rR   rs   c                    |}	| j                  |      } | j                  d|||||||d|\  }}
t        j                  || j                  | j
                        }| j                  |	z  |z   }|}	| j                  |      }| j                  |      }t        j                  || j                  | j
                        }| j                  |	z  |z   }|S )N)r,   r   r-   r  r   r>   r   )pr   r5   )
r  r  r   r   r:   r   r  r  r  r  )rQ   r,   r   r-   r  r   r>   r   rR   residualself_attn_weightss              rT   r   zDogeDecoderLayer.forward  s     !,,];+94>> 	,
' 3)%))	,
 	,
(( 		-43F3FQUQ^Q^_++h6F !55mD/		-43F3FQUQ^Q^_44x?-OrU   r   )NNNFN)r\   r]   r^   r"   r   r   rP   rw   r   r   r   r   r   r   FloatTensorr   rd   re   s   @rT   r  r    s   
Tz 
Thsm 
T  26378<$)59"||" #5<<#=>" !.	"
 u//0" !u||!45" D>" !!1!12" +," 
u  (51B1BEDUDU1U+V"WW	X"rU   r  c                   6    e Zd ZdZdZ eed      eedZ	d Z
y)DogePreTrainedModelFr   )index)r   r,   
attentionsc                    t        j                  |       t        |t              r2t	        |d      r%|j
                  j                  j                          yyt        |t              rdt	        |d      r%|j                  j                  j                  d       t	        |d      r&|j                  j                  j                  d       yyy)zInitialize the weightsr   r  r   r  N)r   _init_weightsr   r   hasattrr   datazero_r  r  fill_r  )rQ   rl   s     rT   r  z!DogePreTrainedModel._init_weights>  s    **62fm,vs###% $ 01v/0%%**005v89..3399#> : 2rU   N)r\   r]   r^   _supports_flash_attn_can_compile_fullgraphr   r   r  r   _can_record_outputsr  r5   rU   rT   r  r  5  s+     "'	;)#
?rU   r  c                       e Zd Zy)	DogeModelNrh   r5   rU   rT   r(  r(  K  ri   rU   r(  gate_logitsrJ   r   r   c                    | t        | t              sy| d   j                  }| d   j                  }g }g }| D ]  }	|	j	                  |      }	|	j                  |d      \  \  }
}\  }}|
j                  d      |j                  d      z   }|j                  d      |z  |j                  d      z   } |j                  g |j                  dd d } |j                  g |j                  dd d }|j                  |d      \  }}|j                  d|      }t        j                  |d      }|j                  |       |j                  |        t        j                  |d      }t        j                  |d      }|}|j                  d      }t        j                  |||      }t        j                   |||      }|j#                  d||      |j                  d   z  }t        j$                  |d      }nD|j                  \  }}t'        |       }|ddddddf   j)                  ||||f      j+                  d      j	                  |      }|j                  d      |j-                            }t        j                  |||      }t        j                   |||      }|j#                  d||      t        j.                  |      z  }|ddddddf   j)                  ||||f      j+                  d|      j	                  |      }t        j.                  ||z  d      t        j.                  |d      z  }t        j.                  ||z        }||z  S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://arxiv.org/abs/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `router_gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [2, batch_size * sequence_length, num_keys].
        num_experts:
            Number of experts
        num_keys:
            Number of keys
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   r   r   ru   r   )r   r   r   r   r   r   r   r   r   r   r   r   appendrw   catr   	ones_likescatter_add_meanlenr   r   r   r   )r)  rJ   r   r   r-   compute_dtypecompute_deviceall_expert_indicesall_routing_weightslayer_gate_logitsr   r   r  r  r  r  r   r  expert_indicesr  tokens_per_expertpadrouter_prob_per_expert
batch_sizesequence_lengthr9   expert_attention_mask router_per_expert_attention_maskoverall_losss                                rT   load_balancing_loss_funcr?  O  s{   @ *[%"@N((M ^**N( 4-00@7H7M7Mh\^7M7_484y)''+h.@.@.DD
))"-89;N;Nr;RR$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC(ooeo<$++B0@A))JB7!!.1""?3!4" #51=))$7Q?/44R8!KK=Q_`oo0n]-::1>PRUVYkYqYqrsYtt "',?Q!G&4&:&:#
O, 4At+,V&
OUKLWR[R	 	 044R89N9S9S9UV "KK=Q_`oo0n]-::1>PRUVY^YbYb!Z
 
 4At+,V&
O[QRWR%R	 	) "'+>Aa+agh!ilqlulu,!m
 "
 99.1GGHL+%%rU   c                   X    e Zd Z fdZ	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deeej                        deej                     deej                     dee
   d	eej                     d
eeej                  f   dee
   dee   defdZ xZS )DogeForCausalLMc                 f    t         |   |       t        |      | _        |j                  | _        y r   )rO   rP   r(  modelrJ   r   s     rT   rP   zDogeForCausalLM.__init__  s*     v&
!--rU   r*   r-   r  r$   r+   labelsr>   r   logits_to_keeprM   rR   rs   c                    |
|
n| j                   j                  }
 | j                  d|||||||d|}|j                  }t	        |	t
              rt        |	 d      n|	}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |}d}|
rt        |j                  | j                  t        j                  t        j                  | j                              | j                   |      }|+|| j"                  |j%                  |j&                        z  z  }t)        ||||j*                  |j,                  |j.                  |j                        S )ah  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, DogeForCausalLM

        >>> model = DogeForCausalLM.from_pretrained("SmallDoge/Doge-320M")
        >>> tokenizer = AutoTokenizer.from_pretrained("SmallDoge/Doge-320M")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r*   r-   r  r$   r+   r>   r   )lossaux_losslogitsr$   r,   r  r   r5   )r   rM   rC  last_hidden_stater   r   slicelm_headloss_functionr6   r?  r   rJ   r   r   r   rK   rN   r   r   r   r$   r,   r  )rQ   r*   r-   r  r$   r+   rD  r>   r   rE  rM   rR   outputsr,   slice_indicesrI  rG  rH  s                     rT   r   zDogeForCausalLM.forward  sp   J %9$D $++JjJj 	
 +5$** 	+
)%+')	+
 	+
  118B>SV8W~ot4]kmA}a,?@A%4%%ffdooPPD/%%  

499T%5%567((H !11HKK4LLL(#33!//))!//
 	
rU   )
NNNNNNNNr   N)r\   r]   r^   rP   r   rw   r   r   listr  r   r   r   r   r   r   r   rd   re   s   @rT   rA  rA    s   . 151537=A59-1$(5934/3Q
E,,-Q
 !.Q
 u//0	Q

 "$u'8'8"9:Q
   1 12Q
 ))*Q
 D>Q
 !!1!12Q
 c5<</0Q
 'tnQ
 +,Q
 
#Q
rU   rA  c                       e Zd Zy)DogeForSequenceClassificationNrh   r5   rU   rT   rR  rR    ri   rU   rR  )r"   rA  r(  r  rR  r   )NNr   N)Er_   r   typingr   r   r   rw   torch.nn.functionalr   
functionalr   activationsr   cache_utilsr	   configuration_utilsr
   integrations.flex_attentionr   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   modeling_utilsr   processing_utilsr   utilsr   r   utils.genericr   llama.modeling_llamar   r   r   r   r   r   r   r   mixtral.modeling_mixtralr   r   !torch.nn.attention.flex_attentionr    r"   rg   rk   Moduler   floatr   r   r   r   r   r   r  r  r(  r   r?  rA  rR  __all__r5   rU   rT   <module>rg     sN  "   , ,     !   3 J 9 Q 9 0 & E +	 	 	 H  !;S
! S
l	, 		. 	  $#(,.*II.*<<.* 
.* <<	.*
 %,,34.* e_.* e_.* %.* 5<<%&.*b -. 1G - .zBII zz	h 	6,		 6,r/1 /d?. ?,	 	 "&"-1g&u||U5<<%8$>?g&#g& smg& 	g&
 U\\*g& 5<<g&TW
( W
t	$B 	rU   