
    rh3                       d Z ddlZddlmZmZmZ ddlZddlmc m	Z
 ddlZddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0m1Z1  e)       rddl2m3Z3 ddl4m5Z5  e*jl                  e7      Z8dejr                  de:de;de<ejr                  ejr                  f   fdZ=dejr                  de:de:dej|                  dejr                  f
dZ? G d d ej                        ZA G d! d"ej                        ZB G d# d$ej                        ZCd%ejr                  d&e:dejr                  fd'ZD	 dZd(ej                  d)ejr                  d*ejr                  d+ejr                  d,eejr                     d-eEd.eEd/e$e&   fd0ZF G d1 d2ej                        ZG G d3 d4ej                        ZH G d5 d6ej                        ZI G d7 d8ej                        ZJ G d9 d:ej                        ZKd; ZLd[d<ZM G d= d>ej                        ZN G d? d@ej                        ZO G dA dBe      ZP G dC dDe      ZQ G dE dFej                        ZRe' G dG dHe"             ZS e'dIJ       G dK dLeS             ZT e'dMJ       G dN dOeS             ZU e'dPJ       G dQ dReSe             ZV e'dSJ       G dT dUeS             ZW e'dVJ       G dW dXeSe             ZXg dYZYy)\zPyTorch Mllama model.    N)CallableOptionalUnion)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)AttentionMaskConverter)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torch_flex_attn_availablelogging)OutputRecordercheck_model_inputs   )MllamaConfigMllamaTextConfigMllamaVisionConfig)	BlockMask)make_flex_block_causal_maskcross_attention_masknum_vision_tokensdtypereturnc                    | j                   ^}}}| j                  |d      } | j                  ||d      } | j                  d      } d| z
  j	                  |      }|j                  |j	                  t        j                        t        j                  |      j                        } t        j                  |      j                  }| |k7  j                  d      j                  |       d   }| |z  } | |fS )Nr   dimr         ?).N)shaperepeat_interleaveview	unsqueezetomasked_filltorchboolfinfominanytype_as)	r$   r%   r&   
batch_sizetext_total_length_inverted_cross_attn_masknegative_inf_valuefull_text_row_masked_out_masks	            }/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/mllama/modeling_mllama.py_prepare_cross_attention_maskr@   0   s    )=(B(B%J!A/AABSYZA[/44ZARTVW/99!< !$&: :>>uE3?? ##EJJ/U1C1G1G U+//	!3	388R8@HHI]^_hi " 99!>>>    aspect_ratio_masknum_patchestarget_lengthc                 |   | j                   \  }}| j                  ||dd      j                  |      }|j                  dd|d      }||z
  }d|d d d d | d f<   d|z
  }|j	                  |||z  d      }||j                  dd      z  t        j                  |      j                  z  }|j                  d      }|S )Nr   r   r+   )
r-   r/   r1   repeatreshape	transposer3   r5   r6   r0   )rB   rC   rD   r&   r9   max_num_tilesattention_maskpad_patchess           r?   $_prepare_aspect_ratio_attention_maskrM   L   s     !2 7 7J&++Jq!LOOPUVN#**1aBN  +-K*+N1a+&' 'N $++J8UWXYN#n&>&>r2&FFUZI[I_I__N#--a0NrA   c                   z     e Zd Zddedef fdZdej                  dej                  dej                  fdZ xZ	S )	%MllamaPrecomputedAspectRatioEmbeddingconfigis_gatedc                 t   t         |           |j                  | _        |j                  | _        |j                  | _        || _        t        j                  | j                  dz   | j                  | j                  z        | _        |r.t        j                  t        j                  d            | _        y y Nr   )super__init__rJ   hidden_sizemax_aspect_ratio_idrQ   r   	Embedding	embedding	Parameterr3   zerosgateselfrP   rQ   	__class__s      r?   rU   z.MllamaPrecomputedAspectRatioEmbedding.__init__h   s    #11!--#)#=#=  d&>&>&BDDVDVY]YiYiDijU[[^4DI rA   hidden_stateaspect_ratio_idsr'   c                     | j                  |      }|j                  d| j                  d| j                        }| j                  r|| j
                  j                         z  }||z   }|S )Nr+   r   )rY   rH   rJ   rV   rQ   r\   tanh)r^   r`   ra   
embeddingss       r?   forwardz-MllamaPrecomputedAspectRatioEmbedding.forwards   s_    ^^$45
''D,>,>4CSCST
==#diinn&66J#j0rA   )T)
__name__
__module____qualname__r!   r4   rU   r3   Tensorre   __classcell__r_   s   @r?   rO   rO   g   s@    	51 	5T 	5ELL ELL UZUaUa rA   rO   c                   t     e Zd Zdef fdZdej                  dej                  dej                  fdZ xZS )"MllamaPrecomputedPositionEmbeddingrP   c                    t         |           |j                  | _        |j                  | _        |j                  |j
                  z  dz  dz   | _        |j                  | _        |j                  dz  | _        t        j                  t        j                  d            | _        t        j                  | j                  | j                        }t        j                  | j                  |z        | _        t        j                   | j                  dz   | j                  | j                  z  | j                  z        | _        y )N   r         )rT   rU   rJ   rW   
image_size
patch_sizerC   rV   scaler   rZ   r3   r[   r\   randnrY   rX   tile_embedding)r^   rP   position_embeddingr_   s      r?   rU   z+MllamaPrecomputedPositionEmbedding.__init__   s    #11#)#=#= "--1B1BBqH1L!--''-
LLQ0	 #[[)9)94;K;KLdjj3E&EF !ll$$q($*<*<t?O?O*ORVRbRb*b
rA   r`   ra   r'   c                    d| j                   j                         z
  | j                  z  }||j                  dd| j                  | j
                        z   }| j                  |      }|j                  d   }|j                  || j                  | j                  | j
                        }| j                   j                         |z  }||z   }|S )Nr   r   )
r\   rc   rY   r/   rC   rV   ru   r-   rH   rJ   )r^   r`   ra   gated_position_embeddingtile_position_embeddingr9   gated_tile_position_embeddings          r?   re   z*MllamaPrecomputedPositionEmbedding.forward   s    $%		(8$8DNN#J #&>&C&CAq$JZJZ\`\l\l&mm #'"5"56F"G!''*
"9"A"A**D,<,<d>N>N#
 )-		(8;R(R%#&CCrA   )	rf   rg   rh   r!   rU   r3   ri   re   rj   rk   s   @r?   rm   rm   ~   s9    
1 
&ELL ELL UZUaUa rA   rm   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )MllamaVisionMLPc                    t         |           || _        t        |j                     | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y N)rT   rU   rP   r   
hidden_actactivation_fnr   LinearrV   intermediate_sizefc1fc2r^   rP   r_   s     r?   rU   zMllamaVisionMLP.__init__   sd    #F$5$5699V//1I1IJ99V55v7I7IJrA   hidden_statesr'   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r~   )r   r   r   )r^   r   s     r?   re   zMllamaVisionMLP.forward   s4    /**=9/rA   )rf   rg   rh   rU   r3   ri   re   rj   rk   s   @r?   r|   r|      s$    KU\\ ell rA   r|   r   n_repc                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r-   expandrH   )r   r   batchnum_key_value_headsslenhead_dims         r?   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTrA   modulequerykeyvaluerK   scalingdropoutkwargsc                 T   t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
|#|d d d d d d d |j
                  d   f   }|
|z   }
t        j                  j                  |
dt        j                        j                  |j                        }
t        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )Nro   r   rF   r+   )r*   r&   )ptrainingr   )r   num_key_value_groupsr3   matmulrI   r-   r   
functionalsoftmaxfloat32r1   r&   r   r   
contiguous)r   r   r   r   rK   r   r   r   
key_statesvalue_statesattn_weightscausal_maskattn_outputs                r?   eager_attention_forwardr      s    3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!$Q1.D
0@0@0D.D%DE#k1==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$rA   c            
            e Zd Zdef fdZ	 ddej                  deej                     deej                  eej                     f   fdZ	 xZ
S )MllamaVisionAttentionrP   c                    t         |           || _        |j                  | _        |j
                  | _        |j                  |j
                  z  | _        | j                  dz  | _        d| _	        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  z  | j                  d      | _        y )Nrp   r   Fbias)rT   rU   rP   rV   	embed_dimattention_heads	num_headsr   r   r   r   r   q_projk_projv_projo_projr   s     r?   rU   zMllamaVisionAttention.__init__   s    ++//**f.D.DD}}d*$%!ii0NUZ[ii0NUZ[ii0NUZ[ii >UZ[rA   r`   rK   r'   c                    | j                  |      }| j                  |      }| j                  |      }|j                  \  }}}	|j                  \  }	}
}	|j	                  ||| j
                  | j                        j                  dd      }|j	                  ||
| j
                  | j                        j                  dd      }|j	                  ||
| j
                  | j                        j                  dd      }t        }| j                  j                  dk7  rt        | j                  j                     } || ||||fd| j                  d|\  }}|j                  ||d      j                         }| j                  |      }||fS )Nr   ro   eager        r   r   r+   )r   r   r   r-   r/   r   r   rI   r   rP   _attn_implementationr   r   rH   r   r   )r^   r`   rK   r   r   r   r   r9   	q_seq_lenr;   
kv_seq_lenattention_interfacer   r   s                 r?   re   zMllamaVisionAttention.forward   sm    L)kk,'L)#(;; 
Iq99:q

:y$..$--PZZ[\^_`hhz:t~~t}}MWWXY[\]

:z4>>4==Q[[\]_`a(?;;++w6"9$++:Z:Z"[$7	%
 LL	%
 	%
!\ "))*iDOOQkk+.L((rA   r~   )rf   rg   rh   r!   rU   r3   ri   r   tuplere   rj   rk   s   @r?   r   r      s\    \1 \$ 26$)ll$) !.$)
 
u||Xell33	4$)rA   r   c                   l     e Zd Zddedef fdZ	 ddej                  deej                     fdZ	 xZ
S )	MllamaVisionEncoderLayerrP   rQ   c                    t         |           |j                  | _        |j                  | _        || _        |j                  | _        t        |      | _        t        |      | _
        t        j                  | j                  |j                        | _        t        j                  | j                  |j                        | _        |rt        j                   t#        j$                  d      t&        j(                  z  dz        | _        t        j                   t#        j$                  d      t&        j(                  z  dz        | _        y y )Nepsr      )rT   rU   rV   r   num_attention_headsrQ   r   r   	self_attnr|   mlpr   	LayerNormnorm_epsinput_layernormpost_attention_layernormrZ   r3   onesmathpi	gate_attngate_ffnr]   s      r?   rU   z!MllamaVisionEncoderLayer.__init__  s    !--#)#9#9  !'!9!9.v6"6*!||D,<,<&//R(*T5E5E6??([%\\%**Q-$''*AA*EFDNLLA)@1)DEDM rA   r`   rK   c                 X   |}| j                  |      }| j                  ||      \  }}| j                  r| j                  j	                         |z  }||z   }|}| j                  |      }| j                  |      }| j                  r| j                  j	                         |z  }||z   }|S )NrK   )r   r   rQ   r   rc   r   r   r   )r^   r`   rK   residualr   s        r?   re   z MllamaVisionEncoderLayer.forward%  s      ++L9%)^^LQ_^%`"l==>>..0<?L,.  44\Bxx-====--/,>L,.rA   Fr~   )rf   rg   rh   r!   r4   rU   r3   ri   r   re   rj   rk   s   @r?   r   r     sC    F1 FT F* 26ll !.rA   r   c                   p     e Zd ZdZddef fdZ	 d	dej                  deej                     de	fdZ
 xZS )
MllamaVisionEncoderz
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`MllamaEncoderLayer`].

    Args:
        config: MllamaConfig
    rP   c           	          t         |           || _        t        j                  t        |      D cg c]  }t        ||       c}      | _        d| _        || _        y c c}w )NF)	rT   rU   rP   r   
ModuleListranger   layersgradient_checkpointing)r^   rP   
num_layersrQ   r;   r_   s        r?   rU   zMllamaVisionEncoder.__init__F  sU    mmY^_iYj$kTU%=fh%O$kl&+# %ls   A!r   rK   r'   c                 P    | j                   D ]  } |||      } t        |      S )a8  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)

        )r`   rK   last_hidden_state)r   r   )r^   r   rK   encoder_layers       r?   re   zMllamaVisionEncoder.forwardM  s4    ( "[[ 	M)*-M	 ??rA   )    Fr~   )rf   rg   rh   __doc__r!   rU   r3   ri   r   r   re   rj   rk   s   @r?   r   r   =  sL    1  26@||@ !.@ 
	@rA   r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )MllamaTextRMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z@
        MllamaTextRMSNorm is equivalent to T5LayerNorm
        N)rT   rU   r   rZ   r3   r   weightvariance_epsilon)r^   rV   r   r_   s      r?   rU   zMllamaTextRMSNorm.__init__l  s1     	ll5::k#:; #rA   c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S )Nro   r+   T)keepdim)	r&   r1   r3   r   powmeanrsqrtr   r   )r^   r   input_dtypevariances       r?   re   zMllamaTextRMSNorm.forwardt  sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::rA   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)r   r   r-   r   r^   s    r?   
extra_reprzMllamaTextRMSNorm.extra_repr{  s*    ))*+6$2G2G1HIIrA   )gư>)rf   rg   rh   rU   re   r   rj   rk   s   @r?   r   r   k  s    $;JrA   r   c                   6    e Zd ZdZ	 	 ddee   dee   f fdZ	 	 	 	 	 ddej                  deej                     dee
   deej                     d	ee   d
eej                     deej                  eej                     eeej                        f   fdZ xZS )MllamaTextCrossAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrP   	layer_idxc                    t         |           || _        | j                  j                  | _        | j                  j
                  | _        |j                  | _        |j                  | _        |j                  | j                  z  | _        || _	        | j                  | j
                  z  | _
        | j                  dz  | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j
                  | j                  z  d      | _        t        j                  | j                  | j
                  | j                  z  d      | _        t        j                  | j                  | j                  z  | j                  d      | _        t%        | j                  |j&                        | _        t%        | j                  |j&                        | _        y )Nrp   Fr   r   )rT   rU   rP   r   r   r   r   rV   r   r   r   r   r   r   r   r   r   r   r   rms_norm_epsq_normk_normr^   rP   r   r_   s      r?   rU   z!MllamaTextCrossAttention.__init__  sk   
 	88#';;#B#B ~~!--**dnn<"$(NNd6N6N$N!}}d*ii 0 0$..4==2PW\]ii 0 0$2J2JT]]2Zafgii 0 0$2J2JT]]2Zafgii >@P@PW\]'6;N;NO'6;N;NOrA   r   cross_attention_statespast_key_valuerK   	use_cachecache_positionr'   c                 v   |j                         \  }}	}
| j                  |      }|j                  ||	| j                  | j                        j                  dd      }| j                  |      }|| j                  |      }| j                  |      }|j                  |d| j                  | j                        j                  dd      }|j                  |d| j                  | j                        j                  dd      }| j                  |      }|~|j                  ||| j                  d|i      \  }}nZ|d   dk7  rG|j                  | j                     j                  |j                  | j                     j                  }}nt!        d      t"        }| j$                  j&                  dk7  rt(        | j$                  j&                     } || ||||f| j*                  sdn| j,                  | j.                  d	|\  }}|j1                  ||	d      j3                         }| j5                  |      }||fS )
z#Input shape: Batch x Time x Channelr   ro   r+   r   r   z^Cross attention layer can't find neither `cross_attn_states` nor cached values for key/values!r   r   r   )sizer   r/   r   r   rI   r   r   r   r   r   updater   r   keysvalues
ValueErrorr   rP   r   r   r   r   r   rH   r   r   )r^   r   r   r   rK   r   r   r   bszq_lenr;   query_statesr   r   r   r   r   s                    r?   re   z MllamaTextCrossAttention.forward  s(    &**,UA{{=1#((eT^^T]]S]]^_abc{{<0!-%;<J;;'=>L#b$2J2JDMMZddefhijJ',,S"d6N6NPTP]P]^hhijlmnLZ0J) ,:+@+@dnn?OQ_>`,(
L A!#%%dnn5::%%dnn5<< %J
 p  )@;;++w6"9$++:Z:Z"[$7	%
  $}}C$,,LL	%
 	%
!\ "))#ub9DDFkk+.L((rA   )NN)NNNNN)rf   rg   rh   r   r   r    intrU   r3   ri   r	   r4   
LongTensorr   re   rj   rk   s   @r?   r   r     s    G .2#'P)*P C=P6 :>*.15$(59:)||:) !) 6:) !	:)
 !.:) D>:) !!1!12:) 
u||Xell3XeELL>Q5RR	S:)rA   r   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr+   ro   r)   )r-   r3   cat)xx1x2s      r?   rotate_halfr
    sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''rA   c                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r0   r
  )qkcossinposition_idsunsqueeze_dimq_embedk_embeds           r?   apply_rotary_pos_embr    sY    ( --
&C
--
&C3w;q>C/0G3w;q>C/0GGrA   c            	            e Zd Zdedef fdZ	 	 	 d	dej                  dej                  dej                  defdZ	 xZ
S )
MllamaTextSelfAttentionrP   r   c                 x   t         |           || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | j                  z  | _        | j                  | j                  z  | _	        | j                  dz  | _
        |j                  | _        || _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  z  | j                  d      | _        y )Nrp   Fr   )rT   rU   rP   r   r   r   rV   r   r   r   r   
rope_thetar   r   r   r   r   r   r   r   s      r?   rU   z MllamaTextSelfAttention.__init__  s>   33~~!--#)#=#= **dnn<$(NNd6N6N$N!}}d* ++"ii 0 0$..4==2PW\]ii 0 0$2J2JT]]2Zafgii 0 0$2J2JT]]2Zafgii >@P@PW\]rA   r   rK   position_embeddingsr   c                    |j                         \  }}	}
| j                  |      }| j                  |      }| j                  |      }|j	                  ||	| j
                  | j                        j                  dd      }|j	                  ||	| j                  | j                        j                  dd      }|j	                  ||	| j                  | j                        j                  dd      }|\  }}t        ||||      \  }}|'|||d}|j                  ||| j                  |      \  }}t        }| j                  j                  dk7  rt        | j                  j                     } || ||||f| j                   sdn| j"                  | j$                  d|\  }}|j'                  ||	d      j)                         }| j+                  |      }||fS )Nr   ro   )r  r  r   r   r   r   r+   )r   r   r   r   r/   r   r   rI   r   r  r   r   r   rP   r   r   r   r   r   rH   r   r   )r^   r   rK   r  r   r   r   r   r   r  r;   r  r   r   r  r  cache_kwargsr   r   r   s                       r?   re   zMllamaTextSelfAttention.forward  s    &**,UA{{=1[[/
{{=1#((eT^^T]]S]]^_abc__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm&S#7jRUWZ#[ j%#&snUL'5'<'<ZW[WeWegs't$J(?;;++w6"9$++:Z:Z"[$7	%
  $}}C$,,LL	%
 	%
!\ "))#ub9DDFkk+.L((rA   )FNN)rf   rg   rh   r    r  rU   r3   ri   r4   re   rj   rk   s   @r?   r  r    s\    ^/ ^C ^.  /)||/) /) #\\	/)
 /)rA   r  c                   $     e Zd Z fdZd Z xZS )MllamaTextMLPc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        t        |j                     | _        y NFr   )rT   rU   rP   rV   r   r   r   	gate_projup_proj	down_projr   r   act_fnr   s     r?   rU   zMllamaTextMLP.__init__B  s    !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXV../rA   c                     | j                  | j                  | j                  |            | j                  |      z        }|S r~   )r"  r#  r   r!  )r^   r  r"  s      r?   re   zMllamaTextMLP.forwardM  s6    NN4;;t~~a/@#ADLLQRO#ST	rA   )rf   rg   rh   rU   re   rj   rk   s   @r?   r  r  A  s    	0rA   r  c                       e Zd Zdedef fdZ	 	 	 	 	 	 	 	 	 ddej                  deej                     deej                     deej                     dee	ej                  ej                  f      d	eej                     d
ee   dee   deej                     dee	ej                  ej                  f      dee   de	ej                  ee	ej                  ej                  f      f   fdZ xZS )MllamaSelfAttentionDecoderLayerrP   r   c                 .   t         |           |j                  | _        t        ||      | _        t        |      | _        t        |j                  |j                        | _	        t        |j                  |j                        | _
        || _        y )N)rP   r   r   )rT   rU   rV   r  r   r  r   r   r   r   r   r   r   s      r?   rU   z(MllamaSelfAttentionDecoderLayer.__init__T  st    !--0)T (01C1CI\I\](9&:L:LRXReRe(f%"rA   r   r   r$   rK   r>   r  r   r   r   r  r   r'   c                     |}| j                  |      } | j                  d||||||	|
d|\  }}||z   }|}| j                  |      }| j                  |      }||z   }|S )aY  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.

            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        )r   rK   r  r   r   r   r   )r   r   r   r   )r^   r   r   r$   rK   r>   r  r   r   r   r  r   r   self_attn_weightss                 r?   re   z'MllamaSelfAttentionDecoderLayer.forward`  s    D !,,]; ,:4>> 	,
')%)) 3	,
 	,
(( !=0 !55mD/ =0rA   )	NNNNNNFNN)rf   rg   rh   r    r  rU   r3   ri   r   r   r  r	   r4   r   r   FloatTensorre   rj   rk   s   @r?   r&  r&  S  sK   
#/ 
#C 
# :>7;15UY37*.$)59KO9||9 !) 69 'u||4	9
 !.9 (0ellELL6P0Q'R9 u//09 !9 D>9 !!1!129 &eELL%,,,F&GH9 -.9 
u  (51B1BEDUDU1U+V"WW	X9rA   r&  c                   p    e Zd ZdZdededdf fdZ	 	 	 	 	 ddej                  dej                  d	ej                  d
ej                  de	ej                  ej                  f   de
ej                     de
e   de
e   de
ej                     de
ej                     dee   de	ej                     fdZ xZS ) MllamaCrossAttentionDecoderLayerzLCross-attention transformer block with tanh-gated attention and feedforward.rP   r   r'   Nc                    t         |           || _        t        ||      | _        t        |j                  |j                        | _        t        j                  j                  t        j                  d            | _        t        |      | _        t        |j                  |j                        | _        t        j                  j                  t        j                  d            | _        y )N)r   r   r   )rT   rU   r   r   
cross_attnr   rV   r   r   r3   r   rZ   r[   cross_attn_attn_gater  r   r   cross_attn_mlp_gater   s      r?   rU   z)MllamaCrossAttentionDecoderLayer.__init__  s    "26YO01C1CI\I\]$)HH$6$6u{{1~$F! ((9&:L:LRXReRe(f%#(88#5#5ekk!n#E rA   r   r   r$   rK   r>   r  r   r   r   r  r   c           	      F   |}| j                  |      } | j                  d|||||	d|\  }}|| j                  j                         |z  z   }|}| j	                  |      }| j                  |      }||d d df   |z  }|| j                  j                         |z  z   }|S )N)r   rK   r   r   r   r   r)  )r   r/  r0  rc   r   r   r1  )r^   r   r   r$   rK   r>   r  r   r   r   r  r   r   r   s                 r?   re   z(MllamaCrossAttentionDecoderLayer.forward  s     !,,];&5doo '
'/#9))'
 '
#| !4#<#<#A#A#Cm#SS 55mD/(49!Q$?-OM 4#;#;#@#@#B]#RRrA   )NNFNN)rf   rg   rh   r   r    r  rU   r3   ri   r   r   r  r	   r4   r   r   re   rj   rk   s   @r?   r-  r-    s	   V
F/ 
FC 
FD 
F& 48*.$)596:"||" !&" $ll	"
 " (-U\\5<<-G'H" u//0" !" D>" !!1!12" &ell3" -." 
u||	"rA   r-  c                   ^     e Zd Zddef fdZ ej                         ed               Z xZ	S )MllamaRotaryEmbeddingrP   c                 ^   t         |           |j                  d   | _        |j                  | _        |j                  | _        || _        t        | j                     | _	        | j                  | j                  |      \  }| _
        | j                  d|d       | j                  | _        y )N	rope_typeinv_freqF)
persistent)rT   rU   rope_scalingr6  max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrP   r   rope_init_fnattention_scalingregister_bufferr7  original_inv_freq)r^   rP   devicer7  r_   s       r?   rU   zMllamaRotaryEmbedding.__init__  s    ,,[9"("@"@$*$B$B!/?+/+<+<T[[&+Q($(ZeD!%rA   c                 0   | j                   d d d d f   j                         j                  |j                  d   dd      }|d d d d d f   j                         }t	        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        j                  |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j                  |j                   
      	j                  |j                   
      fS # 1 sw Y   AxY w)Nr   r+   r   mpscpuF)device_typeenabledro   r)   )r&   )r7  floatr   r-   
isinstancerA  typestrr3   autocastrI   r  r  r>  r  r1   r&   )
r^   r  r  inv_freq_expandedposition_ids_expandedrE  freqsembr  r  s
             r?   re   zMllamaRotaryEmbedding.forward  sD    !MM$4-8>>@GGHZHZ[\H]_acde ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk^^UC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s   BFFr~   )
rf   rg   rh   r    rU   r3   no_gradr   re   rj   rk   s   @r?   r4  r4    s4    
// 
/ U]]_<  <rA   r4  c                   X   e Zd ZU eed<   dZdZg dZdZdZ	dZ
dZdZeeg eedd       eedd	       eedd	      gd
Zd Z	 ddeej,                  df   dej,                  dej,                  dedef
dZedej,                  dededej8                  dej,                  defd       Zy)MllamaPreTrainedModelrP    T)r   r-  r&  Fr   r   )index
layer_namer/  )r   
attentionsc                 r   t        | j                  d| j                  j                         j                        }t	        |t
        j                  t
        j                  f      rY|j                  j                  j                  d|       |j                  %|j                  j                  j                          y y t	        |t
        j                        rf|j                  j                  j                  d|       |j                  2|j                  j                  |j                     j                          y y t	        |t
        j                        rJ|j                  j                  j!                  d       |j                  j                  j                          y t	        |t"              r&|j                  j                  j!                  d       y t	        |t$              r6t
        j&                  j                  |j(                  j                  |       y t	        |t*              rit
        j&                  j                  |j,                  j                  |       t
        j&                  j/                  |j0                  j                         y t	        |t2              rw|j4                  rkt
        j&                  j                  |j6                  j                  |       t
        j&                  j                  |j8                  j                  |       y t	        |t:              rI|j<                  j                  j                          |j>                  j                  j                          y t	        |t@              r2|j4                  r%|j0                  j                  j                          y y y )Ninitializer_ranger   )r   stdr,   )rY  )!getattrrP   get_text_configrX  rH  r   r   Conv2dr   datanormal_r   zero_rX   padding_idxr   fill_r   MllamaVisionModelinitclass_embeddingrm   rY   zeros_r\   r   rQ   r   r   r-  r0  r1  rO   )r^   r   rY  s      r?   _init_weightsz#MllamaPreTrainedModel._init_weights  sj   dkk#68S8S8U8g8ghfryy"))45MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> .-MM$$S)KK""$ 12MM$$S) 12GGOOF2277SOA BCGGOOF,,11sO;GGNN6;;++, 89fooGGOOF,,11sO;GGOOFOO00cO: @A'',,224&&++113 EF  &&(  GrA   rK   r"   input_tensorr   past_key_valuesoutput_attentionsc           	         | j                   j                  dk(  r||dk(  j                         r|S y | j                   j                  dk(  r't        |t        j
                        rt        |      }|S ||j                         nd}||j                  nd}| j                   j                  dk(  r(|s&|s$t        j                  |||| j                        ry |j                  }|j                  d   }	|r|j                         }
n1t        |t        j
                        r|j                  d	   n||	z   dz   }
| j                  ||	|
|||j                  d   
      }| j                   j                  dk(  rQ|O|j                   j"                  dv r7|s5t	        j$                  |      j&                  }t        j(                  ||      }|S )Nflash_attention_2r   flex_attentionr   Fsdpa)inputs_embedspast_key_values_lengthis_trainingr   r+   )sequence_lengthrD   r&   r   r9   )cudaxpunpu)rP   r   r7   rH  r3   ri   r#   get_seq_lengthis_compileabler   _ignore_causal_mask_sdpar   r&   r-   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionrA  rI  r5   r6   _unmask_unattended)r^   rK   rg  r   rh  ri  past_seen_tokensusing_compilable_cacher&   rq  rD   r   	min_dtypes                r?   _update_causal_maskz)MllamaPreTrainedModel._update_causal_mask%  s    ;;++/BB)~/D.I.I.K%%;;++/??.%,,7!<^!L!!
 @O?Z?99;`aCRC^!?!?di ;;++v5>T]n%>>*'7 MM	 ""&,,Q/!+??AM nell; $$R(%7!;  PP+')#))!, Q 
 KK,,6*%%**.DD%
 E*..I0CCKQZ[KrA   rq  rD   r&   r9   c                    | | j                         dk(  r| }|S t        j                  |      j                  }t        j                  ||f|||j
                        }|dk7  rt        j                  |d      }|t        j                  ||j
                        |j                  dd      kD  z  }|ddddddf   j                  |ddd      }| |j                         }| j                  d   }	|ddddddd|	f   | ddddddf   j                  |j
                        z   }
|
dk(  }
|ddddddd|	f   j                  |
|      |ddddddd|	f<   |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        Nr   )
fill_valuer&   rA  r   )diagonalrA  r+   r   )r*   r3   r5   r6   fullrA  triuarangerH   r   cloner-   r1   r2   )rK   rq  rD   r&   r   r9   r   r   r}  mask_lengthpadding_masks              r?   ry  zKMllamaPreTrainedModel._prepare_4d_causal_attention_mask_with_cache_positioni  s   > %.*<*<*>!*C(K* ' E*..I** -0Ye\j\q\qK !##jjqA5<<n>S>STWeWmWmnprsWtttK%dD!Q&67>>z1bRTUK))//1,2226*1aL[L+@ANSTVZ\`bcScDdDgDg&&E    ,q05@Aq,;,AV5W5c5c )6Aq!\k\12 rA   Nr   )rf   rg   rh   r   __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_can_compile_fullgraph_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backendr&  r-  r   r  r   _can_record_outputsrf  r   r3   ri   r	   r4   r~  staticmethodr  r&   ry  r)  rA   r?   rR  rR    s-   &*#
 #N"&9;[\2!T2!U31V
)L #(BellK78B llB 	B
 B  BH 444 4 {{	4
 4 4 4rA   rR  zH
    The Mllama Vision Model which consists of two vision encoders.
    )custom_introc            
            e Zd ZU eed<   dZdef fdZd Zdej                  dej                  fdZ
eedej                  d	ej                  d
ej                  defd              Z xZS )rb  rP   vision_modelc                    t         |   |       |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        | j                  | j                  z  dz  dz   | _        |j
                  dz  | _	        t        j                  |j                  | j
                  | j                  | j                  dd      | _        t        j                  | j                  t        j                  | j
                        z        | _        t#        |      | _        t'        |d      | _        t'        |d      | _        t        j,                  | j
                        | _        t        j,                  | j
                        | _        t3        ||j4                  d      | _        t3        ||j8                  d      | _        | j=                          y )	Nro   r   rp   validF)in_channelsout_channelskernel_sizestridepaddingr   T)rQ   )rT   rU   rq   rr   rJ   rV   num_channelsintermediate_layers_indicesrC   rs   r   r\  patch_embeddingrZ   r3   rt   rd  rm   gated_positional_embeddingrO   pre_tile_positional_embeddingpost_tile_positional_embeddingr   layernorm_prelayernorm_postr   num_hidden_layerstransformernum_global_layersglobal_transformer	post_initr   s     r?   rU   zMllamaVisionModel.__init__  s     ++ ++#11!--"//+1+M+M( OOt>1DqH''-
!yy++))?? 
  "||DJJTEUEU9V,VW*LV*T'-RSYdh-i*.STZei.j+  \\$*:*:; ll4+;+;< /vv7O7OZ_`"5ff>V>Vae"frA   c                     | j                   S )zg
        This function is used to fetch the first embedding layer to activate grads on inputs.
        )r  r   s    r?   get_input_embeddingsz&MllamaVisionModel.get_input_embeddings  s     ###rA   r`   r'   c                     |j                   \  }}}| j                  j                  |d|      }t        j                  ||gd      }|S )Nr   r)   )r-   rd  r   r3   r  )r^   r`   r9   r;   rV   rd  s         r?   apply_class_embeddingz'MllamaVisionModel.apply_class_embedding  sI    %1%7%7"
A{..55j![Qyy/<!@aHrA   pixel_valuesra   rB   c                    |j                   \  }}}}}	}
|j                  ||z  |z  ||	|
      }|j                  ||z  d      }| j                  j                  j                  }| j                  j                  j
                  }| j                  |j                  ||            }|j                  d      j                  dd      }|j                   \  }}}|j                  ||z  |d|      }| j                  ||      }|j                  ||z  |z  ||      }| j                  |      }|dz  }|j                  ||z  |||      }| j                  ||      }| j                  |      }d|j                   d   dz  z
  dz  }ddd|f}t        j                  ||dd      }|dkD  r| nd	}|j                  ||z  d      }t        || j                   |j                   d   | j                  
      }|j#                  ||z  d|      }| j%                  ||      }|j&                  }| j)                  |      }|j                  ||z  |||z   |      }| j+                  ||      }|j                  ||z  |||z   z  |      }| j-                  ||      }|j&                  }|j                  ||z  |||z   |      }|d	d	d	d	d	|f   }|j                  |||||      }| j.                  D cg c]  }|j&                   }}t1        j2                  |d      }|j                  ||z  |||z   d      }|d	d	d	d	d	|f   }|j                  ||||d      }t1        j4                  ||gd      }t7        |      S c c}w )a5  
        aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*):
            Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image.
            These ids correspond to indices in the model's list of supported aspect ratios, offset by 1.

            For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]:
            - An image with aspect ratio [1, 1] would have ID 1
            - An image with aspect ratio [1, 2] would have ID 2
            - An image with aspect ratio [2, 1] would have ID 3

            The id 0 is reserved for padding (i.e., no image).

            If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2.
        aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*):
            Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`:

            - 1 for tiles that are **not masked**,
            - 0 for tiles that are **masked**.

        Example:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, MllamaVisionModel

        >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
        >>> model = MllamaVisionModel.from_pretrained(checkpoint)
        >>> processor = AutoProcessor.from_pretrained(checkpoint)

        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)
        >>> inputs = processor(images=image, return_tensors="pt")

        >>> output = model(**inputs)

        >>> print(output.last_hidden_state.shape)
        torch.Size([1, 1, 4, 1025, 7680])
        ```
        r+   ro   r      rF   r   constant)moder   N)rB   rC   rD   r&   r   r)   r   )r-   rH   r  r   r&   rA  r1   flattenrI   r  r  r  r  FpadrM   rC   r/   r  r   r  r  r  r  r3   stackr  r   )r^   r  ra   rB   r   r9   num_concurrent_media	num_tilesr  heightwidthtarget_dtypetarget_devicepatch_embedsr`   r;   rC   r*   num_padding_patchesr  slice_indexrK   outputglobal_outputall_intermediate_hidden_statesintermediate_hidden_statess                             r?   re   zMllamaVisionModel.forward  s*   Z T`SeSeP
()\65#++J9M,MPY,Y[gioqvw+33JAU4UWYZ ++2288,,33::++LOOM<,XY#++A.88A> +00;#++J9M,MyZ\^ab99,HXY $++J9M,MPY,Y[fhkl11,?q $++J9M,MyZegjk66|EUV)),7  !L$6$6r$:Q$>?1Da/0uu\71M.AA.E**4 +22:@T3TVXY=,((&,,Q/**	
 $((6J)JBPST!!) " 
 //**<8 $++--y+H[:[]`
 ::<IYZ#++--yKJ]<]/^`c
 //) 0 
 %66 $++--y+H[:[]`
 $Aq,;,$67#++J8LiYdfij MQLlLl)mq&*B*B)m&)m%*[[1OUW%X" &@%G%G--y+H[:[]_&
" &@1l{l@R%S"%?%G%G,ib&
"
 yy,0J!KQST>> *ns   %M3)rf   rg   rh   r!   r  r  rU   r  r3   ri   r  r   r   r   re   rj   rk   s   @r?   rb  rb    s     &#1 #J$%,, 5<<  F?!LLF?<ALLF?]b]i]iF?	F?  F?rA   rb  zc
    The Mllama Text Model which consists of transformer with self and cross attention layers.
    c                       e Zd ZU eed<   dZdef fdZeee		 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     de
ej                     de
ej                     de
ej                     d	e
eej                  ej                  f      d
e
eeeej                     f      de
ej                     de
e   de
ej                     dee   defd                     Z xZS )MllamaTextModelrP   language_model.modelc                    t         |   |       |j                  | _        |j                  | _        t        j                  |j                  dz   |j                  | j                        | _        |j                  | _	        g }t        |j                        D ]G  }|| j                  v r|j                  t        ||             -|j                  t        ||             I t        j                  |      | _        t#        |j                  |j$                        | _        t)        |      | _        d| _        | j/                          y )Nr  r   )rP   F)rT   rU   pad_token_idr`  
vocab_sizer   rX   rV   embed_tokenscross_attention_layersr   r  appendr-  r&  r   r   r   r   normr4  
rotary_embr   r  )r^   rP   r   r   r_   s       r?   rU   zMllamaTextModel.__init__p  s    !.. ++LL):):Q)>@R@RTXTdTde&,&C&C#v778 	RID777>vyQR=fiPQ		R mmF+%f&8&8f>Q>QR	/v>&+#rA   	input_idsrK   r  r   r$   r>   rh  rn  r   r   r   r'   c                    |	|	n| j                   j                  }	|du |duz  rt        d      || j                  |      }|}|	r|
t	               }|
F||j                         nd}t        j                  |||j                  d   z   |j                        }
||
j                  d      }| j                  |||
|      }| j                  ||      }t        | j                        D ]M  \  }}|| j                  v }|du xs |duxr |j                  |      dk(  }|r||r; ||f|||||||	|
|d	|}O | j!                  |      }t#        ||      S )aQ  
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
            A tuple containing two tensors that mask out rows in the cross-attention mechanism:
            - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
              A value of 0 indicates that the corresponding text token's entire row in the cross-attention
              matrix should be masked out (all image tokens ignored).
            - The second tensor has the same shape and is used internally to apply the masking during
              the forward pass of cross-attention layers.
            This mask is derived from the cross_attention_mask and is used to handle cases where a text token
            should not attend to any image token.

        Example:

        ```python
        >>> from transformers import AutoProcessor, MllamaTextModel

        >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
        >>> model = MllamaTextModel.from_pretrained(checkpoint)
        >>> processor = AutoProcessor.from_pretrained(checkpoint)

        >>> text = "<|image|>If I had to write a haiku for this one"
        >>> inputs = processor(text=text, return_tensors="pt")

        >>> output = model(**inputs)

        >>> print(output.last_hidden_state.shape)
        torch.Size([1, 13, 4096])
        ```
        N:You must specify exactly one of input_ids or inputs_embedsr   r   r  )	r   r$   rK   r>   r  r   r   r   r  )r   rh  )rP   r   r   r  r
   ru  r3   r  r-   rA  r0   r~  r  	enumerater   r  r  r   )r^   r  rK   r  r   r$   r>   rh  rn  r   r   r   r   r{  r   r  idxdecoder_layeris_cross_attention_layeris_cross_attention_cache_emptys                       r?   re   zMllamaTextModel.forward  s   p "+!6IDKK<Q<Q	-t";<YZZ  --i8M%0*nO!CRC^==?de"\\ "2]5H5H5K"KTaThThN )33A6L..~}n^mn #oom\J #,DKK"8 	C (+d.I.I'I$-<-D .t+X0N0Ns0SWX0X + (,B,JOm)'=%9*.K).#-$7 M	4 		-0&++
 	
rA   )
NNNNNNNNNN)rf   rg   rh   r    r  r  rU   r   r   r   r   r3   r  ri   r+  r   r   r	   listr4   r   r   r   re   rj   rk   s   @r?   r  r  g  sj    ./ (  151537>B7;UYKO59$(59o
E,,-o
 !.o
 u//0	o

 !)):): ;o
 'u||4o
 (0ellELL6P0Q'Ro
 "%tE4E4E/F(F"GHo
   1 12o
 D>o
 !!1!12o
 -.o
 
!o
   o
rA   r  zE
    The Mllama Text Model with a language modeling head on top.
    c            !           e Zd ZU eed<   dZdZdgZ fdZd Z	d Z
ee	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     d	eej                      d
eej                     deej                     deej                     deeej                   ej                   f      deeeeej*                     f      deej*                     deej                     dee   deej                     deeej                   f   dee   deeef   fd              Z xZS )MllamaForCausalLMrP   Tlanguage_modellm_head.weightc                 ~   t         |   |j                                |j                         | _        | j                  j                  | _        t
        j                  | j                        | _        t        j                  | j                  j                  | j                  d      | _        | j                          y r  )rT   rU   r[  text_configr  r  _from_configmodelr   r   rV   lm_headr  r   s     r?   rU   zMllamaForCausalLM.__init__  s    //12!113**55$11$2B2BC
yy!1!1!=!=tUZ[rA   c                     || _         y r~   r  r^   decoders     r?   set_decoderzMllamaForCausalLM.set_decoder  s	    
rA   c                     | j                   S r~   r  r   s    r?   get_decoderzMllamaForCausalLM.get_decoder  s    zzrA   r  rK   r  r   r$   r>   rh  rn  labelsr   r   logits_to_keepr   r'   c                     | j                   d|||||||||
|d
|}|j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         j                         }d}|	 | j                  ||	| j                  fi |}t        |||j                  |j                  |j                        S )a  
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
            A tuple containing two tensors that mask out rows in the cross-attention mechanism:
            - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
              A value of 0 indicates that the corresponding text token's entire row in the cross-attention
              matrix should be masked out (all image tokens ignored).
            - The second tensor has the same shape and is used internally to apply the masking during
              the forward pass of cross-attention layers.
            This mask is derived from the cross_attention_mask and is used to handle cases where a text token
            should not attend to any image token.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MllamaForCausalLM

        >>> model = MllamaForCausalLM.from_pretrained("Llama-3.2-11B-Vision")
        >>> tokenizer = AutoTokenizer.from_pretrained("Llama-3.2-11B-Vision")

        >>> prompt = "If I had to write a haiku, it would be:"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6)
        >>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        >>> print(result)
        If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful.
        I love the idea of snowflakes gently falling, each one
        ```
        )
r  r   rK   r  r$   r>   rh  rn  r   r   Nlosslogitsrh  r   rV  r)  )r  r   rH  r  slicer  rG  loss_functionr  r   rh  r   rV  )r^   r  rK   r  r   r$   r>   rh  rn  r  r   r   r  r   outputsr   slice_indicesr  r  s                      r?   re   zMllamaForCausalLM.forward  s    ~ $** 
#9)%!5*G+')
 
  118B>SV8W~ot4]kmA}a,?@AGGI%4%%ffdooPPD%#33!//))
 	
rA   )NNNNNNNNNNNr   )rf   rg   rh   r    r  r  r  _tied_weights_keysrU   r  r  r   r   r   r3   r  ri   r   r   r	   r  r+  r4   r  r   r   r   re   rj   rk   s   @r?   r  r    s    !(*+  151537=A;?UYKO59-1$(5934Y
E,,-Y
 !.Y
 u//0	Y

 !))9)9 :Y
 'u'7'78Y
 (0ellELL6P0Q'RY
 "%tE4E4E/F(F"GHY
   1 12Y
 ))*Y
 D>Y
 !!1!12Y
 c5<</0Y
 +,Y
 
u,,	-Y
  Y
rA   r  zr
    The Mllama model which consists of a vision encoder and a language model without language modeling head.
    c                        e Zd ZddiZdef fdZd Zd Zd Zd Z	e
ee	 	 	 	 	 	 	 	 	 	 	 	 dd	eej                     d
eej                      deej"                     deej"                     deej"                     deej"                     deej"                     deej                     dee   deej                      dee   deej                     dee   defd                     Z xZS )MllamaModelr  r  rP   c                    t         |   |       |j                  j                  | _        |j                  j                  | _        |j
                  j                  | _        |j
                  j                  | _        | j                  j                  | j                  j                  nd| _	        t        j                  |j
                        | _        t        j                  |j                        | _        t        j                   |j
                  j                  |j                  j                  d      | _        | j%                          y )Nr+   Tr   )rT   rU   r  r  rV   vision_configrJ   vision_output_dimrP   r  rb  r  r  r  r  r   r   multi_modal_projectorr  r   s     r?   rU   zMllamaModel.__init__y  s      ,,77!--99#11??!'!5!5!G!G8<8P8P8\DKK44bd-::6;O;OP-::6;M;MN%'YY  22**&
"
 	rA   c                 6    | j                   j                         S r~   )r  r  r   s    r?   r  z MllamaModel.get_input_embeddings  s    ""7799rA   c                 :    | j                   j                  |       y r~   )r  set_input_embeddingsr^   r   s     r?   r  z MllamaModel.set_input_embeddings  s    007rA   c                     || _         y r~   r  r  s     r?   r  zMllamaModel.set_decoder  s
    %rA   c                     | j                   S r~   r  r   s    r?   r  zMllamaModel.get_decoder  s    """rA   r  r  rB   ra   rK   r$   r   r  rh  rn  r   r   r   r'   c                 `   |du |
duz  rt        d      ||t        d      |f|t        d      | j                  |||      }|j                  }| j                  |      j	                  d|j
                  d   | j                        }|0t        || j                  j                  | j                        \  }}nd}|||dddd|f   }|dddd|f   } | j                  d|||||||	||
|d	
|}t        |j                  |j                  |j                  |j                  
      S )ar  
        aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*):
            Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`:

            - 1 for tiles that are **not masked**,
            - 0 for tiles that are **masked**.
        aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*):
            Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image.
            These ids correspond to indices in the model's list of supported aspect ratios, offset by 1.

            For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]:
            - An image with aspect ratio [1, 1] would have ID 1
            - An image with aspect ratio [1, 2] would have ID 2
            - An image with aspect ratio [2, 1] would have ID 3

            The id 0 is reserved for padding (i.e., no image).

            If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        Nr  zM`pixel_values` and `cross_attention_states` cannot be provided simultaneouslyzA`aspect_ratio_ids` must be provided if `pixel_values` is provided)r  ra   rB   r+   rF   )r%   r&   )
r  rK   r  r   r$   r>   rh  r   rn  r   )r   rh  r   rV  r)  )r   r  r   r  rH   r-   rV   r@   rC   r&   r  r   rh  r   rV  )r^   r  r  rB   ra   rK   r$   r   r  rh  rn  r   r   r   vision_outputsr>   r  s                    r?   re   zMllamaModel.forward  s   ` -t";<YZZ#(>(Jlmm#' !dee!..)!1"3 / N
 &4%E%E"%)%?%?@V%W%_%_*004d6F6F&"  +B_$"&"3"3"?"?jjC? "? -1)+0J#71n8L#M ,I!QP^J^,_)%$%% 
)%#9!5*G+')
 
 '%77#33!//))	
 	
rA   )NNNNNNNNNNNN)rf   rg   rh   _checkpoint_conversion_mappingr   rU   r  r  r  r  r   r   r   r   r3   r  r+  ri   r	   r4   r   r   r   re   rj   rk   s   @r?   r  r  q  s    '=>N%O"| ":8&#  15484837157;9=37+/59$(59a
E,,-a
 u001a
 $ELL1	a

 #5<<0a
 !.a
 'u||4a
 !) 6a
 u//0a
 "%a
   1 12a
 D>a
 !!1!12a
 -.a
 
!a
   a
rA   r  zS
    The Mllama model which consists of a vision encoder and a language model.
    c            %           e Zd ZdddddZdgZdef fdZd	 Zd
 Zd Z	d Z
ed        Zed        Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d"deej$                     deej&                     deej(                     deej(                     deej(                     deej(                     deej(                     deej$                     deeeeej&                     f      deej&                     deej$                     dee   deej$                     deeej(                  f   dee   deeef   f d              Z	 	 	 	 	 	 	 	 	 	 	 	 d# fd 	Z fd!Z  xZ!S )$MllamaForConditionalGenerationzmodel.language_modelzmodel.vision_modelzmodel.multi_modal_projectorr  )z^language_model.modelz^vision_modelz^multi_modal_projectorz^language_model.lm_headr  rP   c                     t         |   |       t        |      | _        t	        j
                  |j                  j                  |j                  j                  d      | _	        | j                          y r  )rT   rU   r  r  r   r   r  rV   r  r  r  r   s     r?   rU   z'MllamaForConditionalGeneration.__init__  sS      (
yy!3!3!?!?ASASA^A^ejkrA   c                 6    | j                   j                         S r~   )r  r  r   s    r?   r  z3MllamaForConditionalGeneration.get_input_embeddings  s    zz..00rA   c                 :    | j                   j                  |       y r~   )r  r  r  s     r?   r  z3MllamaForConditionalGeneration.set_input_embeddings  s    

''.rA   c                 :    | j                   j                  |       y r~   )r  r  r  s     r?   r  z*MllamaForConditionalGeneration.set_decoder  s    

w'rA   c                 6    | j                   j                         S r~   )r  r  r   s    r?   r  z*MllamaForConditionalGeneration.get_decoder  s    zz%%''rA   c                 .    | j                   j                  S r~   )r  r  r   s    r?   r  z-MllamaForConditionalGeneration.language_model  s    zz(((rA   c                 .    | j                   j                  S r~   )r  r  r   s    r?   r  z+MllamaForConditionalGeneration.vision_model"  s    zz&&&rA   r  r  rB   ra   rK   r$   r   r  rh  rn  r  r   r   r  r   r'   c                     | j                   d|||||||||	|
||d|}|j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }d}|3 | j                  ||| j                  j                  j                  fi |}t        |||j                  |j                  |j                        S )af  
        aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*):
            Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`:

            - 1 for tiles that are **not masked**,
            - 0 for tiles that are **masked**.
        aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*):
            Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image.
            These ids correspond to indices in the model's list of supported aspect ratios, offset by 1.

            For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]:
            - An image with aspect ratio [1, 1] would have ID 1
            - An image with aspect ratio [1, 2] would have ID 2
            - An image with aspect ratio [2, 1] would have ID 3

            The id 0 is reserved for padding (i.e., no image).

            If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, MllamaForConditionalGeneration

        >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
        >>> model = MllamaForConditionalGeneration.from_pretrained(checkpoint)
        >>> processor = AutoProcessor.from_pretrained(checkpoint)

        >>> prompt = "<|image|>If I had to write a haiku for this one"
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(text=prompt, images=image, return_tensors="pt")

        >>> # Generate
        >>> output = model.generate(**inputs, max_new_tokens=15)

        >>> prompt_len = inputs.input_ids.shape[-1]
        >>> generated_ids = output[:, prompt_len:]
        >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
        >>> print(generated_text)
        [', it would be:.\\nA stop sign in Chinatown.\\n']
        ```
        )r  r  rB   ra   r$   r   rK   r  rh  rn  r   r   Nr  r)  )r  r   rH  r  r  r  r  rP   r  r  r   rh  r   rV  )r^   r  r  rB   ra   rK   r$   r   r  rh  rn  r  r   r   r  r   r  r   r  r  r  s                        r?   re   z&MllamaForConditionalGeneration.forward&  s    ` $** 
%/-!5#9)%+')
 
   118B>SV8W~ot4]kmA}a,?@A%4%%ffdkk6M6M6X6Xc\bcD%#33!//))
 	
rA   c                 n    t        |   |f|	|
|||||||||d|}|d   dk7  rd |d<   d |d<   d |d<   |S )N)rh  r   rn  r  rK   r  ra   rB   r$   r   r  r   r  ra   rB   )rT   prepare_inputs_for_generation)r^   r  rn  rK   r  r  ra   rB   r$   rh  r   r   r  r   model_inputsr_   s                  r?   r  z<MllamaForConditionalGeneration.prepare_inputs_for_generation  s|    $ w<
+'%)%-/!5))
 
$ !!+/L(/3L+,04L,-rA   c                     |j                  dd       }t        |   d|||d|}|&t        j                  ||d d dd df   gd      |d<   |S )Nr$   )r  model_kwargsis_encoder_decoderr+   .r   r)   r)  )getrT   #_update_model_kwargs_for_generationr3   r  )r^   r  r  r  r   cross_attention_mask_prevr_   s         r?   r  zBMllamaForConditionalGeneration._update_model_kwargs_for_generation  s{    $0$4$45KT$R!wB 
%1
 	
 %03899*,Eack,RSYZ4L/0 rA   )NNNNNNNNNNNNNr   )NNNNNNNNNFNN)"rf   rg   rh   r  r  r   rU   r  r  r  r  propertyr  r  r   r   r   r3   r  r+  ri   r   r	   r  r4   r  r   r   r   r   re   r  r  rj   rk   s   @r?   r  r    sG    "8-"?#,	&" ++| 1/(( ) ) ' '  15484837157;9=37KO59-1$(5934m
E,,-m
 u001m
 $ELL1	m

 #5<<0m
 !.m
 'u||4m
 !) 6m
 u//0m
 "%tE4E4E/F(F"GHm
   1 12m
 ))*m
 D>m
 !!1!12m
 c5<</0m
  +,!m
" 
u,,	-#m
  m
b !)V rA   r  )r  r  r  rb  rR  r  )r   rS   )Zr   r   typingr   r   r   r3   torch.nn.functionalr   r   r  torch.utils.checkpointactivationsr   cache_utilsr	   r
   
generationr   modeling_attn_mask_utilsr   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r   utils.genericr   r   configuration_mllamar   r    r!   !torch.nn.attention.flex_attentionr"   integrations.flex_attentionr#   
get_loggerrf   loggerri   r  rJ  r   r@   r&   rM   ModulerO   rm   r|   r   rG  r   r   r   r   r   r   r
  r  r  r  r&  r-  r4  rR  rb  r  r  r  r  __all__r)  rA   r?   <module>r.     sm     , ,      ! . ) > B 9 ` ` K F & p p ? T T  !;J			H	%?,,?? ? 5<<%&	?8||  ;;	
 \\6BII ." "Lbii  	UU\\ 	U# 	U%,, 	U( %II%<<% 
% <<	%
 U\\*% % % '(%44)BII 4)n(ryy (V*@")) *@\J		 J(U)ryy U)r(6B)bii B)LBII $F&@ FR1'A 1h<BII <: qO q qh 
}?- }?
}?@ 
J
+ J

J
Z 
p
- p

p
f 
D
' D

D
N 
N%:O N
NbrA   