
    rhV                        d Z ddlmZ ddlmZmZ ddlZddlZddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlm Z   ejB                  e"      Z#e ed       G d de                    Z$e ed       G d de                    Z% G d dejL                        Z'e G d de             Z( ed       G d d e(             Z) ed!       G d" d#e(e             Z*g d$Z+y)%zPyTorch Llava model.    )	dataclass)OptionalUnionN)nn   )ACT2FN)Cache)GenerationMixin)FlashAttentionKwargs)BaseModelOutputWithPastModelOutput)PreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging   )	AutoModel   )LlavaConfigzJ
    Base class for Llava outputs, with hidden states and attentions.
    )custom_introc                   :    e Zd ZU dZdZeej                     ed<   y)LlavaModelOutputWithPasta  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
        `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nimage_hidden_states)	__name__
__module____qualname____doc__r   r   torchFloatTensor__annotations__     {/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/llava/modeling_llava.pyr   r   '   s    
 8<%"3"34;r$   r   zQ
    Base class for Llava causal language model (or autoregressive) outputs.
    c                      e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                        ed<   dZeeej                        ed<   dZeeej                        ed<   dZeej                     ed<   y)	LlavaCausalLMOutputWithPasta]  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
        `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nlosslogitspast_key_valueshidden_states
attentionsr   )r   r   r   r   r(   r   r    r!   r"   r)   r*   listr+   tupler,   r   r#   r$   r%   r'   r'   =   s      )-D(5$$
%,*.FHU&&'.9=OXd5#4#456=8<M8E%"3"345<59Ju001297;%"3"34;r$   r'   c                   *     e Zd Zdef fdZd Z xZS )LlavaMultiModalProjectorconfigc                    t         |           t        |j                  t              rdnt        |j                        }t        j                  |j                  j                  |z  |j                  j                  |j                        | _        t        |j                     | _        t        j                  |j                  j                  |j                  j                  |j                        | _        y )Nr   bias)super__init__
isinstancevision_feature_layerintlenr   Linearvision_confighidden_sizetext_configmultimodal_projector_biaslinear_1r   projector_hidden_actactlinear_2)selfr1   num_feature_layers	__class__s      r%   r6   z!LlavaMultiModalProjector.__init__]   s    ",V-H-H#"NQTWX^XsXsTt		  ,,/AA**11

 &556		**F,>,>,J,JQWQqQq
r$   c                 l    | j                  |      }| j                  |      }| j                  |      }|S N)r@   rB   rC   )rD   image_featuresr+   s      r%   forwardz LlavaMultiModalProjector.forwardk   s2    n5/m4r$   )r   r   r   r   r6   rJ   __classcell__rF   s   @r%   r0   r0   \   s    
{ 
r$   r0   c                   8    e Zd ZU eed<   dZdZdZdZdZ	dZ
dZdZy)LlavaPreTrainedModelr1    Tr*   N)r   r   r   r   r"   base_model_prefixsupports_gradient_checkpointing_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraph_supports_flex_attn_supports_attention_backendr#   r$   r%   rN   rN   r   s7    &*#"3N!"&r$   rN   zu
    The Llava model which consists of a vision backbone and a language model, without a language modeling head.
    c            %       F    e Zd ZddiZdef fdZd Zd Zd Zd Z		 	 dd	e
j                  d
eeeee   f      dee   fdZde
j$                  de
j                  de
j                  fdZee	 	 	 	 	 	 	 	 	 	 	 	 	 	 dde
j$                  d	e
j                  dee
j,                     dee
j$                     dee   dee
j                     d
eeeee   f      dee   dee   dee   dee   dee   dee
j$                     de
j,                  dee   deeef   f d              Z xZS )
LlavaModelzlanguage_model.modellanguage_modelr1   c                     t         |   |       t        j                  |j                        | _        t        |      | _        t        j                  |j                        | _	        | j                          y rH   )r5   r6   r   from_configr<   vision_towerr0   multi_modal_projectorr>   rZ   	post_initrD   r1   rF   s     r%   r6   zLlavaModel.__init__   sY     %11&2F2FG%=f%E"'33F4F4FGr$   c                 6    | j                   j                         S rH   )rZ   get_input_embeddingsrD   s    r%   rb   zLlavaModel.get_input_embeddings   s    ""7799r$   c                 :    | j                   j                  |       y rH   )rZ   set_input_embeddingsrD   values     r%   re   zLlavaModel.set_input_embeddings   s    007r$   c                     || _         y rH   rZ   rD   decoders     r%   set_decoderzLlavaModel.set_decoder   s
    %r$   c                     | j                   S rH   ri   rc   s    r%   get_decoderzLlavaModel.get_decoder   s    """r$   pixel_valuesr8   vision_feature_select_strategyc                 z   ||n| j                   j                  }||n| j                   j                  }|dvr"t        d| j                   j                         |j	                         D ci c]  \  }}|	|| }}} | j
                  |fddi|}t        |t              r |j                  |   }|dk(  r\|ddddf   }nP|D 	cg c]  }	|j                  |	    }
}	|dk(  r|
D cg c]  }|ddddf    }
}t        j                  |
d	      }| j                  |      }d
|v ro|d
   D cg c]8  \  }}|| j
                  j                  z  || j
                  j                  z  z  : }}}t        j                  |j                  d      |      }|S t        |      }|S c c}}w c c}	w c c}w c c}}w )a  
        Obtains image last hidden states from the vision tower and apply multimodal projection.

        Args:
            pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`):
               The tensors corresponding to the input images.
            vision_feature_layer (`Union[int, list[int]]`, *optional*):
                The index of the layer to select the vision feature. If multiple indices are provided,
                the vision feature of the corresponding indices will be concatenated to form the
                vision features.
            vision_feature_select_strategy (`str`, *optional*):
                The feature selection strategy used to select the vision feature from the vision backbone.
                Can be one of `"default"` or `"full"`
        Returns:
            image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
        N)defaultfullz$Unexpected select feature strategy: output_hidden_statesTrr   r   dimimage_sizesr   )r1   r8   rp   
ValueErroritemsr]   r7   r9   r+   r    catr^   
patch_sizesplitsqueezer-   )rD   ro   r8   rp   kwargskvimage_outputsselected_image_feature	layer_idxhs_poolhsrI   heightwidthsplit_sizess                   r%   get_image_featureszLlavaModel.get_image_features   s   0 %9$D $++JjJj 	
 .9 +;; 	' *1DDCDKKDnDnCopqq#)<<>C41aQ]!Q$CC))),\T\U[\ *C0%2%@%@AU%V"-:)?12)F&Ocd)}229=dGd-:/672ae977%*YYwB%?"334JKF" &,M%:!FE 4,,777ETEVEVEaEa<abK  #[[)?)?)BKPN  ".1N7 D e 8s   .
F'9F'
F-+F23=F7	input_idsinputs_embedsrI   c                 P   |m| | j                         t        j                  | j                  j                  t        j
                  |j                              k(  }|j                  d      }n|| j                  j                  k(  }|j                         }|j                  d      j                  |      j                  |j                        }|j                  d   |j                  d   z  }||   j                         |j                         k7  rt        d| d|       |S )z
        Obtains multimodal placeholdr mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        )dtypedeviceru   r   r   z6Image features and image tokens do not match: tokens: z, features )rb   r    tensorr1   image_token_idlongr   allsum	unsqueeze	expand_astoshapenumelry   )rD   r   r   rI   special_image_maskn_image_tokensn_image_featuress          r%   get_placeholder_maskzLlavaModel.get_placeholder_mask   s    !.2M$2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*dkk.H.H!H+//1/99"=GGVYYZgZnZno)//2^5I5I!5LL+,2248L8L8NNHHXXcdtcuv  "!r$   attention_maskposition_idsr*   	use_cacheoutput_attentionsrt   return_dictcache_positionrx   r   returnc                    |
|
n| j                   j                  }
||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }||n| j                   j
                  }|d u |d uz  rt        d      | | j                         |      }|v| j                  ||||      }t        j                  |d      j                  |j                  |j                        }| j                  |||      }|j                  ||      } | j                   d	|||||	|
|d|d	|}t#        |j$                  |j&                  |j(                  |j*                  |      S d       S )
Nz:You must specify exactly one of input_ids or inputs_embeds)ro   r8   rp   rx   r   rv   )r   rI   T)	r   r   r*   r   r   r   rt   r   r   )last_hidden_stater*   r+   r,   r   r#   )r1   r   rt   use_return_dictr8   rp   ry   rb   r   r    r{   r   r   r   r   masked_scatterrZ   r   r   r*   r+   r,   )rD   r   ro   r   r   r*   r   r8   rp   r   r   rt   r   r   rx   r   rI   r   outputss                      r%   rJ   zLlavaModel.forward   s   ( 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]$8$D $++JjJj 	
 .9 +;; 	' -t";<YZZ 7D557	BM#!44)%9/M'	 5 N #YY~1=@@AUAUWdWjWjkN!%!:!:~ "; " *889K^\M%$%% 
)%+'/!5)
 
 (%77#33!//))2>2J
 	

 QU
 	
r$   NN)NNNNNNNNNNNNNN)r   r   r   _checkpoint_conversion_mappingr   r6   rb   re   rl   rn   r    r!   r   r   r9   r-   strr   
LongTensorr   r   r   Tensorr	   boolr   r   r.   r   rJ   rK   rL   s   @r%   rY   rY      s    '=>N%O"{ :8&# AE8<	>''> 'uS$s)^'<=> )1	>@"))":?:K:K"]b]n]n"0  '+*.1537+/59@D8<$(,0/3&*59$(F
##F
 ''F
 !.	F

 u//0F
 "%F
   1 12F
 'uS$s)^'<=F
 )1F
 D>F
 $D>F
 'tnF
 d^F
 !!1!12F
 \\F
  -.!F
" 
u..	/#F
  F
r$   rY   zS
    The LLAVA model which consists of a vision backbone and a language model.
    c            )           e Zd ZdddddZdgZdef fdZd	 Zd
 Zde	j                  fdZd Zd Z	 	 d&dej                  deeeee   f      dee   fdZed        Zed        Zed        Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d'dej8                  dej                  deej:                     deej8                     dee   deej                     deeeee   f      dee   deej8                     dee   dee   dee   dee   d eej8                     d!eeej:                  f   d"eej:                     d#e e!   dee"e#f   f$d$              Z$	 	 	 	 	 	 d( fd%	Z% xZ&S ))LlavaForConditionalGenerationzmodel.language_modelzmodel.vision_towerzmodel.multi_modal_projectorlm_head)z^language_model.modelz^vision_towerz^multi_modal_projectorz^language_model.lm_headzlm_head.weightr1   c                     t         |   |       t        |      | _        t	        j
                  |j                  j                  |j                  j                  d      | _	        | j                          y )NFr3   )r5   r6   rY   modelr   r;   r>   r=   
vocab_sizer   r_   r`   s     r%   r6   z&LlavaForConditionalGeneration.__init__N  sS     '
yy!3!3!?!?ASASA^A^ejkr$   c                 6    | j                   j                         S rH   )r   rb   rc   s    r%   rb   z2LlavaForConditionalGeneration.get_input_embeddingsT  s    zz..00r$   c                 :    | j                   j                  |       y rH   )r   re   rf   s     r%   re   z2LlavaForConditionalGeneration.set_input_embeddingsW  s    

''.r$   r   c                     | j                   S rH   )r   rc   s    r%   get_output_embeddingsz3LlavaForConditionalGeneration.get_output_embeddingsZ  s    ||r$   c                 :    | j                   j                  |       y rH   )r   rl   rj   s     r%   rl   z)LlavaForConditionalGeneration.set_decoder]  s    

w'r$   c                 6    | j                   j                         S rH   )r   rn   rc   s    r%   rn   z)LlavaForConditionalGeneration.get_decoder`  s    zz%%''r$   ro   r8   rp   c                 B     | j                   j                  d|||d|S )N)ro   r8   rp   r#   )r   r   )rD   ro   r8   rp   r   s        r%   r   z0LlavaForConditionalGeneration.get_image_featuresc  s5     -tzz,, 
%!5+I
 	
 	
r$   c                 .    | j                   j                  S rH   )r   rZ   rc   s    r%   rZ   z,LlavaForConditionalGeneration.language_modelr  s    zz(((r$   c                 .    | j                   j                  S rH   )r   r]   rc   s    r%   r]   z*LlavaForConditionalGeneration.vision_towerv  s    zz&&&r$   c                 .    | j                   j                  S rH   )r   r^   rc   s    r%   r^   z3LlavaForConditionalGeneration.multi_modal_projectorz  s    zz///r$   r   r   r   r*   r   labelsr   r   rt   r   r   logits_to_keeprx   r   c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }||n| j                   j
                  } | j                  d|||||||||
||d||d|}|d   }t        |t              rt        | d      n|}| j                  |dd|ddf         }d}|	4 | j                  d||	| j                   j                  j                  d|}t        |||j                  |j                   |j"                  |j$                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, LlavaForConditionalGeneration

        >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
        >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")

        >>> prompt = "USER: <image>\nWhat's the content of the image? ASSISTANT:"
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, text=prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs, max_new_tokens=15)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "USER:  \nWhat's the content of the image? ASSISTANT: The image features a busy city street with a stop sign prominently displayed"
        ```NT)r   ro   r   r   r*   r   r8   rp   r   r   rt   r   r   rx   r   )r)   r   r   )r(   r)   r*   r+   r,   r   r#   )r1   r   rt   r   r8   rp   r   r7   r9   slicer   loss_functionr>   r   r'   r*   r+   r,   r   )rD   r   ro   r   r   r*   r   r8   rp   r   r   r   rt   r   r   r   rx   r   r   r+   slice_indicesr)   r(   s                          r%   rJ   z%LlavaForConditionalGeneration.forward~  s   b 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]$8$D $++JjJj 	
 .9 +;; 	' $** 
%)%+'!5+I/!5)#
 
$  
8B>SV8W~ot4]kmA}a,?@A%4%% f9P9P9[9[_eD +#33!//)) ' ; ;
 	
r$   c           	      N    t        
|   |f|||||d|}	|d   dk(  r||	d<   |	S )N)r*   r   r   r   r   r   ro   )r5   prepare_inputs_for_generation)rD   r   r*   r   ro   r   r   r   r   model_inputsrF   s             r%   r   z;LlavaForConditionalGeneration.prepare_inputs_for_generation  sV     w<
+')))
 
 !! ,8L(r$   r   )NNNNNNNNNNNNNNr   N)NNNNNN)'r   r   r   r   _tied_weights_keysr   r6   rb   re   r   Moduler   rl   rn   r    r!   r   r   r9   r-   r   r   propertyrZ   r]   r^   r   r   r   r   r	   r   r   r   r.   r'   rJ   r   rK   rL   s   @r%   r   r   @  s    "8-"?#,	&" ++{ 1/ryy (( AE8<	
''
 'uS$s)^'<=
 )1	
 ) ) ' ' 0 0  '+*.1537+/59@D8<-1$(,0/3&*5934.2#a
##a
 ''a
 !.	a

 u//0a
 "%a
   1 12a
 'uS$s)^'<=a
 )1a
 ))*a
 D>a
 $D>a
 'tna
 d^a
 !!1!12a
  c5<</0!a
" ell+#a
$ +,%a
& 
u11	2'a
  a
L  r$   r   )r   rN   rY   ),r   dataclassesr   typingr   r   r    torch.utils.checkpointr   activationsr   cache_utilsr	   
generationr
   modeling_flash_attention_utilsr   modeling_outputsr   r   modeling_utilsr   processing_utilsr   utilsr   r   r   r   autor   configuration_llavar   
get_loggerr   loggerr   r'   r   r0   rN   rY   r   __all__r#   r$   r%   <module>r      s2    ! "    !   ) B D - & R R  , 
		H	% 
<6 < <  
<+ < <2ryy , '? ' ' 
w
% w

w
t 
z$8/ z
zz Rr$   