
    rh!                     
   d Z ddlmZ ddlmZmZ ddlZddlmZ ddlm	Z	 ddl
mZ dd	lmZ dd
lmZmZmZ ddlmZ e G d de             Ze ed       G d de                    Z ed       G d de             ZddgZy)zPyTorch ColPali model    )	dataclass)OptionalUnionN)nn)AutoModelForImageTextToText   )Cache)PreTrainedModel)ModelOutputauto_docstringcan_return_tuple   )ColPaliConfigc                   2    e Zd ZU eed<   dZg ZdZdZdZ	d Z
y)ColPaliPreTrainedModelconfigmodelTc                    t        | j                  d      r| j                  j                  n)| j                  j                  j                  j                  }t        |t        j                  t        j                  f      rY|j                  j                  j                  d|       |j                  %|j                  j                  j                          y y t        |t        j                        rf|j                  j                  j                  d|       |j                  2|j                  j                  |j                     j                          y y y )Ninitializer_rangeg        )meanstd)hasattrr   r   
vlm_configtext_config
isinstancer   LinearConv2dweightdatanormal_biaszero_	Embeddingpadding_idx)selfmoduler   s      /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/colpali/modeling_colpali.py_init_weightsz$ColPaliPreTrainedModel._init_weights(   s     t{{$78 KK))''33EE 	 fryy"))45MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> . .    N)__name__
__module____qualname__r   __annotations__base_model_prefix_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attnr(    r)   r'   r   r      s*    N?r)   r   z3
    Base class for ColPali embeddings output.
    )custom_introc                      e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZeeeej                     ef      ed<   dZeeej                        ed<   dZeeej                        ed<   dZeej                     ed<   y)	ColPaliForRetrievalOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        The embeddings of the model.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
        `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    Nloss
embeddingspast_key_valueshidden_states
attentionsimage_hidden_states)r*   r+   r,   __doc__r7   r   torchFloatTensorr-   r8   Tensorr9   r   listr	   r:   tupler;   r<   r3   r)   r'   r6   r6   9   s      )-D(5$$
%,)-J&-GKOXeD):):$;U$BCDK8<M8E%"3"345<59Ju001297;%"3"34;r)   r6   u/  
    The ColPali architecture leverages VLMs to construct efficient multi-vector embeddings directly
    from document images (“screenshots”) for document retrieval. The model is trained to maximize the similarity
    between these document embeddings and the corresponding query embeddings, using the late interaction method
    introduced in ColBERT.

    Using ColPali removes the need for potentially complex and brittle layout recognition and OCR pipelines with a
    single model that can take into account both the textual and visual content (layout, charts, etc.) of a document.

    ColPali is part of the ColVision model family, which was first introduced in the following paper:
    [*ColPali: Efficient Document Retrieval with Vision Language Models*](https://huggingface.co/papers/2407.01449).
    c                   6    e Zd ZdddddZdef fdZee	 	 	 	 	 	 ddee	j                     d	ee	j                     d
ee	j                     dee   dee   dee   defd              Zd Zd Zd Zd Zd Z	 	 	 ddee   dee   dedej.                  fdZ xZS )ColPaliForRetrievalzvlm.model.language_modelzvlm.model.vision_towerzvlm.model.multi_modal_projectorzvlm.lm_head)zvlm.language_model.modelzvlm.vision_towerzvlm.multi_modal_projectorzvlm.language_model.lm_headr   c                    t         |   |       || _        |j                  j                  j
                  | _        t        j                  |j                        | _        | j                  j                  xs g D cg c]  }d| 	 c}| _	        | j                  j                  | _
        t        j                  | j                  j                  j                  j                  | j                        | _        | j                          y c c}w )Nzvlm.language_model.)super__init__r   r   r   
vocab_sizer   from_configvlm_tied_weights_keysembedding_dimr   r   hidden_sizeembedding_proj_layer	post_init)r%   r   k	__class__s      r'   rG   zColPaliForRetrieval.__init__n   s      ++77BB.::6;L;LMGKxxGbGbGhfh"j%8#<"j![[66$&IIKK""..::%
!
 	 #ks   =D
	input_idspixel_valuesattention_maskoutput_attentionsoutput_hidden_statesreturn_dictreturnc           
      ,   ||j                  | j                        }||n| j                  j                  }||n| j                  j                  }||n| j                  j
                  } | j                  j                  d|||dd|d|}|r|j                  nd }	||j                  nd }
|d   }| j                  |      }||j                  dd      z  }|||j                  d      z  }t        ||j                  |	|j                  |
      S )	N)dtypeT)rR   rT   rS   rV   rW   rU   r   )dimkeepdim)r8   r9   r:   r;   r<   r3   )torZ   r   rU   rV   use_return_dictrJ   r   r:   r<   rN   norm	unsqueezer6   r9   r;   )r%   rR   rS   rT   rU   rV   rW   kwargs
vlm_outputvlm_hidden_statesvlm_image_hidden_stateslast_hidden_statesr8   s                r'   forwardzColPaliForRetrieval.forward~   sA    #'???<L1B1N-TXT_T_TqTq %9$D $++JjJj 	 &1%<k$++B]B]#TXX^^ 
)%!%/
 

 9MJ44RVDPD\*"@"@bf']../AB
  *//b$/"GG
%#n&>&>r&BBJ(!&66+!,, 7
 	
r)   c                 6    | j                   j                         S N)rJ   get_input_embeddingsr%   s    r'   rj   z(ColPaliForRetrieval.get_input_embeddings   s    xx,,..r)   c                 :    | j                   j                  |       y ri   )rJ   set_input_embeddings)r%   values     r'   rm   z(ColPaliForRetrieval.set_input_embeddings   s    %%e,r)   c                 6    | j                   j                         S ri   )rJ   get_output_embeddingsrk   s    r'   rp   z)ColPaliForRetrieval.get_output_embeddings   s    xx--//r)   c                 :    | j                   j                  |       y ri   )rJ   set_output_embeddings)r%   new_embeddingss     r'   rr   z)ColPaliForRetrieval.set_output_embeddings   s    &&~6r)   c                 6    | j                   j                         S ri   )rJ   tie_weightsrk   s    r'   ru   zColPaliForRetrieval.tie_weights   s    xx##%%r)   new_num_tokenspad_to_multiple_ofmean_resizingc                 B   | j                   j                  |||      }|j                  | j                  j                  j
                  _        |j                  | j                  j                  _        |j                  | j                   _        |j                  | _        |S )N)rv   rw   rx   )rJ   resize_token_embeddingsnum_embeddingsr   r   r   rH   )r%   rv   rw   rx   model_embedss        r'   rz   z+ColPaliForRetrieval.resize_token_embeddings   s     xx77)1' 8 
 9E8S8S**5,8,G,G)*99&55r)   )NNNNNN)NNT)r*   r+   r,   _checkpoint_conversion_mappingr   rG   r   r   r   r>   
LongTensorr?   r@   boolr6   rg   rj   rm   rp   rr   ru   intr   r#   rz   __classcell__)rQ   s   @r'   rD   rD   X   s      %?4%F&3	&"}    154815,0/3&*.
E,,-.
 u001.
 !.	.

 $D>.
 'tn.
 d^.
 
#.
  .
`/-07&
 )-,0"	  %SM 	
 
r)   rD   )r=   dataclassesr   typingr   r   r>   r   transformersr   cache_utilsr	   modeling_utilsr
   utilsr   r   r   configuration_colpalir   r   r6   rD   __all__r3   r)   r'   <module>r      s     ! "   4   - B B 0 ?_ ? ?2 
< < <2 j0 jj\ r)   