
    rh+                        d dl mZ d dlmZmZ d dlmZ d dlmZ ddl	m
Z
 ddlmZ ddlmZmZmZmZ d	d
lmZ  e       rd dlZe G d de             Ze ed       G d de                    Z ed       G d de             ZddgZy)    )	dataclass)OptionalUnion)nn)AutoModelForImageTextToText   )Cache)PreTrainedModel)ModelOutputauto_docstringcan_return_tupleis_torch_available   )ColQwen2ConfigNc                   2    e Zd ZU eed<   dZg ZdZdZdZ	d Z
y)ColQwen2PreTrainedModelconfigmodelTc                    t        | j                  d      r| j                  j                  n)| j                  j                  j                  j                  }t        |t        j                  t        j                  f      rY|j                  j                  j                  d|       |j                  %|j                  j                  j                          y y t        |t        j                        rf|j                  j                  j                  d|       |j                  2|j                  j                  |j                     j                          y y y )Ninitializer_rangeg        )meanstd)hasattrr   r   
vlm_configtext_config
isinstancer   LinearConv2dweightdatanormal_biaszero_	Embeddingpadding_idx)selfmoduler   s      /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/colqwen2/modeling_colqwen2.py_init_weightsz%ColQwen2PreTrainedModel._init_weights0   s     t{{$78 KK))''33EE 	 fryy"))45MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> . .    N)__name__
__module____qualname__r   __annotations__base_model_prefix_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attnr)    r*   r(   r   r   '   s*    N?r*   r   z4
    Base class for ColQwen2 embeddings output.
    )custom_introc                       e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZeeeej                     ef      ed<   dZeeej                        ed<   dZeeej                        ed<   y)ColQwen2ForRetrievalOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        The embeddings of the model.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
        `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    Nloss
embeddingspast_key_valueshidden_states
attentions)r+   r,   r-   __doc__r8   r   torchFloatTensorr.   r9   Tensorr:   r   listr	   r;   tupler<   r4   r*   r(   r7   r7   A   s     )-D(5$$
%,)-J&-GKOXeD):):$;U$BCDK8<M8E%"3"345<59Ju00129r*   r7   uG  
    Following the ColPali approach, ColQwen2 leverages VLMs to construct efficient multi-vector embeddings directly
    from document images (“screenshots”) for document retrieval. The model is trained to maximize the similarity
    between these document embeddings and the corresponding query embeddings, using the late interaction method
    introduced in ColBERT.

    Using ColQwen2 removes the need for potentially complex and brittle layout recognition and OCR pipelines with
    a single model that can take into account both the textual and visual content (layout, charts, ...) of a document.

    ColQwen2 is part of the ColVision model family, which was introduced with ColPali in the following paper:
    [*ColPali: Efficient Document Retrieval with Vision Language Models*](https://huggingface.co/papers/2407.01449).
    c                       e Zd Zi Zdef fdZee	 	 	 	 	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     dee   dee	j                     dee	j                     d	ee   d
ee   dee   dee   dee	j                     dee	j                     dee	j                     defd              Zd Zd Zd Zd Zd Z	 	 	 ddee   dee   dedej0                  fdZ xZS )ColQwen2ForRetrievalr   c                    t         |   |       || _        |j                  j                  j
                  | _        t        j                  |j                        | _        | j                  j                  | _	        t        j                  | j                  j                  j                  j                  | j                        | _        | j                  j                  xs g D cg c]  }d| 	 c}| _        | j                          y c c}w )Nzvlm.)super__init__r   r   r   
vocab_sizer   from_configvlmembedding_dimr   r   hidden_sizeembedding_proj_layer_tied_weights_keys	post_init)r&   r   k	__class__s      r(   rG   zColQwen2ForRetrieval.__init__m   s      ++77BB.::6;L;LM![[66$&IIKK""..::%
! 9=8S8S8YWY"[!T!:"[ #\s   %D
	input_idsattention_maskposition_idsr:   labelsinputs_embeds	use_cacheoutput_attentionsoutput_hidden_statesreturn_dictpixel_valuesimage_grid_thwcache_positionreturnc                 .   ||j                  | j                        }|L|J|dddf   |dddf   z  }t        j                  t	        ||      D cg c]
  \  }}|d|  c}}d      }||n| j
                  j                  }|	|	n| j
                  j                  }	|
|
n| j
                  j                  }
| j                  j                  j                  ||d|      \  }}|| j                  j                  j                  |      }||j                  | j                  j                  j!                               }| j                  j                  ||      }|| j
                  j"                  j$                  k(  j'                  d	      j)                  |      }|j                  |j*                  |j                        }|j-                  ||      }||j                  |j*                        }| j                  j                  d|||||||	|
|

      }|	r|j.                  nd}|d   }| j1                  |      }||j3                  d	d      z  }|||j'                  d	      z  }t5        ||j6                  ||j8                        S c c}}w )z
        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
            The temporal, height and width of feature shape of each image in LLM.
        N)dtyper      r   )dim)rR   r\   video_grid_thwrS   )grid_thw)
rR   rT   rS   r:   rV   rW   rX   rY   rZ   r]   T)rb   keepdim)r9   r:   r;   r<   )tor`   r>   catzipr   rX   rY   use_return_dictrJ   r   get_rope_indexlanguage_modelembed_tokenstypevisual	get_dtyper   image_token_id	unsqueeze	expand_asdevicemasked_scatterr;   rM   normr7   r:   r<   )r&   rR   rS   rT   r:   rU   rV   rW   rX   rY   rZ   r[   r\   r]   offsetspixel_sequenceoffsetrope_deltasimage_embeds
image_mask
vlm_outputvlm_hidden_stateslast_hidden_statesr9   s                           r(   forwardzColQwen2ForRetrieval.forward}   s   , #'???<L #(B$QT*^AqD-AAG 99GJ<Y`Gab-C^V(bL
 2C1N-TXT_T_TqTq %9$D $++JjJj 	 &1%<k$++B]B]$(HHNN$A$A))	 %B %
!k   HH33@@KM'+001J1J1LM#xx|nU$++"8"8"G"GGRRSUV``ano   ,}/C/C]EXEXY - < <Z V)!/!2!2=3G3G!HXX^^%)+'/!5#) $ 

 9MJ44RV']../AB
  *//b$/"GG
%#n&>&>r&BBJ)!&66+!,,	
 	
s cs   J
c                 6    | j                   j                         S N)rJ   get_input_embeddingsr&   s    r(   r   z)ColQwen2ForRetrieval.get_input_embeddings   s    xx,,..r*   c                 :    | j                   j                  |       y r   )rJ   set_input_embeddings)r&   values     r(   r   z)ColQwen2ForRetrieval.set_input_embeddings   s    %%e,r*   c                 6    | j                   j                         S r   )rJ   get_output_embeddingsr   s    r(   r   z*ColQwen2ForRetrieval.get_output_embeddings   s    xx--//r*   c                 :    | j                   j                  |       y r   )rJ   set_output_embeddings)r&   new_embeddingss     r(   r   z*ColQwen2ForRetrieval.set_output_embeddings   s    &&~6r*   c                 6    | j                   j                         S r   )rJ   tie_weightsr   s    r(   r   z ColQwen2ForRetrieval.tie_weights   s    xx##%%r*   new_num_tokenspad_to_multiple_ofmean_resizingc                 B   | j                   j                  |||      }|j                  | j                  j                  j
                  _        |j                  | j                  j                  _        |j                  | j                   _        |j                  | _        |S )N)r   r   r   )rJ   resize_token_embeddingsnum_embeddingsr   r   r   rH   )r&   r   r   r   model_embedss        r(   r   z,ColQwen2ForRetrieval.resize_token_embeddings   s     xx77)1' 8 
 9E8S8S**5,8,G,G)*99&55r*   )NNNNNNNNNNNNN)NNT)r+   r,   r-   _checkpoint_conversion_mappingr   rG   r   r   r   r>   
LongTensorr@   r	   r?   boolr7   r   r   r   r   r   r   intr   r$   r   __classcell__)rQ   s   @r(   rD   rD   \   s    &("~    151537+/-159$(,0/3&*/35959Z
E,,-Z
 !.Z
 u//0	Z

 "%Z
 ))*Z
   1 12Z
 D>Z
 $D>Z
 'tnZ
 d^Z
 u||,Z
 !!1!12Z
 !!1!12Z
 
$Z
  Z
x/-07&
 )-,0"	  %SM 	
 
r*   rD   )dataclassesr   typingr   r   r>   r   transformersr   cache_utilsr	   modeling_utilsr
   utilsr   r   r   r   configuration_colqwen2r   r   r7   rD   __all__r4   r*   r(   <module>r      s   , " "  4   - V V 2  ?o ? ?2 
: : :* Q2 QQh "#<
=r*   