rha[ddlmZmZmZddlZddlmZddlmZddl m Z ddl m Z m Z ddlmZmZddlmZdd lmZmZd d lmZmZmZmZmZmZmZd d lmZd d l m!Z!m"Z"ejFe$Z%GddeZ&GddeZ'Gdde!Z(GddeZ)GddeZ*GddeZ+GddeZ,GddeZ-eGddeZ.Gd d!eZ/gd"Z0y)#)CallableOptionalUnionN)PretrainedConfig)FlashAttentionKwargs)BaseModelOutputBaseModelOutputWithPooling)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringlogging)CLIPMLP CLIPAttention CLIPEncoderCLIPEncoderLayerCLIPVisionEmbeddingsCLIPVisionModelCLIPVisionTransformer)eager_attention_forward)VisionRotaryEmbeddingapply_rotary_pos_emb_visioncFeZdZdZdZdZ dfd ZxZS)MLCDVisionConfiga This is the configuration class to store the configuration of a [`MLCDVisionModel`]. It is used to instantiate a MLCD vision encoder according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the vision encoder of the MLCD [DeepGlint-AI/mlcd-vit-bigG-patch14-336](https://huggingface.co/DeepGlint-AI/mlcd-vit-bigG-patch14-336) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. Args: hidden_size (`int`, *optional*, defaults to 1664): Dimensionality of the encoder layers and the pooler layer. intermediate_size (`int`, *optional*, defaults to 8192): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. projection_dim (`int`, *optional*, defaults to 1024): Dimensionality of text and vision projection layers. num_hidden_layers (`int`, *optional*, defaults to 48): Number of hidden layers in the Transformer encoder. num_attention_heads (`int`, *optional*, defaults to 16): Number of attention heads for each attention layer in the Transformer encoder. num_channels (`int`, *optional*, defaults to 3): The number of input channels. image_size (`int`, *optional*, defaults to 336): The size (resolution) of each image. patch_size (`int`, *optional*, defaults to 14): The size (resolution) of each patch. hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the layer normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. initializer_factor (`float`, *optional*, defaults to 1.0): A factor for initializing all weight matrices (should be kept to 1, used internally for initialization testing). Example: ```python >>> from transformers import MLCDVisionConfig, MLCDVisionModel >>> # Initializing a MLCDVisionConfig with DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> configuration = MLCDVisionConfig() >>> # Initializing a MLCDVisionModel (with random weights) from the DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> model = MLCDVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```mlcd_vision_model vision_configc t|di|||_||_||_||_||_||_||_||_ | |_ | |_ | |_ | |_ | |_y)N)super__init__ hidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_groups num_channels patch_size image_sizeinitializer_rangeinitializer_factorattention_dropoutlayer_norm_eps hidden_act)selfr#r$r%r&r'r(r*r)r/r.r-r+r,kwargs __class__s x/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/mlcd/modular_mlcd.pyr"zMLCDVisionConfig.__init__dsz" "6"&!2!2#6 $8!($$!2"4!2,$) ii 0riPgelugh㈵>g{Gz??)__name__ __module__ __qualname____doc__ model_typebase_config_keyr" __classcell__r2s@r3rr*sH4 l%J%O%%r4rc eZdZy)MLCDMLPN)r<r=r>r r4r3rErEsr4rEc4eZdZdededej fdZy)MLCDRotaryEmbeddingnum_patches_heightnum_patches_widthreturnctj||jjj dj d|}tj||jjj dj |d}tj |j|jgd}t||}tj||jj|jj}tj||j}||jd} | S)a} Calculate the Rotary Position Embedding (RoPE) for MLCDVisionModel based on the grid size. Args: num_patches_height (int): Number of patches in the height dimension. num_patches_width (int): Number of patches in the width dimension. Returns: torch.Tensor: Rotary positional embeddings for the given grid size. )devicer7rdim)rLdtype) torcharangeinv_freqrL unsqueezeexpandstackflattenmaxrPouter) r0rHrIhpos_idswpos_idspos_ids max_grid_sizeseqrotary_pos_emb_fullrotary_pos_embs r3forwardzMLCDRotaryEmbedding.forwards  LL+DMM4H4H I S STU V ] ]^`bs t  LL*4==3G3G H R RST U \ \]oqs t  ++x//183C3C3EFBO.0AB ll=1E1ET]]M`M`a#kk#t}}=-W5==a@r4N)r<r=r>intrQTensorrar r4r3rGrGs ##%,,r4rGc\eZdZdeffd Zdej dejfdZxZ S)MLCDVisionEmbeddingsconfigc(t|||`yN)r!r"position_embeddingr0rfr2s r3r"zMLCDVisionEmbeddings.__init__s   #r4 pixel_valuesrJcT|jd}|jjj}|j|j |}|j dj dd}|jj|dd}tj||gd}|S)Nr)rPrr7rMrN) shapepatch_embeddingweightrPtorW transposeclass_embeddingrUrQcat)r0rk batch_size target_dtype patch_embeds class_embeds embeddingss r3razMLCDVisionEmbeddings.forwards!''* ++2288 ++LOO,O,OP #++A.88A> ++22:q"E YY l;C r4) r<r=r>rr"rQ FloatTensorrcrarBrCs@r3reres-$/$ E$5$5 %,, r4receZdZdZdeffd Z d dejdeejejfde ejde e deeje ejff d Z xZ S) MLCDAttentionzMulti-headed attention with RoPE. Refer to papers: - Attention is all you need: https://huggingface.co/papers/1706.03762 - RoFormer: Enhanced Transformer with Rotary Position Embedding: https://huggingface.co/papers/2104.09864 rfcTt|||j|_d|_yNF)r!r"r' is_causalrjs r3r"zMLCDAttention.__init__s%  $*$?$?!r4 hidden_statesposition_embeddingsattention_maskr1rJc |jdd\}}|j|j|||j|jf}|j |j|||j|jf}|j |j|||j|jf} |djdj} |djdj} t||| | \}}|jddddj}|jddddj}| jddddj} t} |jjdk7rt|jj} | |||| |f|j sdn |j"|j$|j&d|\} }| jddddj} | j)||d} |j+| } | jdddj} | |fS) NrMrr7rreagerr:)dropoutscalingr~)rmq_projreshape num_headshead_dimk_projv_projrTfloatrpermute contiguousrrf_attn_implementationr trainingrscaler~viewout_proj)r0rrrr1rt seq_length query_states key_states value_statescossinattention_interface attn_output attn_weightss r3razMLCDAttention.forwardsE"/!4!4Sb!9 J{{=199:zSWSaSacgcpcp:qr [[/77ZQUQ_Q_aeanan8op {{=199:zSWSaSacgcpcp:qr "!$..q1779!!$..q1779#>|ZY\^a#b j$++Aq!Q7BBD ''1a3>>@ #++Aq!Q7BBD (? ;; + +w 6"9$++:Z:Z"[ $7      % $}}C$,,JJnn %  % ! \"))!Q15@@B !&&z:rB mmK0 !))!Q2==? L((r4rh)r<r=r>r?rr"rQrctuplerr rrarBrCs@r3r{r{s/26 ,)||,)#5<<#=>,)!. ,) -. ,) u||Xell33 4 ,)r4r{c eZdZdeffd Z d dej deej ej fdeej dee deejf dZ xZ S) MLCDEncoderLayerrfcDt||t||_yrh)r!r"r{ self_attnrjs r3r"zMLCDEncoderLayer.__init__s  &v.r4rrroutput_attentionsrJc|}|j|}|j||||\}}||z}|}|j|}|j|}||z}|f}|r||fz }|S)a Args: hidden_states (`torch.FloatTensor`): Input to the layer of shape `(batch, seq_len, embed_dim)`. Represents the hidden states from the previous layer or the input embeddings. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.FloatTensor`): Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values. output_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. rrrr) layer_norm1r layer_norm2mlp)r0rrrrresidualroutputss r3razMLCDEncoderLayer.forwards*!((7 &*nn' 3)/ '5' # | !=0  ((7 /  =0  "    &Gr4r}) r<r=r>rr"rQrcrrboolryrarBrCs@r3rrs{///26,1 *||*#5<<#=>*!. * $D> * u  ! *r4rceZdZdZdeffd Z d dejdeejejfde ejde e de e d e e d e ee ffd ZxZS) MLCDEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`MLCDEncoderLayer`]. Args: config: MLCDVisionConfig rfc$t||y)z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.N)r!r"rjs r3r"zMLCDEncoder.__init__5s  r4 inputs_embedsrrroutput_hidden_states return_dictrJc||n|jj}||n|jj}||n|jj}|rdnd}|rdnd}|} t |j D]*\} } |r|| fz}| | |||} | d} |s"|| dfz},|r|| fz}|st d| ||fDSt| ||S)aj Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr rrr7c3&K|] }|| ywrhr ).0vs r3 z&MLCDEncoder.forward..wseqWXWdes)last_hidden_stater attentions)rfruse_return_dictr enumeratelayersrr ) r0rrrrrrencoder_statesall_attentionsridx encoder_layer layer_outputss r3razMLCDEncoder.forward9sD%9$D $++JjJj &1%>Ne]NN$See e+(%  r4NNNN)r<r=r>r?rr"rQryrrcrrrr rarBrCs@r3rr,s!/!26,0/3&*C ((C #5<<#=>C !. C $D> C 'tn C d^C  uo% &C r4rceZdZdeffd Ze d deejdee dee dee de e e ff dZ xZS) MLCDVisionTransformerrfct||t|j|jzdz|_t jtjd|j|jzdz|_ y)Nrr7) r!r"rGr#r&vision_rotary_embeddingnn ParameterrQrandn class_pos_embrjs r3r"zMLCDVisionTransformer.__init__sh  ':6;M;MQWQkQk;kop;p'q$\\%++a9K9KvOiOi9imn9n*opr4rkrrrrJc||n|jj}||n|jj}||n|jj}| t d|j d|jj z}|j d|jj z}|j||}|j|jj}tj|j|gd}tj||fd}|j|jf} |j|} |j!| } |j#| | |||} | d} | dddddf} |j%| } |s | | f| ddzSt'| | | j(| j*S) Nz You have to specify pixel_valuesrMrrN)rrrrrr7)r pooler_outputrr)rfrrr ValueErrorrmr)rrprrLrQrsrrrx pre_layrnormencoderpost_layernormr rr)r0rkrrrrHrIr`embrrencoder_outputsr pooled_outputs r3razMLCDVisionTransformer.forwards%9$D $++JjJj &1%rr"rrrQryrrrr rarBrCs@r3rrsq/q 59,0/3&* / u001/ $D>/ 'tn / d^ / u00 1 / / r4rc.eZdZUeed<dZdZdZdZdZ y)MLCDPreTrainedModelrfmlcdTcL|jj}t|tr|jj}tj j |jd|jdz|ztj j |jj|jj|zyt|tr,|jj}|jdzd|jjzdzz|z}|jdz|z}tj j |jj|tj j |jj|tj j |j j|tj j |j"j|yt|t$r|jj}|jj&dzd|jjzdzz|z}d|jj&zdz|z}tj j |j(j|tj j |j*j|yt|t,ry|jj}|jj&|jj.zdzdz|z}tj j |j0d|yt|tj2rJ|j4j6j9|jj6j;dyt|tj<r2|j4%|j4j6j9yyy)zInitialize the weightsr:g)meanstd)rrr;N)rfr, isinstancererinitnormal_rr embed_dimrnror+r{r%rrrrrEr#fc1fc2rr&r LayerNormbiasdatazero_fill_Linear)r0modulefactor in_proj_std out_proj_stdfc_std pos_emb_stds r3 _init_weightsz!MLCDPreTrainedModel._init_weightss// f2 3[[33F GGOOF22&BRBRTXBX[aBaO b GGOOF2299v}}?^?^ag?gO h  .[[33F!++T1q6==;Z;Z7Z_c6cdgmmK",,d2fr__annotations__base_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdparr r4r3rrs$ &*#N%r4rcreZdZe ddeej deedeedeedee e ff dZ y) MLCDVisionModelNrkrrrrJc||n|jj}||n|jj}||n|jj}|j ||||S)a Example: ```python >>> import requests >>> from PIL import Image >>> from transformers import AutoProcessor, MLCDVisionModel >>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs, output_attentions=True) >>> features = outputs.last_hidden_state >>> print(f"Extracted features shape: {features.shape}") >>> print(f"Number of attention layers: {len(outputs.attentions)}") >>> print(f"Attention shape: {outputs.attentions[0].shape}") ```)rkrrr)rfrrr vision_model)r0rkrrrs r3razMLCDVisionModel.forwardsu>%9$D $++JjJj &1%rrrQryrrrr rar r4r3rrst59,0/3&* ( u001( $D>( 'tn ( d^ ( u00 1 ( ( r4r)rrr)1typingrrrrQtorch.nnrconfiguration_utilsrmodeling_flash_attention_utilsrmodeling_outputsr r modeling_utilsr r processing_utilsr utilsrrclip.modeling_cliprrrrrrrllama.modeling_llamarqwen2_vl.modeling_qwen2_vlrr get_loggerr<loggerrrErGrer{rrrrr__all__r r4r3rs-, 3BKF&,;[   H %Y%'Y%x g /D/$9)M9)x/'/dP +P f6 16 r$%/$%$%N* o* Z r4