
    rhj                       d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	Z
d dlZd dlmZ d dlmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZmZ ddlmZmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z. ddl/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z=m>Z>m?Z?m@Z@mAZA ddlBmCZCmDZD ddlEmFZF ddlGmHZH ddlImJZJmKZKmLZL  e3       rd dlZd dlMmZ d dlNmc mOZP d dlQZ e4       rd dlRZRddlSmTZT ddl6mUZUmVZV  e5j                  eX      ZY G d deH      ZZ G d  d!e;      Z[ G d" d#eT      Z\e1 G d$ d%e,             Z]e e1d&'       G d( d)e)                    Z^ G d* d+eC      Z_ G d, d-eD      Z` G d. d/eL      Za G d0 d1ej                        Zc G d2 d3ej                        Zd G d4 d5eK      Ze G d6 d7eJ      Zf G d8 d9e9      Zg G d: d;ej                        Zh G d< d=eA      Zi G d> d?e@      Zj G d@ dAe>      Zk G dB dCe?      Zl G dD dEej                        Zm G dF dGej                        Zn G dH dIej                        Zo G dJ dKej                        Zp G dL dMe=      Zq G dN dOej                        Zr G dP dQej                        Zs e1dR'       G dS dTe]             Zt G dU dVe]e      Zu G dW dXe      Zvg dYZwy)Z    N)Iterable)	dataclass)CallableOptionalUnion)nn)BlipImageProcessor   )ACT2FN)Cache)%ClassifierFreeGuidanceLogitsProcessorGenerationMixinGenerationModeLogitsProcessorList)GenerateDecoderOnlyOutput)BatchFeatureget_size_dict)resizeto_channel_dimension_format)ChannelDimension
ImageInputPILImageResamplingget_image_sizeinfer_channel_dimension_formatmake_list_of_imagesto_numpy_array)ModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torch_availableis_vision_availablelogging   )	AutoModel)Blip2VisionModel)ChameleonVQVAEConfig)ChameleonVQVAEChameleonVQVAEEncoderAttnBlock#ChameleonVQVAEEncoderConvDownsample ChameleonVQVAEEncoderResnetBlockChameleonVQVAEVectorQuantizer)IdeficsBaseModelOutputWithPastIdeficsCausalLMOutputWithPast)eager_attention_forward)SiglipVisionConfig)SiglipEncoderSiglipEncoderLayerSiglipVisionEmbeddings)PretrainedConfig)CONFIG_MAPPING
AutoConfigc                   P     e Zd ZdZdZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )JanusVisionConfiga
  
    This is the configuration class to store the configuration of a [`JanusVisionModel`]. It is used to instantiate a
    `JanusVisionModel` according to the specified arguments, defining the model architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.
    Args:
        hidden_size (`int`, *optional*, defaults to 1024):
            Dimensionality of the encoder layers and the pooler layer.
        num_hidden_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer encoder.
        num_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer encoder.
        num_channels (`int`, *optional*, defaults to 3):
            The number of input channels.
        patch_size (`int`, *optional*, defaults to 16):
            The size (resolution) of each patch.
        image_size (`int`, *optional*, defaults to 384):
            The size (resolution) of each image.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            Dropout probability for attention weights.
        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the layer normalization layers.
        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"selu"`, and `"gelu_new"` are supported.
        mlp_ratio (`float`, *optional*, defaults to 4.0):
            Ratio of MLP hidden dimensionality to embedding dimensionality.
        attention_bias (`bool`, *optional*, defaults to `True`):
            Whether to add a bias to the queries, keys, and values in the attention layers.
        hidden_dropout_rate (`float`, *optional*, defaults to 0.0):
            The dropout probability for fully connected layers in the encoder.
        projection_dim (`int`, *optional*, defaults to 2048):
            Dimensionality of the MLP projection head.
        projection_dropout (`float`, *optional*, defaults to 0.0):
            Dropout probability for the projection layer.
        use_qk_norm (`bool`, *optional*, defaults to `False`):
            Whether to normalize the query and key matrices.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated normal initializer for initializing all weight matrices.
        depth (`int`, *optional*, defaults to 2):
            Number of hidden layers in the aligner module.
        num_image_tokens (`int`, *optional*, defaults to 576):
            Number of image tokens.
    janus_vision_modelvision_configc                     t        |   d|||||||||	d	| | `|
| _        || _        || _        || _        || _        || _        || _	        || _
        || _        y )N)	hidden_sizenum_hidden_layersnum_attention_headsnum_channels
patch_size
image_sizeattention_dropoutlayer_norm_eps
hidden_act )super__init__intermediate_size	mlp_ratioattention_biashidden_dropout_rateprojection_dimprojection_dropoutuse_qk_norminitializer_rangedepthnum_image_tokens)selfr?   r@   rA   rB   rC   rD   rE   rF   rG   rL   rM   rN   rO   rP   rQ   rR   rS   rT   kwargs	__class__s                       z/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/janus/modular_janus.pyrJ   zJanusVisionConfig.__init__   s    , 	 	
#/ 3%!!/)!	
 	
 "",#6 ,"4&!2
 0    )i         r
   r[   i          ư>gelug      @Tr\      r\   F{Gz?r'   i@  )__name__
__module____qualname____doc__
model_typebase_config_keyrJ   __classcell__rW   s   @rX   r;   r;   W   sW    ,\ &J%O ',1 ,1rY   r;   c                   |     e Zd ZdZddddddddg d	d
dddd
ddfdededededededededee   dedef fdZ xZ	S )JanusVQVAEConfiga:
  
    This is the configuration class to store the configuration of a [`JanusVQVAEModel`]. It is used to instantiate a
    `JanusVQVAEModel` according to the specified arguments, defining the model architecture.
    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information. Instantiating a
    configuration with the defaults will yield a similar configuration to the VQModel of the
    [deepseek-community/Janus-Pro-1B](https://huggingface.co/deepseek-community/Janus-Pro-1B).

    Args:
        embed_dim (`int`, *optional*, defaults to 8):
            Dimensionality of each embedding vector.
        num_embeddings (`int`, *optional*, defaults to 16384):
            Number of codebook embeddings.
        double_latent (`bool`, *optional*, defaults to `False`):
            Whether to use double z channels.
        latent_channels (`int`, *optional*, defaults to 256):
            Number of channels for the latent space.
        num_patches (`int`, *optional*, defaults to 32):
            Num of patches the input images can be divided into.
        in_channels (`int`, *optional*, defaults to 3):
            Number of input channels.
        out_channels (`int`, *optional*, defaults to 3):
            Number of out channels.
        base_channels (`int`, *optional*, defaults to 128):
            Base channel count.
        channel_multiplier (`list[int]`, *optional*, defaults to `[1, 1, 2, 2, 4]`):
            Channel multipliers for each resolution.
        num_res_blocks (`int`, *optional*, defaults to 2):
            Number of residual blocks.
        dropout (`float`, *optional*, defaults to 0.0):
            Dropout rate.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        projection_dim (`int`, *optional*, defaults to 2048):
            Dimensionality of the MLP projection head.
        num_hidden_layers (`int`, *optional*, defaults to 2):
            Number of hidden layers in VAVAE MLP Connecter module.
        hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"silu"` and `"gelu_new"` are supported.
        image_token_embed_dim (`int`, *optional*, defaults to 2048):
            Dimension of image embeddings. It should be same as the dimensionality of text embeddings.
       i @  F       r
      )   ro   r'   r'      r'   r\   r`   r_   r^   	embed_dimnum_embeddingsdouble_latentlatent_channelsnum_patchesin_channelsout_channelsbase_channelschannel_multipliernum_res_blocksdropoutc                     t        |   d|||||||	|
||d
| || _        || _        || _        || _        || _        || _        | `| `	| `
y )N)
rq   rr   rs   rt   rv   rx   ry   rz   r{   rR   rH   )rI   rJ   ru   rw   rO   r@   rG   image_token_embed_dim
resolutionattn_resolutions	attn_type)rU   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   rR   rO   r@   rG   r}   rV   rW   s                     rX   rJ   zJanusVQVAEConfig.__init__   s    ( 	 	
)'+#'1)/	
 	
 '(,!2$%:"O!NrY   )
ra   rb   rc   rd   intboollistfloatrJ   rg   rh   s   @rX   rj   rj      s    *\ ##" (7"#** * 	*
 * * * * * !I* * * *rY   rj   c                   <     e Zd ZdZdZeeedZ	 	 	 	 d fd	Z	 xZ
S )JanusConfiga;  
    This is the configuration class to store the configuration of a [`JanusModel`]. It is used to instantiate an
    Janus model according to the specified arguments, defining the model architecture. Instantiating a configuration
    with the defaults will yield a similar configuration to that of the Janus-1B or Janus-7B models.

    e.g. [deepseek-community/Janus-Pro-1B](https://huggingface.co/deepseek-community/Janus-Pro-1B) or
    [deepseek-community/Janus-Pro-7B](https://huggingface.co/deepseek-community/Janus-Pro-7B)

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `LlamaConfig`):
            The config object or dictionary of the text backbone.
        vision_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `JanusVisionConfig`):
            The config object or dictionary of the vision backbone.
        vq_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `JanusVQVAEConfig`):
            The config object or dictionary of the VQVAE backbone.
        image_token_id (`int`, *optional*, defaults to 100581):
            Token index of a placeholder image token.

    Example:

    ```python
    >>> from transformers import JanusForConditionalGeneration, JanusConfig, JanusVisionConfig, JanusVQVAEConfig, LlamaConfig

    >>> # Initializing a Janus vision config
    >>> vision_config = JanusVisionConfig()

    >>> # Initializing a Llama config
    >>> text_config = LlamaConfig()

    >>> # Initializing a VQ config
    >>> vq_config = JanusVQVAEConfig()

    >>> # Initializing a Janus Pro 1B style configuration
    >>> configuration = JanusConfig(vision_config=vision_config, text_config=text_config, vq_config=vq_config)

    >>> # Initializing a model from the Janus Pro 1B style configuration
    >>> model = JanusForConditionalGeneration(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```janus)text_configr=   	vq_configc                    t        |t              r,|j                  dd      |d<   t        |d      d	i || _        nY|(t
        j                  d       t        d          | _        n/t        |t              r|| _        nt        dt        |             |%t
        j                  d       t               | _        nPt        |t              rt        d	i || _        n/t        |t              r|| _        nt        dt        |             |%t
        j                  d       t               | _        nPt        |t              rt        d	i || _        n/t        |t              r|| _        nt        dt        |             | j                  j                  | _        | j                  j                  | j                  j                   z  | j                  _        || _        t'        | P  d	i | y )
Nre   llamaz7`text_config` is None. Initializing with default valueszTInvalid type for `text_config`. Must be either `dict` or `LlamaConfig`. Type found: zK`vision_config` is None. Initializing with default JanusVisionConfig valuesz\Invalid type for `vision_config`. Must be either `dict` or `JanusVisionConfig`. Type found: zF`vq_config` is None. Initializing with default JanusVQVAEConfig valueszWInvalid type for `vq_config`. Must be either `dict` or `JanusVQVAEConfig`. Type found: rH   )
isinstancedictgetr8   r   loggerinfor7   
ValueErrortyper;   r=   rj   r   rR   rD   rC   ru   image_token_idrI   rJ   )rU   r   r=   r   r   rV   rW   s         rX   rJ   zJanusConfig.__init__G  s    k4((3g(NK%-k,.GHW;WD KKQR-g68D%56*D  $[ 124 
  KKef!2!4Dt,!2!C]!CD'89!.D  $] 346 
 KK`a-/DN	4(-:	:DN	#34&DN  $Y02 
 "&!3!3!E!E%)%7%7%B%BdFXFXFcFc%c","6"rY   )NNNi )ra   rb   rc   rd   re   r9   r;   rj   sub_configsrJ   rg   rh   s   @rX   r   r     s8    +Z J!*%K 6# 6#rY   r   c                   @    e Zd ZU eed<   dZdZddgZddgZdZ	dZ
dZdZy	)
JanusPreTrainedModelconfigmodelTLlamaDecoderLayerJanusVisionEncoderLayerpast_key_valuescausal_maskFN)ra   rb   rc   r   __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraph!_supports_param_buffer_assignmentrH   rY   rX   r   r     sB    &*#,.GH#4m"DN!(-%rY   r   z9
    Base class for Janus VQ-VAE mode model outputs.
    )custom_introc                   \    e Zd ZU dZdZeej                     ed<   dZ	ej                  ed<   y)JanusVQVAEOutputz
    decoded_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        Reconstructed pixel values after encoding and decoding the input.
    embedding_loss (`torch.FloatTensor`):
        Embedding loss.
    Ndecoded_pixel_valuesembedding_loss)
ra   rb   rc   rd   r   r   torchFloatTensorr   r   rH   rY   rX   r   r     s/     9=(5#4#45<(,NE%%,rY   r   c                       e Zd Zy)JanusBaseModelOutputWithPastNra   rb   rc   rH   rY   rX   r   r         rY   r   c                       e Zd Zy)JanusCausalLMOutputWithPastNr   rH   rY   rX   r   r     r   rY   r   c                   J    e Zd Zddej                  dedej                  fdZy)JanusVisionEmbeddingspixel_valuesinterpolate_pos_encodingreturnc                 X   |j                   \  }}}}| j                  j                  j                  }| j                  |j	                  |            }|j                  d      j                  dd      }|r| j                  |||      }	n| j                  | j                        }	||	z   }|S )Ndtyper'   ro   )
shapepatch_embeddingweightr   toflatten	transposer   position_embeddingposition_ids)
rU   r   r   _heightwidthtarget_dtypepatch_embeds
embeddings
pos_embedss
             rX   forwardzJanusVisionEmbeddings.forward  s    *001fe++2288++LOO,O,OP!))!,66q!<
#66z65QJ001B1BCJ*,
rY   N)F)ra   rb   rc   r   Tensorr   r   rH   rY   rX   r   r     s'    ELL D ]b]i]i rY   r   c                   t     e Zd ZdZdef fdZ	 ddej                  deej                     de	e
   fdZ xZS )	JanusVisionAttentionz(Attention Class for Janus Vision Encoderr   c                 F   t         |           || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                   d| j                   d      | j                  dz  | _	        |j                  | _
        |j                  }|j                  }d| _        d| _        t        j                   | j                  | j                  | j                  z  |j"                        | _        t        j                   | j                  | j                  | j                  z  |j"                        | _        t        j                   | j                  | j                  | j                  z  |j"                        | _        t        j                   | j                  | j                        | _        |dkD  rt        j,                  |      nt        j.                         | _        |rt        j0                  | j                        nt        j.                         | _        |r%t        j0                  | j                        | _        y t        j.                         | _        y )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      Fro   biasr   )rI   rJ   r   r?   rq   rA   	num_headshead_dimr   scalerE   rP   rQ   	is_causalnum_key_value_groupsr   LinearrM   q_projk_projv_projprojection_layerDropoutIdentity	LayerNormq_normk_norm)rU   r   proj_dropoutqk_normrW   s       rX   rJ   zJanusVisionAttention.__init__  s   ++33$..8==4>>)T^^;MdnnM] ^NN#2'  ]]D(
!'!9!900$$ %&!ii0NU[UjUjkii0NU[UjUjkii0NU[UjUjk "		$..$.. I>JQ>N"**\":TVT_T_Ta6=bll4>>22;;=6=bll4>>22;;=rY   hidden_statesattention_maskrV   c                 >   |j                         \  }}}| j                  |      }| j                  |      }| j                  |      }	|j	                  d| j
                  | j                        }| j                  |      }|j	                  d| j
                  | j                        }| j                  |      }|j	                  ||| j
                  | j                        j                  dd      }|j	                  ||| j
                  | j                        j                  dd      }|	j                  ||| j
                  | j                        j                  dd      }	t        }
| j                  j                  dk7  rt        | j                  j                     }
 |
| |||	|f| j                  sdn| j                   | j"                  | j$                  d|\  }}|j	                  ||| j&                        }| j)                  |      }| j+                  |      }||fS )Nro   r'   eagerr\   )r{   scalingr   )sizer   r   r   reshaper   r   r   r   r   viewr2   r   _attn_implementationr   trainingrE   r   r   rq   r   rP   )rU   r   r   rV   
batch_sizeseq_lenr   query_states
key_statesvalue_statesattention_interfaceattn_outputattn_weightsoutputs                 rX   r   zJanusVisionAttention.forward  s    "/!3!3!5
GQ{{=1[[/
{{=1#++BN{{<0''DNNDMMJ
[[,
#++JQUQ^Q^_iijkmno''
GT^^T]][eefgijk
#((Wdnndmm\ffghjkl(?;;++w6"9$++:Z:Z"[$7
%
  $}}C$2H2HJJnn
%
 
%
!\ "))*gt~~N&&{3((0|##rY   N)ra   rb   rc   rd   r;   rJ   r   r   r   r    r!   r   rg   rh   s   @rX   r   r     sO    2Q0 Q@ 26)$||)$ !.)$ +,	)$rY   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )JanusVisionMLPr   c                    t         |           || _        t        |j                  |j
                  z        | _        t        |j                     | _	        t        j                  |j                  | j                        | _        t        j                  | j                  |j                        | _        t        j                  |j                        | _        t        j                  |j                        | _        y r   )rI   rJ   r   r   r?   rL   rK   r   rG   activation_fnr   r   fc1fc2r   rN   dropout1dropout2rU   r   rW   s     rX   rJ   zJanusVisionMLP.__init__  s    !$V%7%7&:J:J%J!K#F$5$5699V//1G1GH99T33V5G5GH

6#=#=>

6#=#=>rY   r   r   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S r   )r   r   r   r   r   rU   r   s     rX   r   zJanusVisionMLP.forward  sP    /**=9m4/m4rY   )	ra   rb   rc   r;   rJ   r   r   r   rg   rh   s   @rX   r   r     s+    ?0 ?U\\ ell rY   r   c                   $     e Zd Zdef fdZ xZS )r   r   c                 R   t         |           || _        |j                  | _        t        |      | _        t        j                  | j                  |j                        | _
        t        j                  | j                  |j                        | _        t        |      | _        y )N)eps)rI   rJ   r   r?   rq   r   	self_attnr   r   rF   layer_norm1layer_norm2r   mlpr   s     rX   rJ   z JanusVisionEncoderLayer.__init__  st    ++-f5<<F<Q<QR<<F<Q<QR!&)rY   ra   rb   rc   r;   rJ   rg   rh   s   @rX   r   r     s    *0 * *rY   r   c                   $     e Zd Zdef fdZ xZS )JanusVisionEncoderr   c                     t         |   |       t        j                  t	        |j
                        D cg c]  }t        |       c}      | _        y c c}w r   )rI   rJ   r   
ModuleListranger@   r   layersrU   r   r   rW   s      rX   rJ   zJanusVisionEncoder.__init__%  s@     mmeTZTlTlNm$n%<V%D$no$ns   Ar	  rh   s   @rX   r  r  $  s    p0 p prY   r  c                   $     e Zd Zdef fdZ xZS )JanusVisionModelr   c                 D    t         |   |       t        |      | _        y r   )rI   rJ   r  encoderr   s     rX   rJ   zJanusVisionModel.__init__+  s     )&1rY   r	  rh   s   @rX   r  r  *  s    20 2 2rY   r  c                   *     e Zd Zdef fdZd Z xZS )JanusVisionAlignerMLPr   c           	         t         |           t        j                  |j                  |j
                        | _        t        j                  t        d|j                        D cg c],  }t        j                  |j
                  |j
                        . c}      | _
        t        |j                     | _        y c c}w Nro   )rI   rJ   r   r   r?   rO   r   r  r  rS   hidden_layersr   rG   r   r  s      rX   rJ   zJanusVisionAlignerMLP.__init__1  s    99V//1F1FG]]NSTUW]WcWcNdeRYYv,,f.C.CDe
 $F$5$56 f   &1B<c                 |    | j                  |      }| j                  D ]  }| j                  |      } ||      } |S r   r   r  r   rU   r   layers      rX   r   zJanusVisionAlignerMLP.forward:  G    /'' 	1E ..}=M!-0M	1 rY   )ra   rb   rc   r;   rJ   r   rg   rh   s   @rX   r  r  0  s    70 7rY   r  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZ	S )JanusVQVAEVectorQuantizerr   c                 N    t         |   |       |j                  gdz  | _        y )Nr'   )rI   rJ   ru   quant_state_dimsr   s     rX   rJ   z"JanusVQVAEVectorQuantizer.__init__C  s&     !'!3!3 4q 8rY   image_tokensr   c                 B   |j                   d   }| j                  j                  j                   d   }| j                  |      }t        j                  |dd      }|j                  |g| j                  |      }|j                  dddd      j                         }|S )Nr   r   r'   )pdimr
   ro   )	r   	embeddingr   F	normalizer   r#  permute
contiguous)rU   r$  r   emb_dimhidden_state_quants        rX   get_codebook_entryz,JanusVQVAEVectorQuantizer.get_codebook_entryG  s    !''*
~~,,2226 "^^L9[[);qbI 044j5b4CXCX5bZa5bc/771aCNNP!!rY   )
ra   rb   rc   rj   rJ   r   
LongTensorr   r/  rg   rh   s   @rX   r!  r!  B  s/    9/ 9"u/?/? "EDUDU "rY   r!  c                       e Zd Zy)JanusVQVAEResnetBlockNr   rH   rY   rX   r2  r2  W  r   rY   r2  c                       e Zd Zy)JanusVQVAEAttnBlockNr   rH   rY   rX   r4  r4  [  r   rY   r4  c                       e Zd Zy)JanusVQVAEConvDownsampleNr   rH   rY   rX   r6  r6  _  r   rY   r6  c                   $     e Zd Z fdZd Z xZS )JanusVQVAEConvUpsamplec                 t    t         |           t        j                  j	                  ||ddd      | _        y )Nr
   ro   kernel_sizestridepadding)rI   rJ   r   r   Conv2dconv)rU   rv   rW   s     rX   rJ   zJanusVQVAEConvUpsample.__init__d  s.    HHOOK!TU_`Oa	rY   c                 X    t        j                  |dd      }| j                  |      }|S )Ng       @nearest)scale_factormode)r)  interpolater?  r  s     rX   r   zJanusVQVAEConvUpsample.forwardh  s(    m#IV		-0rY   )ra   rb   rc   rJ   r   rg   rh   s   @rX   r8  r8  c  s    brY   r8  c                   `     e Zd Zdedef fdZdej                  dej                  fdZ xZ	S )JanusVQVAEMidBlockr   channelsc                     t         |           t        |||      | _        t	        |      | _        t        |||      | _        y )Nr   rv   rw   )rI   rJ   r2  block_1r4  attn_1block_2)rU   r   rG  rW   s      rX   rJ   zJanusVQVAEMidBlock.__init__o  sF    , !

 *(3, !
rY   r   r   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )rJ  rK  rL  r  s     rX   r   zJanusVQVAEMidBlock.forward}  s2    ]3M2]3rY   )
ra   rb   rc   rj   r   rJ   r   r   r   rg   rh   s   @rX   rF  rF  n  s2    
/ 
3 
U\\ ell rY   rF  c                   >     e Zd Z fdZdej
                  fdZ xZS )JanusVQVAEEncoderc           	         t         |           t        |j                        | _        |j
                  | _        |j                  }|j                  }|j                  }|j                  }|j                  }t        j                  j                  ||ddd      | _        dt        |      z   }|| _        t        j                          | _        t%        | j                        D ]   }t        j                          }	t        j                          }
|||   z  }|||   z  }t%        | j
                        D ]N  }|	j'                  t)        |||             |}|| j                  dz
  k(  s5|
j'                  t+        |             P t        j,                         }|	|_        |
|_        || j                  dz
  k7  rt3        |      |_        | j"                  j'                  |        t7        |      | _        t        j                  j;                  d|dd	      | _        t        j                  j                  ||rd
|z  n|ddd      | _        y )Nr
   ro   r:  )ro   rI  rm   r]   T
num_groupsrB   r  affiner'   ) rI   rJ   lenry   num_resolutionsrz   rx   rv   rs   rt   r   r   r>  conv_intuplein_channel_multiplierr  downr  appendr2  r4  Moduleblockattnr6  
downsamplerF  mid	GroupNormnorm_outconv_out)rU   r   rx   rv   rs   rt   ry   rX  i_levelr\  r]  block_in	block_outi_blockrY  rW   s                  rX   rJ   zJanusVQVAEEncoder.__init__  s   "6#<#<=$33,,((,, 00#66xx{MqYZdef $u-?'@ @%:"MMO	T112 	#GMMOE==?D$'<W'EEH%(:7(CCI !4!45 
?)%$,%. %d22Q66KK 3H =>
? 99;DDJDI$..22":8"DIIT"-	#0 &fh7**bxUYbf*g#0Ao ( 
rY   r   c                    | j                  |      g}t        | j                        D ]  }t        | j                        D ]  } | j                  |   j
                  |   |d         }t        | j                  |   j                        dkD  r" | j                  |   j                  |   |      }|j                  |        || j                  dz
  k7  s|j                  | j                  |   j                  |d                 |d   }| j                  |      }| j                  |      }|t        j                  |      z  }| j                  |      }|S )Nr   r   ro   )rV  r  rU  rz   rY  r\  rT  r]  rZ  r^  r_  ra  r   sigmoidrb  )rU   r   r   rc  rf  hidden_statelast_hidden_states          rX   r   zJanusVQVAEEncoder.forward  sT   l34T112 		WG !4!45 3@tyy177@!"%  tyy)../!3#C499W#5#:#:7#CL#QL$$\23 $..22$$TYYw%7%B%B=QSCT%UV		W *"- HH%67 !MM*;<U]]+<== MM*;<  rY   )ra   rb   rc   rJ   r   r0  r   rg   rh   s   @rX   rO  rO    s    1
f!E$4$4 !rY   rO  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )JanusVQVAEDecoderc           	      v   t         |           t        |j                        | _        |j
                  | _        |j                  }|j                  }|j                  }||j                  | j                  dz
     z  }t        j                  j                  ||ddd      | _        t        ||      | _        t        j                         | _        t#        t%        | j                              D ]  }t        j                         }t        j                         }||j                  |   z  }	t%        | j
                  dz         D ]N  }
|j'                  t)        |||	             |	}|| j                  dz
  k(  s5|j'                  t+        |             P t        j,                         }||_        ||_        |dk7  rt3        |      |_        | j                   j'                  |        t        j                  j7                  d|dd	      | _        t        j                  j                  ||ddd      | _        y )
Nro   r
   r:  rI  r   rm   r]   TrQ  )rI   rJ   rT  ry   rU  rz   rx   rt   rw   r   r   r>  rV  rF  r_  r  upreversedr  rZ  r2  r4  r[  r\  r]  r8  upsampler`  ra  rb  )rU   r   rx   rt   rw   rd  rc  r\  r]  re  rf  rn  rW   s               rX   rJ   zJanusVQVAEDecoder.__init__  s   "6#<#<=$33,, 00** !6#<#<T=Q=QTU=U#VV xxaXYcde &fh7 --/d&:&: ;< 	GMMOE==?D%(A(A'(JJI !4!4q!89 
?)%$,%. %d22Q66KK 3H =>
? BBHBG!|4X>GGNN2)	. **bxUYbf*g,AVWabcrY   ri  r   c                 b   | j                  |      }| j                  |      }t        | j                        D ]  }t        | j                  dz         D ]l  } | j
                  |   j                  |   |      }t        | j
                  |   j                        dkD  sK | j
                  |   j                  |   |      }n || j                  dz
  k7  s| j
                  |   j                  |      } | j                  |      }|t        j                  |      z  }| j                  |      }|S )Nro   r   )rV  r_  r  rU  rz   rn  r\  rT  r]  rp  ra  r   rh  rb  )rU   ri  rc  rf  s       rX   r   zJanusVQVAEDecoder.forward   s    ||L1 xx- T112 	GG !4!4q!89 P>twww/55g>|Ltwww',,-1#A4777#3#8#8#A,#OLP $..22#www/88F	G }}\2l33}}\2rY   )ra   rb   rc   rJ   r   r   r   rg   rh   s   @rX   rl  rl    s)    ,d\E$5$5 %:K:K rY   rl  c                        e Zd Zg dZdZdef fdZdej                  dej                  fdZ
eedej                  deej                  ej                  f   fd              Z xZS )	
JanusVQVAE)r4  r2  r!  r   r   c                 r    t         |   |       t        |      | _        d| _        | j                          y )NF)rI   rJ   rl  decodergradient_checkpointing	post_initr   s     rX   rJ   zJanusVQVAE.__init__  s0     (0&+# 	rY   r$  r   c                    |j                   d   | j                  j                  d   | j                  j                  d   z  k7  rMt        d| j                  j                  d   | j                  j                  d   z   d|j                    d      | j                  j	                  |      }| j                  |      }| j                  |      }|S )aG  
        Decodes quantized token IDs into pixel values.
        Args:
            image_tokens (torch.LongTensor): Batch of token IDs.
        Returns:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                Pixel values decoded from the token IDs.
        ro   r   z4Expected `image_tokens` to have shape `(batch_size, z)`, but got shape `z`.)r   quantizer#  r   r/  post_quant_convru  )rU   r$  codebook_entryr   r   s        rX   decodezJanusVQVAE.decode%  s     a DMM$B$B1$EHfHfghHi$iiFt}}GeGefgGhkokxkx  lJ  lJ  KL  lM  HM  GN N""."4"4!5R9  99,G,,^<||M2rY   c                     |j                   d   }| j                  |      \  }}}| j                  |j                  |d            }t	        ||      S )Nr   r   )r   encoder|  r   r   )rU   r   r   quantr   indicesr   s          rX   r   zJanusVQVAE.forward8  sQ     "''*
)-\)B&~w#{{7<<
B+GH 4nEErY   )ra   rb   rc   r   main_input_namerj   rJ   r   r0  r   r|  r#   r"   rW  r   rg   rh   s   @rX   rs  rs    s    
 %O/ 5#3#3 8I8I & F''F 
u  %"3"33	4F  FrY   rs  c                   *     e Zd Zdef fdZd Z xZS )JanusVQVAEAlignerMLPr   c           	         t         |           t        j                  |j                  |j
                        | _        t        j                  t        d|j                        D cg c],  }t        j                  |j
                  |j
                        . c}      | _
        t        |j                     | _        y c c}w r  )rI   rJ   r   r   rq   rO   r   r  r  r@   r  r   rG   r   r  s      rX   rJ   zJanusVQVAEAlignerMLP.__init__F  s    99V--v/D/DE]]NSTUW]WoWoNpqRYYv,,f.C.CDq
 $F$5$56 rr  c                 |    | j                  |      }| j                  D ]  }| j                  |      } ||      } |S r   r  r  s      rX   r   zJanusVQVAEAlignerMLP.forwardO  r  rY   )ra   rb   rc   rj   rJ   r   rg   rh   s   @rX   r  r  E  s    7/ 7rY   r  c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ	 xZ
S )JanusVQVAEHeadzOHead used for sampling tokens in image generation, replacing the usual lm head.r   c                    t         |           t        j                  |j                  |j
                        | _        t        |j                     | _	        t        j                  |j
                  |j                        | _        y r   )rI   rJ   r   r   r}   rO   proj_outr   rG   r   rr   vision_headr   s     rX   rJ   zJanusVQVAEHead.__init__Z  s^    		&">">@U@UV#F$5$5699V%:%:F<Q<QRrY   r   r   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r  r   r  r  s     rX   r   zJanusVQVAEHead.forward`  s6    m4**=9((7rY   )ra   rb   rc   rd   rj   rJ   r   r   tensorr   rg   rh   s   @rX   r  r  W  s0    YS/ SU\\ ell rY   r  zl
    The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model.
    c                       e Zd Zdef fdZd Zd Zd Zdej                  dej                  dej                  fd	Zee	 	 	 	 	 	 	 	 	 ddej                  d
ej                  deej                     deej                     dee   deej                     deej                     dee   deeej                  f   fd              Z xZS )
JanusModelr   c                    t         |   |       || _        t        j	                  |j
                        | _        t        | j                  j                        | _        t        j	                  |j                        | _        t        j                  | j                  j                  j                  | j                  j                  j                        | _        t#        | j                  j                        | _        t'        | j                  j                        | _        t+        j,                  |j.                        | _        d| _        | j5                          y )N)r   F)rI   rJ   r   r  _from_configr=   vision_modelr  alignerrs  r   vqmodelr   	Embeddingrr   rq   generation_embeddingsr  generation_alignerr  generation_headr(   from_configr   language_modelrv  rw  r   s     rX   rJ   zJanusModel.__init__m  s     ,99&:N:NO,T->->-E-EF!..v/?/?@ &(\\$,,2E2E2T2TVZVbVbViViVsVs%t""6t||7J7J"K-dll.A.AB'336;M;MN&+#rY   c                 6    | j                   j                         S r   )r  get_input_embeddingsrU   s    rX   r  zJanusModel.get_input_embeddings  s    ""7799rY   c                 :    | j                   j                  |       y r   )r  set_input_embeddingsrU   values     rX   r  zJanusModel.set_input_embeddings  s    007rY   c                 ^    | j                  |      }| j                  |j                        }|S r   )r  r  rj  )rU   r   image_embedss      rX   get_image_featureszJanusModel.get_image_features  s,    ((6||L$B$BCrY   	input_idsinputs_embedsimage_featuresc                 P   |m| | j                         t        j                  | j                  j                  t        j
                  |j                              k(  }|j                  d      }n|| j                  j                  k(  }|j                         }|j                  d      j                  |      j                  |j                        }||   j                         |j                         k7  r0|j                  d   |j                  d   z  }t        d| d|       |S )z
        Obtains multimodal placeholdr mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        r   devicer   r   ro   z6Image features and image tokens do not match: tokens: z, features )r  r   r  r   r   longr  allsum	unsqueeze	expand_asr   numelr   r   )rU   r  r  r  special_image_maskn_image_tokensn_image_featuress          rX   get_placeholder_maskzJanusModel.get_placeholder_mask  s    !.2M$2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*dkk.H.H!H+//1/99"=GGVYYZgZnZno+,2248L8L8NN-33A69M9Ma9PPHHXXcdtcuv  "!rY   r   r   r   r   cache_position	use_cachelogits_to_keepc
                    |d u |d uz  rt        d      | | j                         |      }||| j                  |      }|j                  d|j                  d         }|j                  |j                  |j                        }| j                  |||      }|j                  ||      } | j                  d|||||||	d|
}t        |j                  |j                  |j                  |j                  |      S d       S )NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either oner   )r  r  )r  r   r   r   r  r  r  )rj  r   r   
attentionsimage_hidden_statesrH   )r   r  r  r   r   r   r  r   r  masked_scatterr  r   rj  r   r   r  )rU   r  r   r   r   r   r  r  r  r  rV   r  r  image_attention_mask	lm_outputs                  rX   r   zJanusModel.forward  sH    -t";<s   7D557	BM#22<@L)11"m6I6I"6MNN+..}/C/C]EXEXYN#'#<#<~ $= $  *889M~^M'D'' 	
')%+))	
 	
	 ,'99%55#11 ++0<0H
 	

 OS
 	
rY   )	NNNNNNNNr   )ra   rb   rc   r   rJ   r  r  r  r   r0  r   r  r#   r"   r   r   r   r   r   r   r   rg   rh   s   @rX   r  r  g  s*   { *:8
"))":?:K:K"]b]n]n"0  '+*.1537+/5959$(34.
##.
 ''.
 !.	.

 u//0.
 "%.
 !!1!12.
   1 12.
 D>.
 c5<</0.
  .
rY   r  c                   b    e Zd ZddgZdZdef fdZd Zd Zde	j                  d	e	j                  fd
Zd Zd Zee	 	 	 	 	 	 	 	 	 	 dde	j                   de	j"                  dee	j                     dee	j                      dee   dee	j                      dee	j"                     dee	j                      dee   deee	j                  f   dee   fd              Z	 	 	 	 	 	 d fd	Zde	j                  fdZe	j8                  	 	 	 d de	j                  dee	j                      dee   f fd       Z xZS )!JanusForConditionalGenerationz(model.language_model.embed_tokens.weightzlm_head.weightTr   c                     t         |   |       || _        t        |      | _        t        j                  |j                  j                  |j                  j                  d      | _
        | j                          y )NFr   )rI   rJ   r   r  r   r   r   r   r?   
vocab_sizelm_headrw  r   s     rX   rJ   z&JanusForConditionalGeneration.__init__  s\     '
yy!3!3!?!?ASASA^A^ejk 	rY   c                 J    | j                   j                  j                         S r   )r   r  r  r  s    rX   r  z2JanusForConditionalGeneration.get_input_embeddings  s    zz((==??rY   c                 N    | j                   j                  j                  |       y r   )r   r  r  r  s     rX   r  z2JanusForConditionalGeneration.set_input_embeddings  s    

!!66u=rY   inputsr   c                 r    | j                   j                  |      }| j                   j                  |      }|S r   )r   r  r  )rU   r  ri  s      rX   'prepare_embeddings_for_image_generationzEJanusForConditionalGeneration.prepare_embeddings_for_image_generation  s0    zz77?zz44\BrY   c                     || _         y r   r   )rU   ru  s     rX   set_decoderz)JanusForConditionalGeneration.set_decoder  s	    
rY   c                     | j                   S r   r  r  s    rX   get_decoderz)JanusForConditionalGeneration.get_decoder  s    zzrY   r  r   r   r   r   r  r  labelsr  r  rV   c                     | j                   d|||||||	|d|}|j                  }t        |
t              rt	        |
 d      n|
}| j                  |dd|ddf         }d}|4 | j                  d||| j                  j                  j                  d|}t        |||j                  |j                  |j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        )r  r   r   r   r   r  r  r  N)logitsr  r  )lossr  r   r   r  r  rH   )r   rj  r   r   slicer  loss_functionr   r   r  r   r   r   r  r  )rU   r  r   r   r   r   r  r  r  r  r  rV   outputsr   slice_indicesr  r  s                    rX   r   z%JanusForConditionalGeneration.forward  s    , $** 

%)%+')

 

  118B>SV8W~ot4]kmA}a,?@A%4%% f9P9P9[9[_eD +#33!//)) ' ; ;
 	
rY   c           	      N    t        
|   |f|||||d|}	|d   dk(  r||	d<   |	S )N)r   r  r   r  r  r   r   )rI   prepare_inputs_for_generation)rU   r  r   r   r   r  r  r  rV   model_inputsrW   s             rX   r  z;JanusForConditionalGeneration.prepare_inputs_for_generation+  sT     w<
+')))
 
 !!+7L(rY   r$  c                 x    | j                   j                  j                  |      }|j                  dddd      }|S )a,  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.
        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
        r   r'   r
   ro   )r   r  r|  r+  )rU   r$  decoded_images      rX   decode_image_tokensz1JanusForConditionalGeneration.decode_image_tokensI  s:     

**11,?%--aAq9rY   logits_processorc           	         |j                  d| j                        }t        j                  |      }|j                  dd      }|dk(  rt	        %|   d|||d d|S  |j                  di |}|j                         t        j                  t        j                  fvrt        d      |j                          | j                  |j                                ||n	t               }d|d<   |j                  t         j#                  d       d	|_        |j                  |d
<   | j%                  ||j&                  |      \  }}	}|j(                  |j*                  }}
t-        |j.                        dk7  rt        d|j.                   d      |d u}| j1                  |||j*                         |j                  r:|j                  dkD  r+|j3                  t5        |j                               d |_        | j7                  ||j.                  d   |d ||      } | j8                  d|||j:                  d|\  }}| j<                  j>                  j@                  jB                  }|j.                  \  }}|jE                  dd      }|j                  dd       }|jE                  dd      }||d<   ||d d d f   |j&                  k7  ||d d d f   |jF                  d   k7  z  }||d d d f   jI                  ||jJ                          | jM                         |      }| jO                  |||      }|jQ                  dd       A| jS                  |jT                  xs d|dz  tW        |jX                  ||z         ||      |d<   t[        j\                  ||f|
|      }|j^                  }|j`                  }|jb                  }|jd                  }|jf                  }|r|rdnd }|r|rdnd }|r|rdnd }|r|rdnd }ti        |      D ]x  } | jj                  d||d|}|d   jm                  |j*                        |d<   |d   jm                  |j*                        |d<    | j<                  jn                  di |||d}| jq                  ||      }|jr                  d d dd d f   ju                         } | j<                  jw                  |       }! |||!      }"|jx                  r>t[        jz                  |"d      }#t[        j|                  |#d      j                  d      }$nt[        j                  |"d      }$|$|d d |f<   t[        j                  |$|$g      }$|$j                  d      }$| j                  |$      }{ |r@|r|!fz  }|r| j                         fz  }|r|j                  z  }|r|j                  z  }|rt        |!|||j                        S |S ) Ngeneration_configgeneration_modetext)r  r   r  guidance_scalezGot incompatible mode for Image Generation, should be one of greedy or sampling. Ensure that beam search is de-activated by setting `num_beams=1` and `num_beam_groups=1`.Tr  zU`guidance_scale` is required for CFG but not provided. Setting to default value of 5.   r  r'   z;Expected input ids of shape (batch_size, seq_len), but got z3Passing `inputs embeds` is not supported currently.)r  ro   )r  input_ids_seq_lengthencoder_input_idsprefix_allowed_tokens_fnr  r  )r  r   expand_sizer   boi_token_idr   static)cache_implementationr   max_cache_lenr  model_kwargsr  rH   )r  r  r  )output_attentionsoutput_hidden_statesr   )r'  )num_samples)	sequencesscoresr  r  r   r   )Ipopr  copydeepcopyrI   generateupdateget_generation_moder   SAMPLEGREEDY_SEARCHr   validate_validate_model_kwargsr   r  r   warning_prepare_model_inputsbos_token_idr   r  rT  r   _prepare_special_tokensrZ  r   _get_logits_processor_expand_inputs_for_generationnum_return_sequencesr   r  r   rT   repeatgeneration_kwargsmasked_fill_pad_token_idr  _get_initial_cache_positionr   
_get_cacher  max
max_lengthr   zerosr  r  output_scoresoutput_logitsreturn_dict_in_generater  r  r   r  #_update_model_kwargs_for_generationrj  cloner  	do_samplesoftmaxmultinomialsqueezeargmaxcatr  r  r   r  r   r   r   )&rU   r  r   r  rV   r  r  r  r  model_input_namer   r  kwargs_has_attention_maskrT   r   r   input_tokensmaskr  generated_tokensr  r  r	  r
  r  
raw_scores
raw_logitsdecoder_hidden_statesdecoder_attentionsir  r  ri  r  next_token_scoresprobs
next_tokenrW   s&                                        rX   r  z&JanusForConditionalGeneration.generateU  s    #JJ':D<R<RS MM*;< !**%6?f$7# -"3#	
   0(//9&9 002>;P;PR`RnRn:ool  	""$##L$5$5$78 0@/K+QdQf %)[!++3NNrs/0,):)I)I%& 594N4N%22L5
1	#\ ")9)9vy1$MiooM^EF  %3$$>!$$%68QZcZjZj$k ++0A0P0PST0T##$IJ[JjJj$kl/3,  55/!*!3'%)- 6 
 #E$"D"D #
))>>#
 	#
	<  ::2299JJ'oo
G ''1-%))*:DA'..q!4)7%& Z[!^,0A0N0NNa(,=,O,OP^,__
 	Z[!^$11$8I8V8VW3113LA77V-t4<.2oo%6%K%K%Wx%>!"3">">@PSZ@Z[) /> /L*+ !;;
4D'EU[ab .??0EE)77)77"3"K"K3RD
3RD
'>CW^b$;@QRX\'( #	UA=4== +|GSL .::J-K-N-N}OcOc-dL)*-9:J-K-N-N}OcOc-dL)*/djj// "3%9G  CCG\ZL"44QAX>DDFL ZZ//=F 0F C !**&7R@"..u!DLLRP
"\\*;D
%/QT" J
#;<J#--b1J HHTMG#	UJ #vi'
|11355
 "g&8&88"#%)>)>>%",*!-3 ' 7 7  $#rY   )
NNNNNNNNNr   )NNNNNN)NNN) ra   rb   rc   _tied_weights_keysr   r   rJ   r  r  r   r   r  r  r  r#   r"   r0  r   r   r   r   r   r   r    r!   r   r  r  no_gradr   r  rg   rh   s   @rX   r  r    s   DFVW!{ @>ell u|| 
  '+*.1537+/5959-1$(341
##1
 ''1
 !.	1

 u//01
 "%1
 !!1!121
   1 121
 ))*1
 D>1
 c5<</01
 +,1
  1
l <
 
 ]]  $59:>	}$}$ !!1!12}$ ##67	}$ }$rY   r  c                       e Zd ZdZdddej
                  ddddddf
dedeee	e
f      de
d	ed
edee
ef   dedeeeee   f      deeeee   f      dee   f fdZ	 	 	 ddej                   dee
ee
e
e
f   f   deee	ef      deee	ef      dej&                  f
dZdej
                  ddfdej                   deee	e
f   e
f   deee
e
e
f      d	edeee	ef      deee	ef      dej                   fdZ	 	 	 	 	 	 	 dded
ee   dee   dee   deee      deee      dee	   dee	   fdZ	 ddej&                  deeee   f   deeee   f   deee	ef      dej&                  f
dZ xZS )JanusImageProcessora
  
    Constructs a JANUS image processor.

    Args:
        do_resize (`bool`, *optional*, defaults to `True`):
            Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the
            `do_resize` parameter in the `preprocess` method.
        size (`dict`, *optional*, defaults to `{"height": 384, "width": 384}`):
            Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess`
            method.
        min_size (`int`, *optional*, defaults to 14):
            The minimum allowed size for the resized image. Ensures that neither the height nor width
            falls below this value after resizing.
        resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
            Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be
            overridden by the `resample` parameter in the `preprocess` method.
        do_rescale (`bool`, *optional*, defaults to `True`):
            Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the
            `do_rescale` parameter in the `preprocess` method.
        rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
            Scale factor to use if rescaling the image. Only has an effect if `do_rescale` is set to `True`. Can be
            overridden by the `rescale_factor` parameter in the `preprocess` method.
        do_normalize (`bool`, *optional*, defaults to `True`):
            Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess`
            method. Can be overridden by the `do_normalize` parameter in the `preprocess` method.
        image_mean (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`):
            Mean to use if normalizing the image. This is a float or list of floats the length of the number of
            channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be
            overridden by the `image_mean` parameter in the `preprocess` method.
        image_std (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`):
            Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
            number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.
            Can be overridden by the `image_std` parameter in the `preprocess` method.
        do_convert_rgb (`bool`, *optional*, defaults to `True`):
            Whether to convert the image to RGB.
    TN   gp?	do_resizer   min_sizeresample
do_rescalerescale_factordo_normalize
image_mean	image_stddo_convert_rgbc           	          t        |   di | || _        |d| _        y t	        |D cg c]  }t        |dz         c}      | _        y c c}w )N)   r0  r0     rH   )rI   rJ   r'  background_colorrW  r   )rU   r&  r   r'  r(  r)  r*  r+  r,  r-  r.  rV   xrW   s                rX   rJ   zJanusImageProcessor.__init__<  sM     	"6" $3D!$)*LA3q3w<*L$MD!*Ls   Aimager2  data_formatinput_data_formatr   c                 N   t        ||      \  }}|t        j                  k(  r|j                  d   n|j                  d   }||k(  r|t	        |||      }|S |}|S t        ||      }t        |t              r|g}nt        |      |k7  rt        d| d      |t        j                  k(  r~t        j                  |||f|j                        }	t        |      D ]  \  }
}||	|
ddddf<    ||kD  r||z
  dz  }||	dd|||z   ddf<   |	S ||z
  dz  }||	dddd|||z   f<   |	S t        j                  |||f|j                        }	t        |      D ]  \  }
}||	dddd|
f<    ||kD  r||z
  dz  }||	|||z   ddddf<   |	S ||z
  dz  }||	dd|||z   ddf<   |	S )a}  
        Pads an image to a square based on the longest edge.

        Args:
            image (`np.ndarray`):
                The image to pad.
            background_color (`int` or `tuple[int, int, int]`, *optional*, defaults to 0):
                The color to use for the padding. Can be an integer for single channel or a
                tuple of integers representing for multi-channel images. If passed as integer
                in mutli-channel mode, it will default to `0` in subsequent channels.
            data_format (`str` or `ChannelDimension`, *optional*):
                The channel dimension format for the output image. Can be one of:
                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                If unset, will use same as the input image.
            input_data_format (`str` or `ChannelDimension`, *optional*):
                The channel dimension format for the input image. Can be one of:
                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.

        Returns:
            `np.ndarray`: The padded image.
        r   r   Nz(background_color must have no more than z) elements to match the number of channelsr   r'   )r   r   FIRSTr   r   r  r   r   rT  r   npr  r   	enumerate)rU   r4  r2  r5  r6  r   r   rB   max_dimresultr  colorstarts                rX   pad_to_squarez!JanusImageProcessor.pad_to_squareR  s+   < 'u.?@):>N>T>T)Tu{{1~Z_ZeZefhZiU? * ,E;@QR 
 L  
 Lfe$ &, 01!"l2:<.Hqr   0 6 66XX|Wg>ekkRF%&67 (5"'q!Qw(v~ 6)a/7<q%%&.0!34  !5Q.6;q!UUU]223  XXw>ekkRF%&67 (5"'q!Qw(v~ 6)a/7<uuv~-q!34
  !5Q.6;q%%%-/23rY   c                    ||n| j                   }|t        |      }t        ||      \  }}	t        ||	      }
t	        |d      }|d   |d   k7  rt        d|d    d|d          |d   }||
z  }t        t        ||z        | j                        t        t        |	|z        | j                        g}t        |f||||d|}| j                  |||      }|S )	a  
        Resize an image to dynamically calculated size.

        Args:
            image (`np.ndarray`):
                Image to resize.
            size (`dict[str, int]` or `int`):
                The size to resize the image to. If a dictionary, it should have the keys `"height"` and `"width"`.
            background_color (`tuple[int, int, int]`):
                The background color to use for the padding.
            resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`):
                `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`.
            data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the output image. If unset, the channel dimension format of the input
                image is used. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `None`: will be inferred from input
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the input image. If unset, the channel dimension format is inferred
                from the input image. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.

        Returns:
            `np.ndarray`: The resized image.
        T)default_to_squarer   r   z5Output height and width must be the same. Got height=z and width=)r   r(  r5  r6  )r4  r2  r6  )
r2  r   r   r  r   r   r   r'  r   r?  )rU   r4  r   r2  r(  r5  r6  rV   r   r   max_sizedeltaoutput_size_nonpaddeds                rX   r   zJanusImageProcessor.resize  s-   L 0@/K+QUQfQf$ >u E&u.?@vu%TT:>T']*GXGWWbcghocpbqr  H~x FUN#T]]3EEM"DMM2!

 
&#/
 
 ""-/ # 

 rY   imagesreturn_tensorsc	                    ||n| j                   }|d| j                  z  n|}||n| j                  }||n| j                  }||n| j                  }t        |      }t        |d   t        j                  j                        rt        |      dkD  r|S |d   S |t        |d         }g }	|D ]  }
t        |
      }
|r| j                  |
|||      }
|rC| j                  |
||      }
|
j                  dd      j                  t         j"                        }
|rB|r@|dk(  r;t%        |
t&        j(                  |	      }
t        j                  j+                  |
      }
|	j-                  |
        d
|	i}|dk7  r|nd}t/        ||      S )znApplies post-processing to the decoded image tokens by reversing transformations applied during preprocessing.Ng      ?r   ro   )r4  r,  r-  r6  )r   r6  r1  zPIL.Image.Image)input_channel_dimr   )datatensor_type)r)  r*  r+  r,  r-  r   r   PILImagerT  r   r   unnormalizerescaleclipastyper9  uint8r   r   LAST	fromarrayrZ  r   )rU   rE  r)  r*  r+  r,  r-  r6  rF  r   r4  rI  s               rX   postprocesszJanusImageProcessor.postprocess  s    $.#9Zt
6D6Lt222R`'3'?|TEVEV#-#9Zt
!*!6IDNN	$V,fQi1 [1_6;&);$ >vay I 	'E"5)E((J)_p )  U.Tef

1c*11"((;
~AR/R3E;K;P;Pduv		++E2&!	'$ -+9=N+NTX>BBrY   c                    d}t        |t              r(t        |      |k7  r t        d| dt        |             |g|z  }t        |t              r(t        |      |k7  r t        d| dt        |             |g|z  }t	        d t        ||      D              }t	        d |D              }| j                  ||||      }|S )a~  
        Unnormalizes `image` using the mean and standard deviation specified by `mean` and `std`.
        image = (image * image_std) + image_mean
        Args:
            image (`torch.Tensor` of shape `(batch_size, num_channels, image_size, image_size)` or `(num_channels, image_size, image_size)`):
                Batch of pixel values to postprocess.
            image_mean (`float` or `Iterable[float]`):
                The mean to use for unnormalization.
            image_std (`float` or `Iterable[float]`):
                The standard deviation to use for unnormalization.
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the input image. If unset, the channel dimension format is inferred
                from the input image. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
        r
   zmean must have z$ elements if it is an iterable, got zstd must have c              3   .   K   | ]  \  }}| |z    y wr   rH   ).0meanstds      rX   	<genexpr>z2JanusImageProcessor.unnormalize.<locals>.<genexpr>@  s     WytSus{Ws   c              3   &   K   | ]	  }d |z    yw)ro   NrH   )rW  rY  s     rX   rZ  z2JanusImageProcessor.unnormalize.<locals>.<genexpr>A  s     ;#a#g;s   )r4  rX  rY  r6  )r   r   rT  r   rW  zipr*  )rU   r4  r,  r-  r6  rB   rev_image_meanrev_image_stds           rX   rM  zJanusImageProcessor.unnormalize  s    0 j(+:,. ?<.@dehisetdu!vww$4Ji*9~- >,?cdghqdrcs!tuu"l2IWC
I<VWW;;;n-Sd  
 rY   )r   NN)NNNNNNNr   )ra   rb   rc   rd   r   BICUBICr   r   r   strr   r   r   r   rJ   r9  ndarrayrW  r   arrayr?  r   r   rT  r   rM  rg   rh   s   @rX   r$  r$    s    #N )-'9'A'A,3!:>9=)-NN tCH~&N 	N
 %N N c5j)N N U5$u+#567N E%e"456N !N2 >?>BDHHzzH  U3S=%9 9:H eC)9$9:;	H
 $E#/?*?$@AH 
H\ <@'9'A'A>BDHIzzI DcNC'(I #5c3#78	I
 %I eC)9$9:;I $E#/?*?$@AI 
I\ &**.'+,0+/+/(,1C1C TN1C !	1C
 tn1C T%[)1C DK(1C $C=1C !1Cp EI+xx+ %%01+ /0	+
 $E#/?*?$@A+ 
+rY   r$  )	r$  r   r  r  rs  r  rj   r;   r   )xr  collections.abcr   dataclassesr   typingr   r   r   numpyr9  r   r   .transformers.models.blip.image_processing_blipr	   activationsr   cache_utilsr   
generationr   r   r   r   generation.utilsr   image_processing_utilsr   r   image_transformsr   r   image_utilsr   r   r   r   r   r   r   modeling_outputsr   modeling_utilsr   r   processing_utilsr    utilsr!   r"   r#   r$   r%   r&   autor(   blip_2.modeling_blip_2r)   !chameleon.configuration_chameleonr*   chameleon.modeling_chameleonr+   r,   r-   r.   r/   idefics.modeling_ideficsr0   r1   llama.modeling_llamar2   siglip.configuration_siglipr3   siglip.modeling_siglipr4   r5   r6   torch.nntorch.nn.functional
functionalr)  torch.utils.checkpointrK  configuration_utilsr7   r8   r9   
get_loggerra   r   r;   rj   r   r   r   r   r   r   r[  r   r   r   r  r  r  r!  r2  r4  r6  r8  rF  rO  rl  rs  r  r  r  r  r$  __all__rH   rY   rX   <module>r     s     $ ! , ,    M !   u u 9 A C   , F &   5 D  e : < ^ ^ ##! 3 - 
		H	%
^1* ^1BW+ Wtk#" k#\ 
.? 
. 
. 
	-{ 	- 	-	#A 		"? 	2 "I$299 I$XRYY (*0 *p p2' 2BII $" = "*	< 		8 		B 	RYY  ,J!		 J!ZA		 AH-F -F`299 $RYY   
i
% i

i
X{$$8/ {$|	o, od	
rY   