
    rh<                         d dl mZmZ d dlZddlmZ ddlmZ ddl	m
Z
mZmZmZmZmZ ddlmZmZ ddlmZ  G d	 d
ed      Z G d de
      Z G d ded      Z G d de      ZdgZy)    )OptionalUnionN   )BatchFeature)
ImageInput)ImagesKwargsMultiModalDataProcessingKwargsProcessorMixinUnpackVideosKwargs)PreTokenizedInput	TextInput)
VideoInputc                   (    e Zd ZU eee   ef   ed<   y)Glm4vVideosProcessorKwargsfpsN)__name__
__module____qualname__r   listfloat__annotations__     }/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/glm4v/processing_glm4v.pyr   r       s    	tE{E!	""r   r   F)totalc                   >    e Zd ZU ee   ed<   ee   ed<   ee   ed<   y)Glm4vImagesKwargs
patch_sizetemporal_patch_size
merge_sizeN)r   r   r   r   intr   r   r   r   r   r   $   s     !#&r   r   c                   0    e Zd ZU eed<   eed<   ddddiZy)Glm4vProcessorKwargsimages_kwargsvideos_kwargstext_kwargsF)paddingreturn_mm_token_type_idsN)r   r   r   r   r   r   	_defaultsr   r   r   r%   r%   *   s#    $$--(-
Ir   r%   c                        e Zd ZdZg dZdZdZdZd fd	Z	 	 	 dde	de
eeee   ee   f   d	ed
ee   def
dZddZd Zd Z	 ddZed        Z xZS )Glm4vProcessora  
    Constructs a GLM-4V processor which wraps a GLM-4V image processor and a GLM-4 tokenizer into a single processor.
    [`~Glm4vProcessor.__call__`] and [`~Glm4vProcessor.decode`] for more information.
    Args:
        image_processor ([`Glm4vProcessor`], *optional*):
            The image processor is a required input.
        tokenizer ([`PreTrainedTokenizerFast`], *optional*):
            The tokenizer is a required input.
        video_processor ([`Glm4vVideoProcessor`], *optional*):
            The video processor is a required input.
        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
            in a chat into a tokenizable string.
    )image_processor	tokenizervideo_processorAutoImageProcessorAutoVideoProcessor)PreTrainedTokenizerPreTrainedTokenizerFastc                    t         |   ||||       t        |d      sdn|j                  | _        t        |d      sdn|j                  | _        t        |dd       r|j                  n|j                  | j                        | _        t        |dd       r|j                  | _        y |j                  | j                        | _        y )N)chat_templateimage_tokenz	<|image|>video_tokenz	<|video|>image_token_idvideo_token_id)	super__init__hasattrr7   r8   getattrr9   convert_tokens_to_idsr:   )selfr.   r/   r0   r6   kwargs	__class__s         r   r<   zGlm4vProcessor.__init__K   s    )_Tab.5i.O;U^UjUj.5i.O;U^UjUj y"2D9 $$001A1AB 	 y"2D9 $$ 	 001A1AB 	r   imagestextvideosrA   returnc                     | j                   t        fd| j                  j                  i|}| | j                  dd|i|d   }|d   }ni }d}|. | j
                  dd|i|d   }|j                  d      }	|d	   }
ni }g }	d}
t        |t              s|g}|j                         }|| j                  j                  d
z  }d}t        t        |            D ]  }| j                  ||   v rS||   j                         |z  }||   j                  | j                  d|z  d      ||<   |dz  }| j                  ||   v rS||   j                  d| j                        ||<    |
| j
                  j                  d
z  }d}t        t        |            D ]  }| j                   ||   v r||
|   d   }d}t#        |	d      r|	j%                         d   }nt        |	d   t              r|	d   n|	}g }t        dt        |            D ]  }|j'                  ||           |d| }t        |      |k  r'|j'                  |r|d   nd       t        |      |k  r't        |      D ]  }||   }d| j                   d| }||z  }  ||   j                  | j                   |d      ||<   |
|   j                         |z  |
|   d   z  }t        |      D ]:  }| j                  ||   v s||   j                  | j                  d|z  d      ||<   < |dz  }| j                   ||   v r|||   j                  d| j                        ||<    |d   j                  dd      }|d   j                  dd      } | j                  |fi |d   }| j)                  ||ddg       |rUt+        j,                  |d         }t+        j.                  |d         }d||| j0                  k(  <   |j%                         |d<   t3        i ||||      S )a^
  
        Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
        and `kwargs` arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizerFast.__call__`] if `text` is not `None` to encode
        the text.

        Args:
            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
                tensor. Both channels-first and channels-last formats are supported.
            text (`str`, `List[str]`, `List[List[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
            videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`):
                The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch
                tensor, or a nested list of 3D frames. Both channels-first and channels-last formats are supported.
            return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors of a particular framework. Acceptable values are:
                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return NumPy `np.ndarray` objects.
                - `'jax'`: Return JAX `jnp.ndarray` objects.

        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
            - **pixel_values_videos** -- Pixel values of videos to be fed to a model. Returned when `videos` is not `None`.
            - **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`.
            - **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`.
        tokenizer_init_kwargsNrC   r&   image_grid_thwrE   r'   
timestampsvideo_grid_thw   r   z<|placeholder|>    tolistz<|begin_of_image|>z<|end_of_image|>r(   return_tensorsr*   Fimagevideo)
modalities	input_idsmm_token_type_ids)datatensor_typer   )_merge_kwargsr%   r/   init_kwargsr.   r0   pop
isinstancer   copyr"   rangelenr7   prodreplacer8   r=   rO   append_check_special_mm_tokensnparray
zeros_liker9   r   )r@   rC   rD   rE   rA   output_kwargsimage_inputsrI   videos_inputsrJ   rK   merge_lengthindexinum_image_tokensvideo_index
num_framesvideo_structuretimestamps_listunique_timestampsidxselected_timestamps	frame_idxtimestamp_secframe_structurerQ   r*   text_inputs	array_idsrV   s                                 r   __call__zGlm4vProcessor.__call__Z   s   T +** 
"&.."<"<
 

 /4//`v`A_`L)*:;NL!N0D00aa-P_B`aM&**<8J*+;<NMJ!N$%6Dyy{%//::A=LE3t9% O&&$q'1'5e'<'A'A'C|'S$"1good.>.>@QTd@dfghDGQJE &&$q'1 q'//*;T=M=MNQO %//::A=LK3t9% !O&&$q'1!/!<Q!?J&(Oz84*4*;*;*=a*@;EjQRmUY;Z*Q-`j(*%$QO(<= G)001EFG +<KZ*H'12Z?+22Na3Fr3Jghi 12Z? &+:%6 ;	(;I(F,>t?O?O>PP`an`o*p'?:;
 #1good.>.>QRSDG&{388:lJn]hNijkNll % &+:%6 q	++tAw6&*1good6F6FHY\lHlno&pDGq  1$K= &&$q'1@ q'//*;T=M=MNQC!OD '}599:JDQ#0#?#C#CD^`e#f $dnnTJ]=-IJ%%dKWgDV%W#[!9:I "k+.F GBCi4+>+>>?/@/G/G/IK+,!QK!Q<!Q=!Q_mnnr   c                    i }|t         j                  j                  di       }|j                  |       |j                  dd      xs | j                  j
                  }|D cg c]   } | j                  j                  g || " }}|D 	cg c]
  }	|	|dz  z   }
}	|j                  |
|d       |vt         j                  j                  di       }|j                  |       |D cg c]   } | j                  j                  g || " }}|D 	cg c]
  }	|	dz  z   }}	||d<   t        di |S c c}w c c}	w c c}w c c}	w )	aK  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.
        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.
            video_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (num_frames, height, width) per each video.
        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        Nr&   r"   rL   )rm   num_image_patchesr'   num_video_tokensr   )
r%   r+   getupdater.   r"   get_number_of_image_patchesr0   get_number_of_video_patchesr	   )r@   image_sizesvideo_sizesrA   vision_datar&   r"   
image_sizer|   num_patchesrm   r'   
video_sizenum_video_patchesr}   s                  r   _get_num_multimodal_tokensz)Glm4vProcessor._get_num_multimodal_tokens   s    "0::>>PRSM  (&**<>a$BVBVBaBaJ #.! A$$@@\*\m\! ! Sdd;
A!=dd4D[lmn"0::>>PRSM  ( #.! A$$@@\*\m\! ! Sdd;
A!=dd.>K*+,,,#!  e!  es   $%D7D<+%EEc                 :     | j                   j                  |i |S )z
        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
        refer to the docstring of this method for more information.
        r/   batch_decoder@   argsrA   s      r   r   zGlm4vProcessor.batch_decode   s     
 +t~~**D;F;;r   c                 :     | j                   j                  |i |S )z
        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
        the docstring of this method for more information.
        )r/   decoder   s      r   r   zGlm4vProcessor.decode  s     
 %t~~$$d5f55r   c                 B     | j                   j                  |f||d|S )a  
        Post-process the output of the model to decode the text.

        Args:
            generated_outputs (`torch.Tensor` or `np.ndarray`):
                The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
                or `(sequence_length,)`.
            skip_special_tokens (`bool`, *optional*, defaults to `True`):
                Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
            clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
                Whether or not to clean up the tokenization spaces. Argument passed to the tokenizer's `batch_decode` method.
            **kwargs:
                Additional arguments to be passed to the tokenizer's `batch_decode method`.

        Returns:
            `list[str]`: The decoded text.
        )skip_special_tokensclean_up_tokenization_spacesr   )r@   generated_outputsr   r   rA   s        r   post_process_image_text_to_textz.Glm4vProcessor.post_process_image_text_to_text  s5    ( +t~~**
 3)E
 	
 	
r   c                     | j                   j                  }| j                  j                  }t        t        j                  ||z               }|dgz   S )Nsecond_per_grid_ts)r/   model_input_namesr.   r   dictfromkeys)r@   tokenizer_input_namesimage_processor_input_namesnames_from_processors       r   r   z Glm4vProcessor.model_input_names'  sN     $ @ @&*&:&:&L&L##DMM2GJe2e$fg#';&<<<r   )NNNN)NNN)NN)TF)r   r   r   __doc__
attributesimage_processor_classvideo_processor_classtokenizer_classr<   r   r   r   r   r   r   r   r%   r   rz   r   r   r   r   propertyr   __classcell__)rB   s   @r   r-   r-   5   s     EJ00HO
" "^b!	|o|o I0$y/4HYCZZ[|o 	|o
 -.|o 
|o|$-L<6 Y^
6 = =r   r-   )typingr   r   numpyrd   feature_extraction_utilsr   image_utilsr   processing_utilsr   r	   r
   r   r   r   tokenization_utils_baser   r   video_utilsr   r   r   r%   r-   __all__r   r   r   <module>r      sa   * #  4 % t t C %#U # +5 w=^ w=t 
r   