
    rhN                        d Z ddlmZ ddlmZmZmZ ddlmZ ddl	m
Z
mZ ddlmZmZmZmZmZ ddlmZmZ dd	lmZmZmZ dd
lmZ  e       r
ddlmZmZmZ  e       r
ddlmZmZmZ erddlmZ  ej@                  e!      Z" e       rddl#m#Z# ndZ#dZ$d Z%d Z&d Z' G d ded      Z( G d ded      Z) G d de      Z*dgZ+y)z
Processor class for SmolVLM.
    )	timedelta)TYPE_CHECKINGOptionalUnion   )BatchFeature)
ImageInputmake_nested_list_of_images)AllKwargsForChatTemplateImagesKwargsProcessingKwargsProcessorMixinUnpack)BatchEncoding	TextInput)is_num2words_availableis_vision_availablelogging)
VideoInput   )DEFAULT_MEDIA_OUTTRODEFAULT_VIDEO_INTROFRAME_TIMESTAMP_MESSAGE)PreTokenizedInput)	num2wordsNa  <|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% elif line['type'] == 'video' %}{{ '<video>' }}{% endif %}{% endfor %}<end_of_utterance>
{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}c           	          d}t        |      D ]4  }t        |      D ]  }|| d|dz    d|dz    dz   | | z  z   z  }! |dz  }6 |d| | z   | | z  z   | z   z  }|S )zKPrompt with expanded image tokens for when the image is split into patches. z<row_r   _col_>
)range)	image_seq_len
image_rows
image_colsfake_token_around_imageimage_tokenglobal_image_tokentext_split_imagesn_hn_ws	            /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/smolvlm/processing_smolvlm.py_prompt_split_imager,   <   s     Z  "$ 	C*+sQwiuS1WIQ/OOU`TaerRrr	 	T!" 
$%& 	"M]
*	+ %%	'     c                 &    | | z   | | z  z   | z   S )z5Prompt with expanded image tokens for a single image. )r"   r%   r&   r'   s       r+   _prompt_single_imager0   Q   s6     #
# 	"M]
*	+ %%	'r-   c                 T    | dk(  r|dk(  rt        ||||      S t        || ||||      S )Nr   )r%   r&   r'   )r0   r,   )r#   r$   r"   r%   r&   r'   s         r+   get_image_prompt_stringr2   [   sH     Q:?#$;#1	
 	
 z:/FUg r-   c                   8    e Zd ZU ee   ed<   eeeef      ed<   y)SmolVLMImagesKwargsreturn_row_col_infomax_image_sizeN)	__name__
__module____qualname__r   bool__annotations__dictstrintr/   r-   r+   r4   r4   j   s    !$'T#s(^,,r-   r4   F)totalc                   .    e Zd ZU eed<   ddddddidZy)SmolVLMProcessorKwargsimages_kwargsTF)add_special_tokenspaddingis_split_into_wordsr5   )text_kwargsrB   N)r7   r8   r9   r4   r;   	_defaultsr/   r-   r+   rA   rA   o   s+    && #'#(
 "4
	Ir-   rA   c                   4    e Zd ZdZg dZdZdZdZ	 	 ddede	e
   f fdZd	 Zd
 Z	 	 	 	 ddeeee   eee      f   deedee   ed   f   dedee   def
dZd Zd Zed        Z	 ddeeee
e
f      eeee
e
f         f   de	e
   dee   de
f fdZ xZS )SmolVLMProcessora}  
    Constructs a SmolVLM processor which wraps a LLama tokenizer and SmolVLM image processor into a single processor.

    [`SmolVLMProcessor`] offers all the functionalities of [`SmolVLMImageProcessor`] and [`SmolVLMTokenizerFast`]. See
    the docstring of [`~IdeficsProcessor.__call__`] and [`~IdeficsProcessor.decode`] for more information.

    Args:
        image_processor (`SmolVLMImageProcessor`):
            An instance of [`SmolVLMImageProcessor`]. The image processor is a required input.
        tokenizer (`PreTrainedTokenizerBase`):
            An instance of [`PreTrainedTokenizerBase`]. This should correspond with the model's text model. The tokenizer is a required input.
        video_processor (`SmolVLMImageProcessor`):
            n instance of [`SmolVLMImageProcessor`]. The video processor is a required input.
        image_seq_len (`int`, *optional*, defaults to 169):
            The length of the image sequence i.e. the number of <image> tokens per image in the input.
            This parameter is used to build the string from the input prompt and image tokens and should match the
            value the model used. It is computed as: image_seq_len = int(((image_size // patch_size) ** 2) / (scale_factor**2))
        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
            in a chat into a tokenizable string.
    )image_processor	tokenizervideo_processorSmolVLMImageProcessorSmolVLMVideoProcessorAutoTokenizerr"   chat_templatec                 R   t        |dd      | _        t        |dd      | _        |j                  | j                        | _        t        |dd      | _        t        |dd      | _        || _        t        |d	d
      | _        t        st        d      t        | 0  |||fd|i| y )Nfake_image_tokenz<fake_token_around_image>r&   z<image>end_of_utterance_tokenz<end_of_utterance>r'   z<global-img>video_tokenz<video>zbPackage `num2words` is required to run SmolVLM processor. Install it with `pip install num2words`.rP   )getattrrR   r&   convert_tokens_to_idsimage_token_idrS   r'   r"   rT   r   ImportErrorsuper__init__)selfrJ   rK   rL   r"   rP   kwargs	__class__s          r+   rZ   zSmolVLMProcessor.__init__   s     !(	3EGb c"9mYG'==d>N>NO&-i9QSg&h#"))5I>"Z*"9mYGt  	)_lTaleklr-   c           
      N   |(|D cg c]  }|j                  | j                         }}|D cg c]  }t        |       }} | j                  |fi |d   }|d |fS |k7  rt	        d| d| d      |j                  ddgt        |      z  g      }	|j                  ddgt        |      z  g      }
g }t        ||	|
      D ]  \  }}}g }t        ||      D ]O  \  }}t        ||| j                  | j                  | j                  | j                        }|j                  |       Q |j                  | j                        }t        |      dk(  rt	        d	      |d   }t        |      D ]  \  }}||||d
z      z   z  } |j                  |        ||fS c c}w c c}w )NrB   z!The number of images in the text z and images  should be the same.rowsr   colsr&   r%   r'   z.The image token should be present in the text.r   )countr&   lenrJ   
ValueErrorpopzipr2   r"   rR   r'   appendsplit	enumerate)r[   textimagesoutput_kwargssamplen_images_in_textsublistn_images_in_imagesimage_inputsr#   r$   prompt_stringssample_rowssample_colsimage_prompt_stringsn_rowsn_colsimage_prompt_stringsplit_sampleis                       r+   process_visionzSmolVLMProcessor.process_vision   s   MQR6T-=-= >RR:@Awc'lAA+t++FUmO6TU<%%!1134D3E\RdQeeyz  "%%fsSY.?@
!%%fsSY.?@
03D*j0Q 	*,FK#% "%k;"? 	A&=&& $ 0 0,0,A,A'+'>'>'# %++,?@	A "<<(8(89L< A% !QRR "!_F*34H*I D&&-QU0CCCD!!&)-	*0 |++Q  SAs
   "FF"c                    |(|D cg c]  }|j                  | j                         }}|D cg c]  }t        |       }} | j                  |fi |d   }|d   j                  d   }	t        |j                  d            }
t        |j                  d            }|d |fS |k7  rt        d| d| d      g }|D ]  }| j                  |v rt        |
      }t        |      }t        t        |      	      }t        j                  t        |	      t        |      
      }|D ]f  }t        | j                   | j"                  | j$                  | j&                        }|d   dd|d   d}t)        j                  |      |z   }||z  }h |t*        z  }|j-                  | j                  |d      }| j                  |v r|j/                  |        ||fS c c}w c c}w )Nvideos_kwargspixel_valuesr   
timestamps	durationsz!The number of videos in the text z and videos r_   )seconds)frame_countvideo_durationrb   r   02d:)	timestamp)rc   rT   rd   rL   shapeiterrf   re   nextr   r>   r   formatr   r=   r0   r"   r&   rR   r'   r   r   replacerh   )r[   rk   videosrm   rn   n_videos_in_textrp   n_videos_in_videosvideo_inputs
num_framesbatch_timestampsbatch_durationsrs   r   durationduration_tdrv   r   ry   s                      r+   process_videozSmolVLMProcessor.process_video   s(   MQR6T-=-= >RR:@Awc'lAA+t++FUmO6TU!.177:
 0 0 >?|//<=<%%!1134D3E\RdQeeyz   	*F""f,!"23
0'H>':'A'A )* 5c+FV($ ", 	@I*>**$($4$4040E0E+/+B+B	+' $-Q<"4Ail35G HI*A*H*HS\*]`s*s'(,??(	@ %(<<$(8(8:NPQR' ""f,( !!&)+	*, |++Q  SAs
   "GG#rl   rk   r   r   r\   returnc                 P   |||t        d      ||du |duz  rt        d       | j                  t        fd| j                  j                  i|}|t        |t              r|g}n.t        |t              st        |d   t              st        d      t        |D cg c]  }|j                  | j                         c}      }|dkD  r||t        d| d      i }	|3t        |      }| j                  |||      \  }}
|	j                  |
       n)|'| j                  |||      \  }}
|	j                  |
       |d	   j                  d
d      }|< | j                  |fi |d	   }| j!                  ||dg       |	j                  |       t#        |	|      S c c}w )a,  
        Processes the input prompts and returns a BatchEncoding.

        Example:

        ```python
        >>> import requests
        >>> from transformers import SmolVLMProcessor
        >>> from transformers.image_utils import load_image

        >>> processor = SmolVLMProcessor.from_pretrained("HuggingFaceM4/SmolVLM2-256M-Video-Instruct")
        >>> processor.image_processor.do_image_splitting = False  # Force as False to simplify the example

        >>> url1 = "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
        >>> url2 = "https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg"

        >>> image1, image2 = load_image(url1), load_image(url2)
        >>> images = [[image1], [image2]]

        >>> text = [
        ...     "<image>In this image, we see",
        ...     "bla bla bla<image>",
        ... ]
        >>> outputs = processor(images=images, text=text, return_tensors="pt", padding=True)
        >>> input_ids = outputs.input_ids
        >>> input_tokens = processor.tokenizer.batch_decode(input_ids)
        >>> print(input_tokens)
        ['<|begin_of_text|><fake_token_around_image><global-img>((<image>)*169)<fake_token_around_image> In this image, we see', '<|reserved_special_token_0|><|reserved_special_token_0|><|reserved_special_token_0|><|begin_of_text|>bla bla bla<fake_token_around_image><global-img>((<image>)*169)<fake_token_around_image>']
        ```

        Args:
            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`, *optional*):
                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
                tensor. If is of type `list[ImageInput]`, it's assumed that this is for a single prompt i.e. of batch size 1.
            text (`Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]]`, *optional*):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
                Wherever an image token, `<image>` is encountered it is expanded to
                `<fake_token_around_image>` + `<row_x_col_y>` + `<image>` * `image_seq_len` * <fake_token_around_image>`.
            videos (`list[PIL.Image.Image]`, `np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`, *optional*):
                The video or batch of videos to be prepared. Each video can be a list of PIL frames, NumPy array or PyTorch
                tensor. If is of type `list[VideoInput]`, it's assumed that this is for a single prompt i.e. of batch size 1.
            return_tensors (`Union[str, TensorType]`, *optional*):
                If set, will return tensors of a particular framework. See [`PreTrainedTokenizerFast.__call__`] for more
                information.
        Nz5You must provide one of `text`, `images` or `videos'.z4You must specify exactly one of `images` or `videos`tokenizer_init_kwargsr   zAInvalid input text. Please provide a string, or a list of stringszWe detected z4 tokens in the text but no images/videos were passedrF   return_tensorsimage)
modalities)tensor_type)re   _merge_kwargsrA   rK   init_kwargs
isinstancer=   listsumrc   r&   r
   r|   updater   rf   _check_special_mm_tokensr   )r[   rl   rk   audior   r\   rm   rn   ro   inputsvision_inputsr   text_inputss                r+   __call__zSmolVLMProcessor.__call__	  s   n <FNv~TUU<fnt1CDSTT***"
"&.."<"<
 
 $$vd+JtAw4L !dee"QU#VvFLL1A1A$B#VW!#FN <0@/AAu!vww/7F"&"5"5#D-
 MM-("&"4"4#D-
 MM-(&}599:JDQ($..N}1MNK))$	)RMM+&F??; $Ws   ""F#c                 >     | j                   j                  |i |}|S )z
        This method forwards all its arguments to SmolVLMTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
        refer to the docstring of this method for more information.
        )rK   batch_decode)r[   argsr\   batched_decode_outputs       r+   r   zSmolVLMProcessor.batch_decodep  s&    
 !< ; ;T LV L$$r-   c                 >     | j                   j                  |i |}|S )z
        This method forwards all its arguments to SmolVLMTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
        the docstring of this method for more information.
        )rK   decode)r[   r   r\   decode_outputs       r+   r   zSmolVLMProcessor.decodex  s%    
 .--t>v>r-   c                     | j                   j                  }| j                  j                  }t        t        j                  ||z               S N)rK   model_input_namesrJ   r   r<   fromkeys)r[   tokenizer_input_namesimage_processor_input_namess      r+   r   z"SmolVLMProcessor.model_input_names  s?     $ @ @&*&:&:&L&L#DMM"=@U"UVWWr-   conversationc                 ~   t        |t        t        f      r+t        |d   t        t        f      st        |d   d      r|}n|g}t	        d |D              }||rt
        }|j                  d| j                  j                         |j                  d| j                  j                         t        | ,  ||fi |S )a  
        Similar to the `apply_chat_template` method on tokenizers, this method applies a Jinja template to input
        conversations to turn them into a single tokenizable string.

        The input is expected to be in the following format, where each message content is a list consisting of text and
        optionally image or video inputs. One can also provide an image, video, URL or local path which will be used to form
        `pixel_values` when `return_dict=True`. If not provided, one will get only the formatted text, optionally tokenized text.

        conversation = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "url": "https://www.ilankelman.org/stopsigns/australia.jpg"},
                    {"type": "text", "text": "Please describe this image in detail."},
                ],
            },
        ]

        Args:
            conversation (`Union[list[Dict, [str, str]], list[list[dict[str, str]]]]`):
                The conversation to format.
            chat_template (`Optional[str]`, *optional*):
                The Jinja template to use for formatting the conversation. If not provided, the tokenizer's
                chat template is used.
        r   contentc              3   r   K   | ]/  }|D ](  }|d    D ]  }t        |t              xr |d   dk(     * 1 yw)r   typevideoN)r   r<   ).0r   messager   s       r+   	<genexpr>z7SmolVLMProcessor.apply_chat_template.<locals>.<genexpr>  s_      
'
 "9-	
  &E76?g+EE
E
E
s   57r   fps)r   r   tuplehasattranyDEFAULT_CHAT_TEMPLATE
setdefaultrL   r   r   rY   apply_chat_template)r[   r   rP   r\   conversations	has_videor]   s         r+   r   z$SmolVLMProcessor.apply_chat_template  s    > lT5M2|Au6',q/S\:](M)NM 
 -
 
	  Y1M,(<(<(G(GH%!5!5!9!9:w*<Q&QQr-   )   N)NNNNr   )r7   r8   r9   __doc__
attributesimage_processor_classvideo_processor_classtokenizer_classr>   r   r=   rZ   r|   r   r   r	   r   r   r   r   rA   r   r   r   r   propertyr   r<   r   r   __classcell__)r]   s   @r+   rI   rI   ~   so   * EJ33%O !'+m
 m  }m0*,X*,\ OSbf!e@j$z"2Dj9I4JJKe@ I2DOTJ]E^^_e@
 e@ /0e@ 
e@N% X X (,3RDc3h0$tDcN7K2LLM3R  }3R 12	3R
 
3R 3Rr-   rI   ),r   datetimer   typingr   r   r   feature_extraction_utilsr   image_utilsr	   r
   processing_utilsr   r   r   r   r   tokenization_utils_baser   r   utilsr   r   r   video_utilsr   video_processing_smolvlmr   r   r   r   
get_loggerr7   loggerr   r   r,   r0   r2   r4   rA   rI   __all__r/   r-   r+   <module>r      s     1 1 4 A p p ? I I %     <			H	% #I ` *-,e -
-U {R~ {R|	 
r-   