rh dZddlZddlmZmZddlZddlmZm Z m Z ddl m Z m Z mZddlmZmZmZmZmZmZmZmZmZmZddlmZmZmZmZmZm Z e jBe"Z#er ddl$Z$ddl%m&cm'Z( dd d d ee)e*e*fd ee*fd Z+dde,e*e*ffdZ-dZ.ddZ/ dde0de0dee,e*e*ffdZ1de)e2e*fde,e*e*ffdZ3GddeZ4dgZ5y)zImage processor class for EoMT.N)OptionalUnion)BaseImageProcessor BatchFeature get_size_dict) PaddingModepadresize) ChannelDimension ImageInputPILImageResamplingget_image_sizeinfer_channel_dimension_formatmake_flat_list_of_imagesmake_list_of_imagesto_numpy_array valid_imagesvalidate_preprocess_arguments)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD TensorTypefilter_out_non_signature_kwargsis_torch_availableloggingsegmentation_mapz np.ndarrayinstance_id_to_semantic_id ignore_indexc4|tj|dk(||dz }tj|}||||k7}|Dcgc]}||k( }}|rtj|d}n#tjdg|j }|Etj|j d}|D]}|||dzn|}||dz n||||k(<n|}|j tj|j tjfScc}w)Nr)axis) npwhereuniquestackzerosshapeastypefloat32int64) rrr all_labelsi binary_maskslabelslabelclass_ids /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/eomt/image_processing_eomt.py(convert_segmentation_map_to_binary_masksr2:s7 88$4$96'?F7O ;;rEc||k(}|j}|||k\}|j}||z} | j} |dkDxr |dkDxr| dkD} | r||z } | j|kDsd} | | fS)NrF)sumitem) mask_labels mask_probskmask_thresholdoverlap_mask_area_thresholdmask_k mask_k_area original_mask original_area final_maskfinal_mask_area mask_exists area_ratios r1check_segment_validityr_s A F**,KqM^3M!%%'M-'J nn&O/Oma&7OOa>K  ""rErUrV target_sizecb||jdn|d}||jdn|d}tj||ftj|jdz } g} |j }|ddddf|zj d} d} i} t|jdD]~}||j}t| ||||\}}|s+|r||vr|| vr | || |<>| | |<| | |<t||jd}| j| ||d| dz } | | fS)Nr r)dtypedeviceidlabel_idscore) r'torchr&longrdsigmoidargmaxrangerQr_r9append)rS pred_scores pred_labels stuff_classesrUrVr`r=r> segmentationsegmentsrRcurrent_segment_idstuff_memory_listrT pred_classr]r[ segment_scores r1compute_segmentsrys%0$7Z  a [^F#.#6J  Q KNE;;ejjIZIZ[^__LH##%Jq$}- :BB1EK(* ;$$Q' (  ^((* #9 Q8S#  Z  Z=8..+|d}|jdxs|}||fS)z.Returns the height and width from a size dict. shortest_edge longest_edge)get)rz target_height target_widths r1get_target_sizers*o.M==0AML , &&rEc'eZdZdZdgZddej dddddddddf dedee e e fd ed ed e d ed ededee e ee fdee e ee fdee dee ffd Zej ddfdej"de d edee e efdej"f dZdede de deeeffdZdede dej"fdZ d2dedeedee e e fd ed eedeed eed ee d eedee e ee fdee e ee fdee e efdee e efdej"fdZ d3dedeedeedee e e fd ede e efdee e efdej"fdZeddddddddddddddej6dfded ee ee e e fe e e ffd!ee e e fd eedeedee e e fd ed eed ee d eedeedee e ee fdee e ee fdee d"ee e efde e efdee e efdef$d#Z d4d$eed ed!ee ee e e fe e e ffdee d"ee e efdee e eff d%Zd&e jBd'eee e e fd(eee e fde e e fdee jBf d)Z"d&e jBd(eee e fde e e fdee jBfd*Z# d5d(eee e fdee e e fdej"fd+Z$ d6d(eee e fd,e d-e d.e d/eee dee e e ff d0Z%e d7d(eee e fd,e dee e e ffd1Z&xZ'S)8EomtImageProcessoraY Constructs a EoMT image processor. The image processor can be used to prepare image(s) and optional targets for the model. This image processor inherits from [`BaseImageProcessor`] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods. Args: do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the input to a certain `size`. size (`int`, *optional*, defaults to 640): Resize the input to the given size. Only has an effect if `do_resize` is set to `True`. If size is a sequence like `(width, height)`, output size will be matched to this. If size is an int, smaller edge of the image will be matched to this number. i.e, if `height > width`, then image will be rescaled to `(size * height / width, size)`. resample (`int`, *optional*, defaults to `Resampling.BILINEAR`): An optional resampling filter. This can be one of `PIL.Image.Resampling.NEAREST`, `PIL.Image.Resampling.BOX`, `PIL.Image.Resampling.BILINEAR`, `PIL.Image.Resampling.HAMMING`, `PIL.Image.Resampling.BICUBIC` or `PIL.Image.Resampling.LANCZOS`. Only has an effect if `do_resize` is set to `True`. do_rescale (`bool`, *optional*, defaults to `True`): Whether to rescale the input to a certain `scale`. rescale_factor (`float`, *optional*, defaults to `1/ 255`): Rescale the input by the given factor. Only has an effect if `do_rescale` is set to `True`. do_normalize (`bool`, *optional*, defaults to `True`): Whether or not to normalize the input with mean and standard deviation. do_split_image (`bool`, *optional*, defaults to `False`): Whether to split the input images into overlapping patches for semantic segmentation. If set to `True`, the input images will be split into patches of size `size["shortest_edge"]` with an overlap between patches. Otherwise, the input images will be padded to the target size. do_pad (`bool`, *optional*, defaults to `False`): Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest number of patches in the batch. Padding will be applied to the bottom and right with zeros. image_mean (`int`, *optional*, defaults to `[0.485, 0.456, 0.406]`): The sequence of means for each channel, to be used when normalizing images. Defaults to the ImageNet mean. image_std (`int`, *optional*, defaults to `[0.229, 0.224, 0.225]`): The sequence of standard deviations for each channel, to be used when normalizing images. Defaults to the ImageNet std. ignore_index (`int`, *optional*): Label to be assigned to background pixels in segmentation maps. If provided, segmentation map pixels denoted with 0 (background) will be replaced with `ignore_index`. num_labels (`int`, *optional*): The number of labels in the segmentation map. pixel_valuesTNgp?F do_resizer;resample do_rescalerescale_factor do_normalizedo_split_imagedo_pad image_mean image_stdrrLc  t|di| ||nddd}t|d}||_||_||_||_||_||_||_ ||_ | | nt|_ | | nt|_| |_| |_y)Ni)r|r}Fdefault_to_square)super__init__rrr;rrrrrrrrrrrrL)selfrr;rrrrrrrrrrLkwargs __class__s r1rzEomtImageProcessor.__init__'s "6"'tsTW-XTU;"   $,(, (2(>*DY&/&;AU($rEimageinput_data_formatr3c ft|}t||d|d}td|||||dd|}|S)a  Resize an image. The shortest edge of the image is resized to size["shortest_edge"], with the longest edge resized to keep the input aspect ratio. Args: image (`np.ndarray`): Image to resize. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred. r|r}T)rr;r data_formatr return_numpyr)rrDr ) rrr;rrrrr: output_sizes r1r zEomtImageProcessor.resizeIsZ.$E* 0T/=RTXYgThi  #/    rE image_indexcgg}}t|}|d}t|}tj||z } | |z|z } | dkDr| | dz z nd} t | D]e} t | || z z} | |z}|d|dkDr|dd| |ddf}n|dddd| |f}|j ||j || |gg||fS)zCSlices an image into overlapping patches for semantic segmentation.r|r rN)rr7mathceilrnr8ro)rrr;rpatches patch_offsetsr: patch_size longer_side num_patches total_overlapoverlap_per_patchr,startendpatchs r1 _split_imagezEomtImageProcessor._split_imageos"$R#E* /* *o ii j 89 #j0;> ALqM[1_=VW{# rrpad_hpad_wr padded_images r1_padzEomtImageProcessor._padsr&u- &5d&;# |A}v-.A|e+,u:5z*k>R>Rdgh rEimagesrc ,|Dcgc] }t|}}|r#|Dcgc]}|j|||| | }}gg}}|rMt|D]=\}}|j|||\}}|j ||j |?|}|r|Dcgc]}|j ||}}|r!|Dcgc]}|j ||| }}| r"|Dcgc]}|j|| | | }}||fScc}wcc}wcc}wcc}wcc}w)zPreprocesses a batch of images.)r;rrr)scaler)meanstdr)rr enumeraterextendrrescale normalize)rrrr;rrrrrrrrrrrprocessed_imagesridximgroffsetss r1_preprocess_imagesz%EomtImageProcessor._preprocess_imagessn"6<vsC=DD8D Drc|jdk(rd}|d}tj}nd}| t|}|r|j ||||}|r|j ||}|r|j d}tj|S)zPreprocesses a single mask.rbTN.F)r;rrr) ndimr FIRSTrr rsqueezerj from_numpy) rrrrr;rrradded_channel_dims r1_preprocess_maskz#EomtImageProcessor._preprocess_masks  A % $ / :  0 6 6  %  ($BCS$T! #{{ !' +   #yy)94@  /77:  011rEsegmentation_mapsrreturn_tensorsc~||n |j}||n |j}||n |j}t|d}||n |j}||n |j }| | n |j } | | n |j} | | n |j} | | n |j} | | n |j} ||n |j}t|}t|s tdt|| | | | ||||j!|||||| || | | | || \}}|Xt#|d}|Dcgc] }t%|}}|Dcgc](}|j'||| |t(j*||*}}|j-|||||| }|r(|r&|Dcgc]}t/j0|c}|d <|Scc}wcc}wcc}w) aZ Preprocesses images or a batch of images. Args: images (`ImageInput`): Image or batch of images to preprocess. segmentation_maps (`ImageInput`, *optional*): The corresponding semantic segmentation maps with the pixel-wise annotations. instance_id_to_semantic_id (`list[dict[int, int]]` or `dict[int, int]`, *optional*): A mapping between object instance ids and class ids. do_split_image (`bool`, *optional*, defaults to `self.do_split_image`): Whether to split the input images into overlapping patches for semantic segmentation. do_resize (`bool`, *optional*, defaults to `self.do_resize`): Whether to resize the input images. size (`dict[str, int]`, *optional*, defaults to `self.size`): Target size as a dictionary with `"shortest_edge"` and `"longest_edge"` keys. resample (`PILImageResampling`, *optional*, defaults to `self.resample`): Resampling filter to use when resizing. do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): Whether to rescale the input images by `rescale_factor`. rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): Factor to scale image pixel values. do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): Whether to normalize the input images. do_pad (`bool`, *optional*, defaults to `False`): Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest number of patches in the batch. Padding will be applied to the bottom and right with zeros. image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`): Mean for normalization. Single value or list for each channel. image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`): Standard deviation for normalization. Single value or list for each channel. ignore_index (`int`, *optional*): Label to be assigned to background pixels in segmentation maps. If provided, segmentation map pixels denoted with 0 (background) will be replaced with `ignore_index`. return_tensors (`str` or `TensorType`, *optional*): The type of tensors to return. Can be `"pt"`, `"tf"`, `"np"`, or `"jax"`. data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): Channel format of the output image. Either `"channels_first"` or `"channels_last"`. input_data_format (`ChannelDimension` or `str`, *optional*): Channel format of the input image. FrzkInvalid image type. Must be of type PIL.Image.Image, numpy.ndarray, torch.Tensor, tf.Tensor or jax.ndarray.)rrrrrrr;r) rrr;rrrrrrrrrrrb)expected_ndims)rrr;rrr)rr)rrr;rrrrrrrrrrrrGrrrrrrNEAREST encode_inputsrjtensor)rrrrrrr;rrrrrrrrrrrpixel_values_listrmaskrencoded_inputsrs r1 preprocesszEomtImageProcessor.preprocesss6~,:+E4K^K^!*!6IDNN 'tTYYTU;'38#-#9Zt +9+E4K^K^'3'?|TEVEV !-4;;#-#9Zt !*!6IDNN '3'?|TEVEV )&1F#:  &!)%! ,0+B+B)!)%!#/,C, (=  ( 34EVW X BS T$!5 T  T): !%%%$'!/77 +&7& !  !++   &  ) ,  mTa.bu||G/D.bN? +7!U !./cs&F0>-F5 F:rc| |jn|}|Dcgc] }t|}}|t|d}td|i|}|g} g} t |D]\} } t| } t |t r|| } n|} t| | |\}}| jtj|| jtj|| |d<| |d<|Scc}w)a Pad images up to the largest image in a batch and create a corresponding `pixel_mask`. EoMT addresses semantic segmentation with a mask classification paradigm, thus input segmentation maps will be converted to lists of binary masks and their respective labels. Let's see an example, assuming `segmentation_maps = [[2,6,7,9]]`, the output will contain `mask_labels = [[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]]` (four binary masks) and `class_labels = [2,6,7,9]`, the labels for each mask. Args: pixel_values_list (`list[ImageInput]`): list of images (pixel values) to be padded. Each image should be a tensor of shape `(channels, height, width)`. segmentation_maps (`ImageInput`, *optional*): The corresponding semantic segmentation maps with the pixel-wise annotations. (`bool`, *optional*, defaults to `True`): Whether or not to pad images up to the largest image in a batch and create a pixel mask. If left to the default, will return a pixel mask that is: - 1 for pixels that are real (i.e. **not masked**), - 0 for pixels that are padding (i.e. **masked**). instance_id_to_semantic_id (`list[dict[int, int]]` or `dict[int, int]`, *optional*): A mapping between object instance ids and class ids. If passed, `segmentation_maps` is treated as an instance segmentation map where each pixel represents an instance id. Can be provided as a single dictionary with a global/dataset-level mapping or as a list of dictionaries (one per image), to map instance ids in each image separately. return_tensors (`str` or [`~file_utils.TensorType`], *optional*): If set, will return tensors instead of NumPy arrays. If set to `'pt'`, return PyTorch `torch.Tensor` objects. input_data_format (`ChannelDimension` or `str`, *optional*): The channel dimension format of the input image. If not provided, it will be inferred. Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: - **pixel_values** -- Pixel values to be fed to a model. - **mask_labels** -- Optional list of mask labels of shape `(labels, height, width)` to be fed to a model (when `annotations` are provided). - **class_labels** -- Optional list of class labels of shape `(labels)` to be fed to a model (when `annotations` are provided). They identify the labels of `mask_labels`, e.g. the label of `mask_labels[i][j]` if `class_labels[i][j]`. rr) tensor_type)rrR class_labels) rrrrr isinstancelistr2rorjr)rrrrrrrrrrRrrr instance_idrIclassess r1rz EomtImageProcessor.encode_inputssr-9,@t((l N_`l^L9``  $ >?PQR?S T %~7H&IWef  (KL)23D)E ?%%#12B#C 8$?" ?"-8N= )-9N> *?asC)segmentation_logitsr target_sizesc,|jd}g}g}|D]~}t||d|d\} } |jtj|| | f|j |jtj|| | f|j t |D]\} \} } }|| d|| dkDr6|| dd| |ddfxx|| z cc<|| dd| |ddfxxdz cc<Q|| dddd| |fxx|| z cc<|| dddd| |fxxdz cc<g}t t||D]P\}\}}||jdz }tj|d||d d d}|j|R|S) a Reconstructs full-size semantic segmentation logits from patch predictions. Args: segmentation_logits (`torch.Tensor`): A tensor of shape `(num_patches, num_classes, patch_height, patch_width)` representing predicted logits for each image patch. patch_offsets (`list[tuple[int, int, int]]`): A list of tuples where each tuple contains: - `image_index` (int): Index of the original image this patch belongs to. - `start` (int): Start pixel index of the patch along the long dimension (height or width). - `end` (int): End pixel index of the patch along the long dimension. target_sizes (`list[tuple[int, int]]`): list of original (height, width) dimensions for each image before preprocessing. size (`dict[str, int]`): A size dict which was used to resize. r r|r}rdrN)r6rbilinearFr;r align_corners) r'rDrorjr&rdrzipclampF interpolate)rrrrr; num_classesaggregated_logits patch_countsr:r=r> patch_idx image_idx patch_start patch_endreconstructed_logitsr logit_sumcountaveraged_logitsresized_logitss r1merge_image_patchesz&EomtImageProcessor.merge_image_patchess0*//2  & nJ6z4CXZ^_mZnoMFE  $ $U[[+vu1MViVpVp%q r    [&%,HQdQkQk l m n ?H >V J :I: ; I&q)L,CA,FF!),Q I0Eq-HIM`ajMkkIY';y+@!(CDID!),Q;y3H-HIM`ajMkkIY'1k).C(CDID  J "'05F 1U'V 8 #C#)U'%++!+*<6J+ 'M<15a-,6VWN }}y) J^c     ! !"2 3 4rEc||n |j}|j}|j}|j}t |}t j ||d}|jddddf}|j} tjd|| } |j| |||} | D cgc]} | jd} } | Scc} w) zIPost-processes model outputs into final semantic segmentation prediction.Nrr;rdim.zbqc, bqhw -> bchwr) r;masks_queries_logitsclass_queries_logitsrrrrsoftmaxrlrjeinsumrrm)routputsrr;rrrr masks_classes masks_probsr output_logitslogitpredss r1"post_process_semantic_segmentationz5EomtImageProcessor.post_process_semantic_segmentation1s'tTYY&;;&;;-- %d+  }}  -444 {[001DmUacgh 2?@!$@@ As0C  thresholdrUrVrrc ||n |j}|j}|j} | jd} | jddz } t |} t j || d}|j|||} | jdjd\}}g}t| D]}t| ||||||| \}}}|jddkrH|||n|jdd\}}tj||fdz }|j|gdyt|||||||||nd \}}|j||d|S) zIPost-processes model outputs into final panoptic segmentation prediction.Nrrr rrrrs segments_info)rSrprqrrrUrVr`)r;rrr'rrrrrr7rnrNrjr&rory)rrrr rUrVrrr;rr batch_sizerLrmask_probs_batchpred_scores_batchpred_labels_batchresultsr,rSrprqr=r>rsrts r1"post_process_panoptic_segmentationz5EomtImageProcessor.post_process_panoptic_segmentationQs'tTYY&;;&;;)//2 )//3a7 %d+  }}    ++,@,PTU/C/K/KPR/K/S/W/WXZ/[,,z" VA3L #%6q%9;LQ;OQZ\f4 0J [ "a'3?3K QQ[QaQabcbdQe ${{FE?;a?  rRS%5%''+-,G/;/GLOT& "L( NNL8T U- V.rEc @||n |j}|j}|j}t|}t j ||d}|j |||}|j} |jd} |jd} g} t| D]} || }|| }|jddddfjd\}}|dkDj}|jjd |jd zjd |jd jd d zz }||z}t!j"|| | d z }gg}}d}t| D]}||j%}t!j&||dk(r2||k\s8||||d k(<|j)|||j%t+|d d |d z }|j)||| j)||d| S)zDPost-processes model outputs into Instance Segmentation Predictions.Nrrrrr.r gư>rrerfr )r;rrrrrrrdr'rnrr7r5rlflattenrPrjr&rQallror9)rrrr r;rrrrrdr num_queriesrr, mask_pred mask_classrJ pred_classes pred_masks mask_scoresrprs instance_mapsrtrujris r1"post_process_instance_segmentationz5EomtImageProcessor.post_process_instance_segmentationsP'tTYY&;;&;;%d+  }}    ++,@,PTU%,,)//2 *004 z"! VA(+I-a0J$.#5#5"#5#=c3B3h#G#K#KB#O FL#a-..0J%,,.66q9J%*5!_'!+&!((A7 8 NNL8T UC! VDrE) NNNNNNNNNNNN)FFNNNN)NNNNNN)皙??r"NN)r#N)(__name__ __module__ __qualname____doc__model_input_namesrBILINEARboolrdictstrr8r5rrrr"ndarrayr r r tuplerrrrrrrrrrrjTensorrrr rr  __classcell__)rs@r1rrs+Z(()-'9'B'B '!$:>9=&*$( % %tCH~& %% %  %  % % % %U5$u+#567 %E%e"456 %sm %SM %L(:'B'BDH $zz$$% $ $E#/?*?$@A $ $L&*&D&s&uUY[_U_O`&6 * D RZZ "%))-'+)-!%%)*.'+:>9=>BDH:%:%D>:%tCH~& :% % :% ! :%:%TN:%!:%tn:%U5$u+#567:%E%e"456:%eC)9$9:;:%$E#/?*?$@A:% :%~%*!&)-'+48DH#2$#2D>#2 #2 tCH~& #2 % #23 001#2$E#/?*?$@A#2 #2J%&TX?C)-$()-'+%)*.'+!%:>9=&*;?4D4J4JDH%KK$E$tCH~*>S#X*N$OPK%-T#s(^$< K ! K D> KtCH~&K%KTNK!KtnKKU5$u+#567KE%e"456KsmK !sJ!78!K"3 001#K$$E#/?*?$@A%K& 'K'K`)-\`&*;?DHZ +Z&Z%-U4S#X3GcSVh3W-X$Y Z sm Z !sJ!78 Z$E#/?*?$@AZx7$"\\7$E#sC-017$5c?+ 7$ 38n 7$ ell  7$r"\\5c?+38n  ell  2*. 5c?+tCH~&   H #-0-1)-75c?+7 7  7 &+ 7 S *7tCH~&7r%& )- ?5c?+? ? tCH~& ?'?rEr)NNr!)r#r")r#r"N)6r'rtypingrrnumpyr"image_processing_utilsrrrimage_transformsr r r image_utilsr r rrrrrrrrutilsrrrrrr get_loggerr$loggerrjtorch.nn.functionalnn functionalrr+r8r2r.rDrNr_r5ryr,rr__all__rrEr1r=s5& "UU      H %## <@"&"D""D (c3h 8"D3-"DJ$5c?$P<8#6 ),-13"  3" "' 3"%S/*3"l'tCH~'%S/'S +S l rE