rh dZddlmZmZddlZddlZddlmZddlmZm Z m Z ddl m Z ddl mZdd lmZmZmZmZdd lmZdd lmZmZd d lmZej6eZdz-infinfmin_valmax_valc.t|t||SN)rminrr"r#s rclipr(:s wGU+ ,,r ceZdZ ddedededededededed ed eeefd d ffd Zde jd e jfdZ xZ S)MobileViTV2ConvLayerconfig in_channels out_channels kernel_sizestridegroupsbiasdilationuse_normalizationuse_activationrNc $t |t|dz dz |z} ||zdk7rtd|d|d||zdk7rtd|d|dt j ||||| |||d |_| r t j|d d d d |_nd|_| rdt| trt| |_ yt|jtrt|j|_ y|j|_ yd|_ y)NrrrzInput channels (z) are not divisible by z groups.zOutput channels (zeros) r,r-r.r/paddingr2r0r1 padding_modegh㈵>g?T) num_featuresepsmomentumaffinetrack_running_stats)super__init__r ValueErrorrConv2d convolution BatchNorm2d normalization isinstancestrr activation hidden_act) selfr+r,r-r.r/r0r1r2r3r4r7 __class__s rr?zMobileViTV2ConvLayer.__init__@s* {Q!+,x7  1 $/ }UV\U]]efg g99#%#    !#)$( "D "&D  .#."("8F--s3"():):";"("3"3"DOr featuresc|j|}|j|j|}|j|j|}|Sr%)rBrDrG)rIrKs rforwardzMobileViTV2ConvLayer.forwardvsK##H-    )))(3H ?? &x0Hr )rrFrTT) __name__ __module__ __qualname__rrboolrrFr?torchTensorrM __classcell__rJs@rr*r*?s"&+/4#!4#4# 4#  4#  4#4#4#4# 4#dCi(4# 4#l r r*c xeZdZdZ d dedededededdf fd Zd ejdejfd Z xZ S) MobileViTV2InvertedResidualzY Inverted residual block (MobileNetv2): https://huggingface.co/papers/1801.04381 r+r,r-r/r2rNc @t|ttt ||j zd}|dvrt d|d|dk(xr||k(|_t|||d|_ t|||d||||_ t|||dd |_ y) N)rrzInvalid stride .r)r,r-r.r )r,r-r.r/r0r2Fr,r-r.r4) r>r?rrround expand_ratior@ use_residualr* expand_1x1conv_3x3 reduce_1x1)rIr+r,r-r/r2expanded_channelsrJs rr?z$MobileViTV2InvertedResidual.__init__s *3u[6CVCV5V/W+XZ[\  vha89 9#q[K{l/J.  :KYZ - )*$  / )%  r rKc|}|j|}|j|}|j|}|jr||zS|Sr%)r_r`rar^)rIrKresiduals rrMz#MobileViTV2InvertedResidual.forwardsI??8,==*??8,&*&7&7x("EXEr )r rNrOrP__doc__rrr?rRrSrMrTrUs@rrWrWsc lm ' 69 IL VY eh  BF FFr rWc teZdZ d dedededededdf fd Zd ejdejfd ZxZ S) MobileViTV2MobileNetLayerr+r,r-r/ num_stagesrNct|tj|_t |D]5}t ||||dk(r|nd}|jj||}7y)Nrr)r,r-r/)r>r?r ModuleListlayerrangerWappend) rIr+r,r-r/riirlrJs rr?z"MobileViTV2MobileNetLayer.__init__sh ]]_ z" 'A/')!"avQ E JJ  e $&K 'r rKc8|jD] }||} |Sr%rl)rIrK layer_modules rrMz!MobileViTV2MobileNetLayer.forwards$ JJ .L#H-H .r )rr rNrOrPrrr?rRrSrMrTrUs@rrhrhsVqr'''69'IL'VY'kn' '  r rhcheZdZdZdededdffd ZdejdejfdZ xZ S) MobileViTV2LinearSelfAttentionay This layer applies a self-attention with linear complexity, as described in MobileViTV2 paper: https://huggingface.co/papers/2206.02680 Args: config (`MobileVitv2Config`): Model configuration object embed_dim (`int`): `input_channels` from an expected input of size :math:`(batch_size, input_channels, height, width)` r+ embed_dimrNc t|t||dd|zzdddd|_t j |j |_t|||dddd|_||_y)NrrTF)r+r,r-r1r.r3r4p) r>r?r*qkv_projrDropout attn_dropoutout_projrv)rIr+rvrJs rr?z'MobileViTV2LinearSelfAttention.__init__s{ ,!a)m,#   JJ)<)<=,!"#   #r hidden_statesc|j|}tj|d|j|jgd\}}}tjj j |d}|j|}||z}tj|dd}tjj j||j|z}|j|}|S)Nr)split_size_or_sectionsdimrTrkeepdim) rzrRsplitrvr functionalsoftmaxr|sumrelu expand_asr}) rIr~qkvquerykeyrcontext_scorescontext_vectorouts rrMz&MobileViTV2LinearSelfAttention.forwardsmmM* "KKQX\XfXfDgmnosE,,44U4C**>:~->r4Hhh!!&&u-0H0H0OOmmC  r rerUs@rrurus> #0#S#T#2U\\ellr ruc peZdZ d dededededdf fd Zdejdejfd Z xZ S) MobileViTV2FFNr+rvffn_latent_dim ffn_dropoutrNc t|t|||ddddd|_t j ||_t|||ddddd|_t j ||_y)NrTF)r+r,r-r.r/r1r3r4) r>r?r*conv1rr{dropout1conv2dropout2)rIr+rvrrrJs rr?zMobileViTV2FFN.__init__s| )!'#    ;/ )&"#     ;/ r r~c|j|}|j|}|j|}|j|}|Sr%)rrrr)rIr~s rrMzMobileViTV2FFN.forward(s@ =1  m4  =1  m4 r  rNrOrPrrfloatr?rRrSrMrTrUs@rrrsY ! 0!00 0  0  0@U\\ellr rc peZdZ d dededededdf fd Zdejdejfd Z xZ S) MobileViTV2TransformerLayerr+rvrdropoutrNcPt|tjd||j|_t |||_tj||_ tjd||j|_ t||||j|_ y)Nr num_groups num_channelsr:rx)r>r?r GroupNormlayer_norm_epslayernorm_beforeru attentionr{rlayernorm_afterrrffn)rIr+rvrrrJs rr?z$MobileViTV2TransformerLayer.__init__1s~  "  W]WlWl m7 J W- !||qyV\VkVkl!&)^VEWEWXr r~c|j|}|j|}||z}|j|}|j|}||z}|Sr%)rrrr)rIr~layernorm_1_outattention_output layer_outputs rrMz#MobileViTV2TransformerLayer.forward?sY// >>>/:(=8 ++M: xx - #m3 r rrrUs@rrr0s^  Y! Y Y Y  Y  Y U\\ ell r rcheZdZdedededdffd ZdejdejfdZxZ S) MobileViTV2Transformerr+n_layersd_modelrNc<t ||j}||zg|z}|Dcgc]}t|dzdz}}t j |_t|D].}t||||}|j j|0ycc}w)N)rvr) r>r?ffn_multiplierrrrkrlrmrrn) rIr+rrrffn_dimsd block_idxtransformer_layerrJs rr?zMobileViTV2Transformer.__init__Ls .."W,-82::ACbB'::]]_ x 1I ;'(9:M!  JJ  / 0  1;sBr~c8|jD] }||} |Sr%rq)rIr~rrs rrMzMobileViTV2Transformer.forward]s% JJ 8L(7M 8r rsrUs@rrrKsA101C1#1RV1"U\\ellr rceZdZdZ ddededededededed d ffd Zd ejd e eje eefffd Z dejde eefd ejfdZ dejd ejfdZ xZ S)MobileViTV2LayerzE MobileViTV2 layer: https://huggingface.co/papers/2206.02680 r+r,r- attn_unit_dim n_attn_blocksr2r/rNct ||j|_|j|_|}|dk(r)t ||||dk(r|nd|dkDr|dznd|_|}nd|_t||||j||_ t|||ddd|_ t||||_ tjd||j|_t|||dd d|_y) Nrr)r,r-r/r2)r,r-r.r0F)r,r-r.r3r4)rrrT)r>r? patch_size patch_width patch_heightrWdownsampling_layerr*conv_kernel_sizeconv_kxkconv_1x1r transformerrrr layernormconv_projection) rIr+r,r-rrr2r/ cnn_out_dimrJs rr?zMobileViTV2Layer.__init__hs  !,,"--# Q;&A')!)QvA*2Q,QA 'D #'K&*D #- #$//   - #$#   2&-ZghTZTiTij 4 #$"  r feature_mapc"|j\}}}}tjj||j|j f|j|j f}|j |||j|j zd}|||ffS)N)r.r/r)shaperrunfoldrrreshape)rIr batch_sizer, img_height img_widthpatchess r unfoldingzMobileViTV2Layer.unfoldings9D9J9J6 KY--&& **D,<,<=%%t'7'78'  //*k4;L;LtO_O_;_acdY///r r output_sizec|j\}}}}|j|||z|}tjj |||j |j f|j |j f}|S)N)rr.r/)rrrrfoldrr)rIrrrin_dimr n_patchesrs rfoldingzMobileViTV2Layer.foldingsz4;MM1 FJ //*fz.A9Mmm(( #**D,<,<=%%t'7'78 ) r rKc6|jr|j|}|j|}|j|}|j|\}}|j |}|j |}|j ||}|j|}|Sr%)rrrrrrrr)rIrKrrs rrMzMobileViTV2Layer.forwards  " "..x8H==*==* $~~h7""7+..)<<5''1r )rrr)rNrOrPrfrrr?rRrStuplerrrMrTrUs@rrrcs; !; ;  ;  ;  ; ; ;  ; z 0U\\ 0eELL%PSUXPX/r?r+rrkrlgradient_checkpointing output_striderr(width_multiplierrhrnrbase_attn_unit_dimsr)rIr+dilate_layer_4dilate_layer_5r2 layer_0_dim layer_1_dim layer_2_dim layer_3_dim layer_4_dim layer_5_dimlayer_1layer_2layer_3layer_4layer_5rJs rr?zMobileViTV2Encoder.__init__s|  ]]_ &+#+0/   1 $!N!N  ! !R '!N$ rF333R LVWce %R&*A*A%A2N $S6+B+B%BAN $S6+B+B%BAN $S6+B+B%BAN $S6+B+B%BAN + #$   '"+ #$   '"" #$()C)CA)FI`I`)`jkl ..q1   '"  MH" #$()C)CA)FI`I`)`jkl ..q1   '"  MH" #$()C)CA)FI`I`)`jkl ..q1   '"r r~output_hidden_states return_dictc|rdnd}t|jD]\}}||}|s||fz}|std||fDSt||S)Nc3&K|] }|| ywr%r).0vs r z-MobileViTV2Encoder.forward..6sXq!-Xs)last_hidden_stater~) enumeraterlrr )rIr~rrall_hidden_statesrorrs rrMzMobileViTV2Encoder.forward'sq #7BD(4 IOA|(7M#$58H$H!  I X]4E$FXX X- ]noor )FT) rNrOrPrr?rRrSrQrrr rMrTrUs@rrrsbO#0O#TO#h&+ p||p#p p u44 5 pr rcNeZdZUeed<dZdZdZdgZde jddfd Z y) MobileViTV2PreTrainedModelr+ mobilevitv2 pixel_valuesTrmodulerNct|tjtjtjfrm|j j jd|jj|j%|jj jyyt|tjrJ|jj j|j j jdyy)zInitialize the weightsr)meanstdNg?)rErLinearrArCweightdatanormal_r+initializer_ranger1zero_rfill_)rIr s r _init_weightsz(MobileViTV2PreTrainedModel._init_weightsCs fryy"))R^^D E MM   & &CT[[5R5R & S{{&   &&('  - KK   " " $ MM   $ $S ).r ) rNrOrPr__annotations__base_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modulesrModulerrr rrr;s9 %$O&*#+, *BII *$ *r rc eZdZd dedeffd ZdZe d dee jdeedeede e e ffd ZxZS) MobileViTV2Modelr+ expand_outputc t||||_||_t t d|j zdddd}t||j|ddd d |_ t||_ |jy ) a expand_output (`bool`, *optional*, defaults to `True`): Whether to expand the output of the model. If `True`, the model will output pooled features in addition to hidden states. If `False`, only the hidden states will be returned. rrrr'rYrr rTr,r-r.r/r3r4N) r>r?r+rrr(rr*r conv_stemrencoder post_init)rIr+rrrJs rr?zMobileViTV2Model.__init__Rs   *$ rF333R LVWce . ++$" *&1  r c|jD]e\}}|jj|}t|ts0|j jD]}|j j|gy)zPrunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsr!rlrErrr prune_heads)rIheads_to_prune layer_indexheadsmobilevitv2_layerrs r _prune_headszMobileViTV2Model._prune_headsnsv#1"6"6"8 C K $ 2 2; ? +-=>):)F)F)L)LC%%//;;EBC Cr rrrrc||n|jj}||n|jj}| td|j |}|j |||}|j r |d}tj|ddgd}n|d}d}|s|||fn|f}||ddzSt|||j S) Nz You have to specify pixel_valuesrrrrFrr)r pooler_outputr~) r+ruse_return_dictr@r r!rrRr r r~) rIrrrembedding_outputencoder_outputsr pooled_outputoutputs rrMzMobileViTV2Model.forwardxs%9$D $++JjJj &1%>,7,, !5#'     / 2 "JJ'8r2hPUVM / 2  M;H;T'7[lZnFOAB// /7/')77  r )T)NNN)rNrOrPrrQr?r*rrrRrSrrr rMrTrUs@rrrPs~08C04/3&* ' u||,' 'tn' d^ ' u>> ? ' ' r rz MobileViTV2 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. ) custom_introceZdZdeddffd Ze d deejdee deejdee de e e ff d Z xZS) !MobileViTV2ForImageClassificationr+rNcLt|||j|_t||_t d|j zd}|jdkDr!tj||jntj|_ |jy)NrrYrr) in_features out_features) r>r? num_labelsrrrrrr Identity classifierr")rIr+r-rJs rr?z*MobileViTV2ForImageClassification.__init__s   +++F3%cF,C,C&CQO   1$ II,V=N=N O  r rrlabelsrc||n|jj}|j|||}|r |jn|d}|j |}d}||jj |j dk(rd|j_nl|j dkDrL|jtjk(s|jtjk(rd|j_nd|j_|jj dk(rIt} |j dk(r&| |j|j}n| ||}n|jj dk(r=t} | |jd|j |jd}n,|jj dk(rt} | ||}|s|f|ddz} ||f| zS| St!|||j" S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr,r regressionsingle_label_classificationmulti_label_classificationrr)losslogitsr~)r+r/rr.r< problem_typer:dtyperRlongrrsqueezerviewrrr~) rIrrr=routputsr2rCrBloss_fctr3s rrMz)MobileViTV2ForImageClassification.forwards&1%r?rAdaptiveAvgPool2d global_poolr*r)rIr+r,r-rJs rr?zMobileViTV2ASPPPooling.__init__sB //A>, #%"!  r rKc|jdd}|j|}|j|}tjj ||dd}|S)Nr-bilinearFsizemode align_corners)rrPrrr interpolate)rIrK spatial_sizes rrMzMobileViTV2ASPPPooling.forwardsS~~bc* ##H-==*==,,XLzin,or rsrUs@rrMrMsB  0  s  RU  Z^   r rMcdeZdZdZdeddffd ZdejdejfdZxZ S)MobileViTV2ASPPz ASPP module defined in DeepLab papers: https://huggingface.co/papers/1606.00915, https://huggingface.co/papers/1706.05587 r+rNc t|td|jzd}|}|j}t |j dk7r tdtj|_ t|||dd}|jj||jj|j Dcgc]}t|||d|d c}t|||}|jj|t|d |z|dd|_tj |j" |_ycc}w) NrrYrr z"Expected 3 values for atrous_ratesrrr[)r,r-r.r2r4rx)r>r?rraspp_out_channelslen atrous_ratesr@rrkconvsr*rnextendrMprojectr{aspp_dropout_probr) rIr+encoder_out_channelsr,r- in_projectionrate pool_layerrJs rr?zMobileViTV2ASPP.__init__s6 -cF4K4K.KUVW* // v"" #q (AB B]]_ , #%!   -( #// % +!- !!#)   ,FKN  *%+ L 0|YZkq  zzF$<$<= ) sErKcg}|jD]}|j||tj|d}|j |}|j |}|S)Nrr)r`rnrRcatrbr)rIrKpyramidconvpooled_featuress rrMzMobileViTV2ASPP.forward=s\JJ +D NN4> * +))G+,,w/,,7r rNrOrPrfrr?rRrSrMrTrUs@rrZrZ s8*>0*>T*>X r rZcdeZdZdZdeddffd ZdejdejfdZxZ S)MobileViTV2DeepLabV3zJ DeepLabv3 architecture: https://huggingface.co/papers/1706.05587 r+rNc t|t||_t j |j |_t||j|jdddd|_ y)NrFT)r,r-r.r3r4r1) r>r?rZasppr Dropout2dclassifier_dropout_probrr*r]r:r<rIr+rJs rr?zMobileViTV2DeepLabV3.__init__Ns] #F+ ||F$B$BC . 00**#  r r~cr|j|d}|j|}|j|}|S)Nr)rqrr<)rIr~rKs rrMzMobileViTV2DeepLabV3.forward^s699]2./<<)??8,r rmrUs@rroroIs7 0 T U\\ellr rozZ MobileViTV2 model with a semantic segmentation head on top, e.g. for Pascal VOC. ceZdZdeddffd Ze d deejdeejdee dee de e e ff d Z xZS) "MobileViTV2ForSemanticSegmentationr+rNct|||j|_t|d|_t ||_|jy)NF)r)r>r?r:rrrosegmentation_headr"rts rr?z+MobileViTV2ForSemanticSegmentation.__init__ksE   +++F%H!5f!= r rr=rrch||n|jj}||n|jj}|$|jjdk(r t d|j |d|}|r |j n|d}|j|}d}|Ytjj||jdddd } t|jj } | | |}|s|r |f|ddz} n |f|d dz} ||f| zS| St|||r|j d Sdd S) a labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). Examples: ```python >>> import requests >>> import torch >>> from PIL import Image >>> from transformers import AutoImageProcessor, MobileViTV2ForSemanticSegmentation >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> image_processor = AutoImageProcessor.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") >>> model = MobileViTV2ForSemanticSegmentation.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") >>> inputs = image_processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs) >>> # logits are of shape (batch_size, num_labels, height, width) >>> logits = outputs.logits ```Nrz/The number of labels should be greater than oneTr,r-rRFrS) ignore_indexr)rBrCr~ attentions)r+rr/r:r@rr~ryrrrWrrsemantic_loss_ignore_indexr) rIrr=rrrIencoder_hidden_statesrCrBupsampled_logitsrJr3s rrMz*MobileViTV2ForSemanticSegmentation.forwardussH%9$D $++JjJj &1%  !}}88V\\"#.Zu 9  (T[[5[5[\H,f5D# WQR[0 WQR[0)-)9TGf$ Ev E&3G'//   NR   r rK)rNrOrPrr?rrrRrSrQrrrrMrTrUs@rrwrwes 0T04)-/3&* I u||,I &I 'tn I d^ I u-- . I I r rw)r6rwrr)rYN)4rftypingrrrRtorch.utils.checkpointrtorch.nnrrr activationsr modeling_layersr modeling_outputsr r rrmodeling_utilsrutilsrrconfiguration_mobilevitv2r get_loggerrNloggerrrrr(rr*rWrhrurrrrrrrr6rMrZrorw__all__rr rrs"!" AA!9 .,8   H % #  HSM UX ).f e ----Y^- =299=B-F"))-Fb .<RYY<~&RYY&R"))6RYY0o1odcpcpL***(O 1O O d G (BG  G VRYY09bii9z2998 U )CU  U p r