rhC5ddlZddlmZddlmZmZmZmZddlZddlm Z ddl m Z ddl m Z ddlmZdd lmZdd lmZmZmZmZdd lmZmZdd lmZdd lmZmZmZddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&ddl'm(Z(m)Z)m*Z*ddl+m,Z,m-Z-m.Z.e%j^e0Z1Gdde jdZ3 dFde jddejhdejhdejhdeejhde5de5fdZ6Gdde jdZ7Gdd e jdZ8Gd!d"eZ9Gd#d$e jdZ:Gd%d&e jdZ;Gd'd(e jdZ<Gd)d*e jdZ=Gd+d,e jdZ>Gd-d.e jdZ?Gd/d0eZ@Gd1d2e jdZAGd3d4e jdZBe#Gd5d6eZCGd7d8eCZDGd9d:eCZEee#d;<Gd=d>e!ZFe#d?<Gd@dAeCZGe#dB<GdCdDeCe ZHgdEZIy)GN) dataclass)AnyCallableOptionalUnion)nn)ACT2FN)GenerationMixin)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsBaseModelOutputWithPooling,BaseModelOutputWithPoolingAndCrossAttentions)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer) ModelOutputTransformersKwargsauto_docstringcan_return_tuplelogging torch_int) AutoModelAutoModelForCausalLMAutoModelForSeq2SeqLM)InstructBlipVideoConfigInstructBlipVideoQFormerConfigInstructBlipVideoVisionConfigceZdZdeffd Zdej dededej fdZd dejd e dej fd Z xZ S) !InstructBlipVideoVisionEmbeddingsconfigcpt|||_|j|_|j |_|j |_tjtjdd|j|_ tjd|j|j |j |_ |j |j zdz|_|jdz|_tjtjd|j|j|_y)Nr"r ) in_channels out_channels kernel_sizestrider)super__init__r( hidden_size embed_dim image_size patch_sizer Parametertorchrandnclass_embeddingConv2dpatch_embedding num_patches num_positionsposition_embeddingselfr( __class__s /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/instructblipvideo/modeling_instructblipvideo.pyr/z*InstructBlipVideoVisionEmbeddings.__init__7s  ++ ++ ++!||EKK1dnn,MN!yyDOO\`\k\k !OOt>1D!--1"$,,u{{1d>P>PRVR`R`/a"b embeddingsheightwidthreturnc|jddz }|jjddz }tjj s||k(r||k(r |jS|jddddf}|jddddf}|jd}||j z} ||j z} t |dz} |jd| | |}|jdddd}tjj|| | fdd }|jddddjdd|}tj||fd S) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 r"Ng?rr rbicubicF)sizemode align_cornersdim)shaper<r5jit is_tracingr3rreshapepermuter functional interpolateviewcat) r>rBrCrDr:r;class_pos_embedpatch_pos_embedrM new_height new_widthsqrt_num_positionss r@interpolate_pos_encodingz:InstructBlipVideoVisionEmbeddings.interpolate_pos_encodingIs`!&&q)A- //55a81< yy##%+*F6UZ?** *11!RaR%811!QR%8r"t. T__, &}c'9:)11!5GI[]`a)11!Q1=--33 i( 4 *11!Q1=BB1b#Nyy/?;CCrA pixel_valuesr\c|j\}}}}|jjj}|j|j |}|j dj dd}|jj|ddj |} tj| |gd} |r|j| ||} n |j} | | ddd| jdddfj |z} | S)Ndtyperr"rGrL)rNr9weightr`toflatten transposer7expandr5rVr\r<rI) r>r]r\ batch_size_rCrD target_dtype patch_embeds class_embedsrBr<s r@forwardz)InstructBlipVideoVisionEmbeddings.forwardqs'3'9'9$ Avu++2288 ++LOO,O,OP #++A.88A> ++22:q"EHHV YY l;C #!%!>!>z6SX!Y !%!8!8 "4Q8L*//!:L8La5O"P"S"ST`"aa rAF) __name__ __module__ __qualname__r%r/r5Tensorintr\ FloatTensorboolrk __classcell__r?s@r@r'r'6smc<c$&D5<<&D&DUX&D]b]i]i&DP E$5$5 QU bgbnbn rAr'modulequerykeyvalueattention_maskscalingdropoutc ptj||jdd|z}|||z}tjj |d}tjj |||j}tj||} | jddj} | |fS)NrGrL)ptrainingr"r) r5matmulrdrrSsoftmaxr|r contiguous) rvrwrxryrzr{r|kwargs attn_weights attn_outputs r@eager_attention_forwardrs<<s}}R'<=GL!#n4 ==((2(>L==((6??([L,,|U3K''1-88:K  $$rAceZdZdZfdZdej dedefdZ d dej de ej d e e d e ej e ej e e ej ffd Z xZ S) InstructBlipVideoAttentionz=Multi-headed attention from 'Attention Is All You Need' paperct|||_|j|_|j |_|j|j z|_|j|j z|jk7r&td|jd|j d|jdz|_ d|_ |j|_ tj|jd|jzd|_|jretj t#j$|j}tj t#j$|j}nd}d}|Qt#j&|t#j(|d|f}tj ||j_tj|j|j|_y) Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gFr )bias) requires_grad)r.r/r(r0r1num_attention_heads num_headshead_dim ValueErrorscale is_causalattention_dropoutrLinearqkvqkv_biasr4r5zerosrV zeros_liker projection)r>r(q_biasv_biasrr?s r@r/z#InstructBlipVideoAttention.__init__su  ++33$..8 ==4>> )T^^ ;MdnnM]^NN#2' ]]D( !'!9!999T^^Q-?eL ??\\%++dnn"=>F\\%++dnn"=>FFF  yy&%*:*:6QV*WY_!`aHLL2DHHM))DNNDNNCrAtensorseq_lenbszc|j|||j|jjddj S)Nr"r)rUrrrdr)r>rrrs r@_shapez!InstructBlipVideoAttention._shapes7{{3GQQRSUVWbbddrA hidden_states head_maskoutput_attentionsrEc |j\}}}|j|}|j||d|j||jzj ddddd}|d|d|d} } } t } |j jdk7rN|j jdk(r|rtjdnt|j j} | || | | fd |jsd n |j|jd |\} }| j||d j} |j| } |r| |f}|S| d f}|S) z#Input shape: Batch x Time x Channelr rrr"eagersdpaz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.N)rzr|r{rG)rIrrQrrRrr(_attn_implementationlogger warning_oncerrrrrr)r>rrrrrtgt_lenr1 mixed_qkv query_states key_states value_statesattention_interfacerroutputss r@rkz"InstructBlipVideoAttention.forwardsr#0"4"4"6WiHH]+ %%c7At~~yTXTbTbGbckk q!Q 2;1y|YWX\,j (? ;; + +w 6{{//69>O##L '>dkk>^>^&_#$7     %  #}}C$2H2HJJ %  % ! \"))#w;FFH ook2 1B; -JUVZH[rANF)rmrnro__doc__r/r5rprqrrrstuplerkrtrus@r@rrsGD>eU\\eCece -1,1 ,||,ELL),$D> , u||Xell3XeELL>Q5RR S ,rArcVeZdZfdZdej dej fdZxZS)InstructBlipVideoMLPct|||_t|j|_t j|j|j|_ t j|j|j|_ yN) r.r/r(r hidden_act activation_fnrrr0intermediate_sizefc1fc2r=s r@r/zInstructBlipVideoMLP.__init__sd  #F$5$5699V//1I1IJ99V55v7I7IJrArrEcl|j|}|j|}|j|}|Sr)rrrr>rs r@rkzInstructBlipVideoMLP.forwards4/ **=9 / rArmrnror/r5rprkrtrus@r@rrs$KU\\ellrArc eZdZdeffd Z ddej dej deede ejfdZ xZ S) InstructBlipVideoEncoderLayerr(cDt||j|_t ||_t j|j|j|_ t||_ t j|j|j|_ yNeps) r.r/r0r1r self_attnr LayerNormlayer_norm_eps layer_norm1rmlp layer_norm2r=s r@r/z&InstructBlipVideoEncoderLayer.__init__sm ++3F;<<Frrzrresidualrrs r@rkz%InstructBlipVideoEncoderLayer.forwards !((7 &*nn'$/'5' # | &0  ((7 / %0  "    &GrArl) rmrnror#r/r5rprrsrrrrkrtrus@r@rrsYS6S-2 $||$ $$D> $ u  ! $rArc eZdZdZdeffd Z d deejdee dee dee de e e ff d Z xZS) InstructBlipVideoEncodera" Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`InstructBlipVideoEncoderLayer`]. Args: config (`InstructBlipVideoConfig`): The corresponding vision configuration for the `InstructBlipVideoEncoder`. r(ct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}wr) r.r/r(r ModuleListrangenum_hidden_layersrlayersgradient_checkpointing)r>r(rgr?s r@r/z!InstructBlipVideoEncoder.__init__5sQ  mmTYZ`ZrZrTs$tq%B6%J$tu &+#%usA#rzroutput_hidden_states return_dictrEc||n|jj}||n|jj}||n|jj}|rdnd}|rdnd}|}t |j D])\} } |r||fz}| |||} | d}|s!|| dfz}+|r||fz}|st d|||fDSt|||S)a Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Embedded representation of the inputs. Should be float, not int tokens. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. N)rzrrr"c3&K|] }|| ywrr.0vs r@ z3InstructBlipVideoEncoder.forward..tseqWXWdes)last_hidden_stater attentions)r(rruse_return_dict enumeraterrr) r> inputs_embedsrzrrrencoder_statesall_attentionsridx encoder_layer layer_outputss r@rkz InstructBlipVideoEncoder.forward;s82C1N-TXT_T_TqTq$8$D $++JjJj &1%>Ne]NN$See e+>Vd  rA)NNNN)rmrnrorr#r/rr5rprsrrrrkrtrus@r@rr+sy,6,26,0/3&* < !.< $D> < 'tn < d^ <  uo% &< rArcPeZdZdfd ZdZdZdZdZdZ d dZ xZ S) *InstructBlipVideoQFormerMultiHeadAttentionct|||_|j|jzdk7r0t |ds$t d|j|jfz|j|_t|j|jz |_|j|jz|_ tj|j|j|_ |r_tj|j|j|_tj|j|j|_n^tj|j|j|_tj|j|j|_tj |j"|_t'|dd|_|j(dk(s|j(dk(rF|j*|_tj,d|j*zd z |j|_d |_y) Nrembedding_sizezLThe hidden size (%d) is not a multiple of the number of attention heads (%d)position_embedding_typeabsolute relative_keyrelative_key_queryrr"F)r.r/r(r0rhasattrrrqattention_head_size all_head_sizerrrwencoder_hidden_sizerxryDropoutattention_probs_dropout_probr|getattrrmax_position_embeddings Embeddingdistance_embeddingsave_attentionr>r(is_cross_attentionr?s r@r/z3InstructBlipVideoQFormerMultiHeadAttention.__init__{s     : : :a ?PVXhHi^%%v'A'ABC  $*#=#= #&v'9'9F 9T=Y=Y]q=q+1+I+ID (&(ll1v7U7U3UXY3Y[_[s[s&tD ##rAc||_yrattn_gradients)r>rs r@save_attn_gradientsz>InstructBlipVideoQFormerMultiHeadAttention.save_attn_gradientss ,rAc|jSrrr>s r@get_attn_gradientsz=InstructBlipVideoQFormerMultiHeadAttention.get_attn_gradientss"""rAc||_yr attention_map)r>r s r@save_attention_mapz=InstructBlipVideoQFormerMultiHeadAttention.save_attention_maps *rAc|jSrrrs r@get_attention_mapzx new_x_shapes r@transpose_for_scoresz?InstructBlipVideoQFormerMultiHeadAttention.transpose_for_scoressNffhsmt'?'?AYAY&ZZ AFFK yyAq!$$rAc|du}|rC|j|j|}|j|j|} |}n@|j|j|}|j|j|} |j|} |j| } t j | |j dd} |jdk(s|jdk(rF|jd} t j| tj|jjdd}t j| tj|jjdd}||z }|j||jzdz }|j| j }|jdk(rt j"d| |}| |z} nE|jdk(r6t j"d| |}t j"d ||}| |z|z} | t%j&|j(z } | j }|| |z} t+j,d | j|}|r8|j.r,|j1||j3|j4|j7|}|||z}t j || }|j9d d dd j;}|jdd|j<fz}|j|}|r||f}|S|f}|S)NrGr~rrr"r`devicer_zbhld,lrd->bhlrzbhrd,lrd->bhlrrLrrr )rrxryrwr5rrdrrIarangelongrrUrrrbr`einsummathsqrtrrSoftmaxrr  register_hookrr|rRrr)r>rrzrencoder_hidden_statesencoder_attention_maskrr key_layer value_layermixed_query_layer query_layerattention_scores seq_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_scores_dtypeattention_probsattention_probs_dropped context_layernew_context_layer_shapers r@rkz2InstructBlipVideoQFormerMultiHeadAttention.forwards83$> 11$((;P2QRI33DJJ?T4UVK3N11$((=2IJI33DJJ}4MNK JJ}5//0AB !<< Y5H5HR5PQ  ' '> 9T=Y=Y]q=q&++-a0J"\\*EJJ}OcOcdiijlnopN"\\*EJJ}OcOcdiijkmopN%6H#'#:#:8dFbFb;bef;f#g #7#:#:ARAR#:#S ++~=+0<<8H+Wk+l(#36N#N --1EE16>NP[]q1r./4||AABXY $"5"5  # #O 4  ) )$*B*B C#',,"?  &= &I # %S"S* **,CD 6G=/2O\M]rArlNNNNF) rmrnror/rrr r rrkrtrus@r@rrzs8$8-#+"%"#KrArcneZdZfdZdej dej dej fdZxZS)"InstructBlipVideoQFormerSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yr) r.r/rrr0denserrrhidden_dropout_probr|r=s r@r/z+InstructBlipVideoQFormerSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= rAr input_tensorrEcr|j|}|j|}|j||z}|Srr3r|rr>rr5s r@rkz*InstructBlipVideoQFormerSelfOutput.forward7 =1  ]3 }|'CD rArrus@r@r1r11> U\\RWR^R^rAr1ceZdZd fd ZdZ d dej deejdeejdeejdeejdee d e ej fd Z xZ S) !InstructBlipVideoQFormerAttentionct|t|||_t ||_t |_yr)r.r/r attentionr1outputset pruned_headsrs r@r/z*InstructBlipVideoQFormerAttention.__init__s3 CFL^_8@ ErAc>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)Nrr"rL)lenrr>rrrArrwrxryr?r3runion)r>headsindexs r@ prune_headsz-InstructBlipVideoQFormerAttention.prune_heads s u:? 7 4>>55t~~7Y7Y[_[l[l u 2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:rArrzrrrrrEcp|j||||||}|j|d|}|f|ddz} | S)N)rrzrrrrrr")r>r?) r>rrzrrrr self_outputsattention_outputrs r@rkz)InstructBlipVideoQFormerAttention.forwardsY~~')"7#9/ &  ;;|A F#% QR(88rArlr/) rmrnror/rGr5rprrrrsrrkrtrus@r@r<r<s" ;*7;15=A>B,1||!!2!23E--.  ((9(9:  !)):): ; $D> u|| rAr<cVeZdZfdZdej dej fdZxZS)$InstructBlipVideoQFormerIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r.r/rrr0rr3 isinstancerstrr intermediate_act_fnr=s r@r/z-InstructBlipVideoQFormerIntermediate.__init__4s] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $rArrEcJ|j|}|j|}|Sr)r3rPrs r@rkz,InstructBlipVideoQFormerIntermediate.forward<s& =1 00? rArrus@r@rLrL3s#9U\\ellrArLcneZdZfdZdej dej dej fdZxZS)InstructBlipVideoQFormerOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) r.r/rrrr0r3rrrr4r|r=s r@r/z'InstructBlipVideoQFormerOutput.__init__Cs` YYv779K9KL f&8&8f>S>STzz&"<"<= rArr5rEcr|j|}|j|}|j||z}|Srr7r8s r@rkz&InstructBlipVideoQFormerOutput.forwardIr9rArrus@r@rSrSBr:rArSc>eZdZfdZ ddZdZdZxZS)InstructBlipVideoQFormerLayercft||j|_d|_t ||_||_||jzdk(rt |d|_d|_ nd|_ t||_ t||_ t||_t||_y)Nr"rT)rF)r.r/chunk_size_feed_forward seq_len_dimr<r> layer_idxcross_attention_frequencycrossattentionhas_cross_attentionrL intermediaterSr?intermediate_query output_queryr>r(r[r?s r@r/z&InstructBlipVideoQFormerLayer.__init__Qs '-'E'E$:6B" v77 71 <"CF_c"dD '+D $',D $@H4V< "Fv"N:6BrAc |j||||}|d} |dd} |dkDr| ddd|ddf} |jr1| td|j| |||||} | d} | | ddz} t |j |j |j| } | jd|kDrt |j|j |j| dd|dddfj| j}tj| |gd} n,t |j|j |j| } | f| z} | S)N)rzrrrr"z>encoder_hidden_states must be given for cross-attention layers)rzrrrrrL)r>r^rr]rfeed_forward_chunk_queryrYrZrNfeed_forward_chunkrbrr5rV)r>rrzrrrr query_lengthself_attention_outputsrJrquery_attention_outputcross_attention_outputs layer_outputlayer_output_texts r@rkz%InstructBlipVideoQFormerLayer.forwardes"& )/ "0"  2!4(, ! %5a,6I%J "''(0$%eff*.*=*=*#1'*?+A&7 +>+'*A)C&!$;AB$??4--,,  & L %%a(<7$=++00$$$Q q%89 % "\(() " %yy,8I)JPQR 4'',,   L  /G+rAcL|j|}|j||}|Sr)r_r?r>rJintermediate_outputrjs r@rez0InstructBlipVideoQFormerLayer.feed_forward_chunks,"//0@A{{#68HI rAcL|j|}|j||}|Sr)r`rarms r@rdz6InstructBlipVideoQFormerLayer.feed_forward_chunk_querys."556FG(()<>NO rA)NNNNFr)rmrnror/rkrerdrtrus@r@rWrWPs-C."#=~ rArWc6eZdZfdZ ddZxZS)InstructBlipVideoQFormerEncoderc t|||_tjt |j Dcgc]}t||c}|_d|_ ycc}wr) r.r/r(rrrrrWlayerrrbs r@r/z(InstructBlipVideoQFormerEncoder.__init__sW  ]]OTU[UmUmOn o) *69 = o  ',# psA$c |rdnd} |rdnd} |rdnd} t|jjD]^} |j| }|r| |fz} ||| nd}|||||||| }|d}|s:| |dfz} | dkDsI|jsV| |dfz} `|r| |fz} |st d|| | | fDSt || | | S)Nr)rrrfrr"rc3$K|]}|| ywrrrs r@rz:InstructBlipVideoQFormerEncoder.forward..s  = s)rrrcross_attentions)rr(rrsr^rr)r>rrzrrrrrrrfall_hidden_statesall_self_attentionsall_cross_attentionsi layer_modulelayer_head_maskrs r@rkz'InstructBlipVideoQFormerEncoder.forwards0#7BD$5b4%6rDt{{445 VA::a=L#$58H$H!.7.CilO(%'="3)M*!,M &9]1=MS>STzz&"<"<=   ELL)G)GHOOPWXej  (/v7PR\']$ rAc||jd}nd}|&|jdd|||zfj}|k|j|}|jdk(r/|j |j |j}||z}|tj||fd}n|}|j |jjj}|j|}|j|}|S)Nr"rrrL)rIrclonerrrrbrr5rVrrar`r|)r> input_idsr query_embedspast_key_values_lengthr"rBrs r@rkz*InstructBlipVideoQFormerEmbeddings.forwards  ")!,JJ  ,,Q0FVlIl0l-lmssuL  --i8J++z9&*&>&>|zO`O`?a&b#'*== '"YY j'AqI %J]]4>>#8#8#>#>? ^^J/ \\*- rA)NNNr)rmrnrorr/rkrtrus@r@r~r~sE$ rAr~cBeZdZUeed<dZdZdZdZdZ dZ dZ gdZ dZ y) InstructBlipVideoPreTrainedModelr(blipT)r~rrr1c|jj}t|tjtj frY|j jjd||j%|jjjyyt|tjr(|j jjd|yt|tjrJ|jjj|j jjdyt|trYtjj!|j"d|tjj!|j$d|yt|t&t(fr%|j*jjyy)zInitialize the weightsr)meanstdN?)r(initializer_rangerNrrr8radatanormal_rzero_rrfill_r'init trunc_normal_r<r7)InstructBlipVideoForConditionalGenerationInstructBlipVideoModel query_tokens)r>rvfactors r@ _init_weightsz.InstructBlipVideoPreTrainedModel._init_weights;sI.. fryy"))4 5 MM   & &CV & <{{&   &&('  - MM   & &CV & <  - KK   " " $ MM   $ $S )  A B GG ! !&";";#6 ! R GG ! !&"8"8s ! O !JLb c d    $ $ * * ,erAN)rmrnror#__annotations__base_model_prefixsupports_gradient_checkpointing_supports_attention_backend_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_no_split_modulesrrrAr@rr(s> ##&*#"&N!-rArceZdZUdZeed<deffd Ze d dee jdee dee dee de de e eff d Zd ZxZS) InstructBlipVideoVisionModelr]r(ct||||_|j}t ||_t ||_tj||j|_ |jyr) r.r/r(r0r'rBrencoderrrrpost_layernorm post_init)r>r(r1r?s r@r/z%InstructBlipVideoVisionModel.__init__Ss]   && ;FC/7  ll9&:O:OP rArrrr\rEc||n|jj}||n|jj}||n|jj}| t d|j ||}|j ||||}|d}|j|}|dddddf} |j| } |s || f|ddzSt|| |j|jS)Nz You have to specify pixel_values)r\)rrrrrr")r pooler_outputrr) r(rrrrrBrrrrr) r>r]rrrr\rencoder_outputsr pooled_outputs r@rkz$InstructBlipVideoVisionModel.forward^s2C1N-TXT_T_TqTq$8$D $++JjJj &1%( 'tn ( d^ ( #' (  u00 1( ( TrArceZdZdZdZdZdZdZdeffd Z dZ dZ dZ dde jd eed e j"d ed e jf d Z dde j(dee j,dee j(dee jdee j,dee j,dee j,deedeedeed eee j,effdZxZS)InstructBlipVideoQFormerModelz Querying Transformer (Q-Former), used in InstructBlipVideo. Slightly modified from BLIP-2 as it also takes the instruction as input. Fr(ct||||_t||_t ||_|jyr)r.r/r(r~rBrqrrr=s r@r/z&InstructBlipVideoQFormerModel.__init__s9     rAc.|jjSrrBrrs r@rz2InstructBlipVideoQFormerModel.get_input_embeddingss...rAc&||j_yrrr>rys r@set_input_embeddingsz2InstructBlipVideoQFormerModel.set_input_embeddingss*/'rAc|jD]7\}}|jj|jj |9y)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrsr>rG)r>heads_to_prunersrEs r@ _prune_headsz*InstructBlipVideoQFormerModel._prune_headssE +002 CLE5 LL  u % / / ; ;E B CrArz input_shaper has_queryrEc|jdk(r|dddddddf}n=|jdk(r|ddddddf}ntd|d|jd|j|j}d|z d z}|S) a> Makes broadcastable attention and causal masks so that future and masked tokens are ignored. Arguments: attention_mask (`torch.Tensor`): Mask with ones indicating tokens to attend to, zeros for tokens to ignore. input_shape (`tuple[int]`): The shape of the input to the model. device: (`torch.device`): The device of the input to the model. Returns: `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. r Nrz!Wrong shape for input_ids (shape z) or attention_mask (shape )r_rg)rMrrNrbr`)r>rzrrrextended_attention_masks r@get_extended_attention_maskz9InstructBlipVideoQFormerModel.get_extended_attention_masks.    1 $&4Qa]&C #    !Q &'5QdA5E&F #3K=@[\j\p\p[qqrs #:"<"<4::"<"N#&)@#@H"L&&rArrrrrrrrrc D||n|jj}| | n|jj} | | n|jj} | | t d||j dnd} |j |||} | jdd} | \}}| j}|tj||f|}|j|| |}|t|tr|dj\}}}n|j\}}}||f}t|tr|Dcgc]}|j|}}n?|)tj||}|j|}n|j|}nd}|j||jj }|j#| |||||| | |  }|d}|dddddf}| s ||f|ddzSt%|||j&|j(|j*|j, Scc}w) a encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`Cache` of length `config.n_layers` with each tuple having 4 tensors of: shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). Nz7You have to specify query_embeds when input_ids is Noner"r)rrrrG)r)rzrrrrrrrf)rrpast_key_valuesrrrv)r(rrrrrNrBrIrr5onesrrNlistinvert_attention_mask get_head_maskrrrrrrrv)r>rrzrrrrrrrrrfembedding_outputrrfr"rrencoder_batch_sizeencoder_sequence_lengthrgencoder_hidden_shapemaskencoder_extended_attention_maskrsequence_outputrs r@rkz%InstructBlipVideoQFormerModel.forwards@2C1N-TXT_T_TqTq$8$D $++JjJj &1%S^`f"g ! ,/6AVWXAYA^A^A`>"$;QAVA[A[A]>"$;Q$68O#P 0$7`v2wX\43M3Md3S2w/2w'/).4HQW)X&262L2LMc2d/262L2LMc2d/.2 +&&y$++2O2OP ,, 2"7#B/!5#%'  *!,'1a0 #]3oab6II I;-'+;;)77&11,==   C3xs-Hrl) NNNNNNNNN)rmrnrorrrrrr$r/rrrr5rprrqrrsr LongTensorrrrrrrkrtrus@r@rrs #( N=/0C )' )'3Z)' )'  )'  )'\7;37/315=A>B,0/3&*n ##n !!2!23n u//0 n u||, n E--. n  ((9(9:n !)):): ;n $D>n 'tnn d^n  uU&&')UU Vn rArzV Class defining the outputs of [`InstructBlipVideoForConditionalGeneration`]. ) custom_introceZdZUdZdZeeeje d<dZ eeeje d<dZ eeje d<dZ eeeje d<dZ eeeje d<deefd Zy) 4InstructBlipVideoForConditionalGenerationModelOutputa loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Language modeling loss from the language model. logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head of the language model. vision_outputs (`BaseModelOutputWithPooling`): Outputs of the vision encoder. qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`): Outputs of the Q-Former (Querying Transformer). language_model_outputs (`CausalLMOutputWithPast` or `Seq2SeqLMOutput`): Outputs of the language model. Nlosslogitsvision_outputsqformer_outputslanguage_model_outputsrEcHtfdjDS)Nc3dK|]'}|dvr|nt|j)yw)rrrN)rto_tuple)rkr>s r@rzPInstructBlipVideoForConditionalGenerationModelOutput.to_tuple..gsB WW Gq!**, - s-0)rkeysrs`r@rz=InstructBlipVideoForConditionalGenerationModelOutput.to_tuplefs% YY[    rA)rmrnrorrrrr5rrrrrrrrrrrAr@rrLs 04D(5**+ ,315FHU5,,- .526NHU../6:>OXeE$5$567>AEHU5+<+<%=>E %* rArz` InstructBlipVideo base Model consisting of language model, qformer and vision encoder. c#eZdZdZdgZdeffd ZdZdZdZ dZ d e jd e jfd Zee dde jd e jd ee jd ee jdee jdee jdee jd ee j$deedeedeededeedeedeeeffdZxZS)rr]rr(c(t||t|j|_t j tjd|j|jj|_ t|j|_t j|jj|j j|_t%j&|j |_|j(j*/|j*j-|j(j*|j(j./|j.j-|j(j.|j1yNr")r.r/r vision_config vision_modelrr4r5rnum_query_tokensqformer_configr0rrqformerr text_configlanguage_projectionr from_configlanguage_modelrextend_keep_in_fp32_modulesrr=s r@r/zInstructBlipVideoModel.__init__xs  89M9MNLLQ8O8OQWQfQfQrQr)st4V5J5JK #%99V-B-B-N-NPVPbPbPnPn#o '33F4F4FG    0 0 <  " " ) )$*=*=*O*O P    4 4 @  & & - -d.A.A.W.W X rAc6|jjSrrrrs r@rz+InstructBlipVideoModel.get_input_embeddings""7799rAc:|jj|yrrrrs r@rz+InstructBlipVideoModel.set_input_embeddings 007rAc|jjs_|jj|jj_|jj|jj _yyrr(use_decoder_only_language_modelrsharedr embed_tokensdecoderrs r@ _tie_weightsz#InstructBlipVideoModel._tie_weightsT{{::7;7J7J7Q7QD   ' ' 47;7J7J7Q7QD   ' ' 4;rAc|j}t|dkDr:d|vr6tjj dkDrt j dt|jdrd|jj_ yyz Some pre-processing hacks to make the model `accelerate` compatible. Check https://github.com/huggingface/transformers/pull/21707 for more details. r"raThe `language_model` is not in the `hf_device_map` dictionary and you are running your script in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`. Please pass a `device_map` that contains `language_model` to remove this warning. Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for more details on creating a `device_map` for large models._hf_hookTN hf_device_maprCr5cuda device_countrwarningrrrio_same_devicer>rs r@_preprocess_acceleratez-InstructBlipVideoModel._preprocess_acceleratey ** }  !&6m&KPUPZPZPgPgPilmPm NNM  4&& 3:>D   ( ( 7 4rArrc|m||jtj|jjtj |j k(}|jd}n||jjk(}|jdj|j|j }|SzY Obtains multimodal placeholdr mask from `input_ids` or `inputs_embeds`. rrG) rr5rr(image_token_idrrall unsqueeze expand_asrbr>rrspecial_image_masks r@get_placeholder_maskz+InstructBlipVideoModel.get_placeholder_mask  !.2M$2K2K2M T[[77uzzR_RfRfg3" "4!7!7!; !*dkk.H.H!H /99"=GG VYYZgZnZno!!rAqformer_input_idsqformer_attention_maskrzdecoder_input_idsdecoder_attention_maskrrrr\ use_cacherrEc | | n|jj} |j\}}}}}|j||z|||}|j || | | | }|d}t j |jddt j|j}|jj|jddd}t j |jddt j|j}|t j|}|j|d}|j|d}t j||gd}|j|||||| | | }|dddd|jdddf}|j!|}|j||jj"|zd}|Q|j$j'|}||jj(k(}|t j|}nl||j't j*|jj(t j|jk(}|j-d}|j/dj1|j3|j}|j3|j|j4}|j7||}|jj8r|j$d ||| | | | d |}n|j$d ||||| | | | d |}t;||| S) a qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided to serve as text prompt, which the Q-Former model will encode. Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for details. [What are input IDs?](../glossary#input-ids) qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be provided to serve as text prompt, which the language model can continue. Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for details. [What are input IDs?](../glossary#input-ids) decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. Only relevant in case an encoder-decoder language model (like T5) is used. N)r]rrrr\rrGrrLr")rrzrrrrrrrrzrrrr)rrzrrrrrrrr)r(rrNrQrr5rrIrrrre ones_likerepeat_interleaverVrrrrrvideo_token_idrrrrrbr`masked_scatterrr)r>r]rrrrzrrrrrrr\rrrfframeschannelrCrDr image_embedsimage_attention_maskrquery_attention_mask query_outputs query_outputlanguage_model_inputsrrs r@rkzInstructBlipVideoModel.forwardsb&1%->q-A+A1(DE !% 8 8 F!6 = =j$++JfJfioJoqs t  FD//DDFyQM!*dkk.H.H!H %!&!;!.2M$2K2K2M T[[77uzzR_RfRfg3" "4!7!7!; /99"=GG VYYZgZnZno 5 8 89M9M}ObOb c%445GI^_ ;; 6 6)d))+-"3%9'# G*d)) +-"3'="3%9'#  GD))#*  rA) NNNNNNNNNFN)rmrnrorrr#r/rrrr r5rrrrrrrrprsrr rrrrkrtrus@r@rros %O+,6&:8R ?( "e.>.> "uO`O` " >B15598<=A04,0/3&*).$(G ''G !,,G !))9)9 : G E--. G !!1!12 G $E$4$45G !))9)9 :G   -G $D>G 'tnG d^G #'G D>G -.G uJJ K!G G rAra InstructBlipVideo Model for generating text given an image and an optional text prompt. The model consists of a vision encoder, Querying Transformer (Q-Former) and a language model. One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the language model continue the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token. c%(eZdZUeed<dZdZdgZdeffd ZdZ dZ dZ d e jfd Zd Zd Zd ZdZ d"dej(dej*deej*deedeef dZdej*dej(fdZee d#dej(dej(deej*deej(deej*deej*deej*deej(deedeedeej*deededeedeed eee ff dZ!ejD d$dej(deej*deej*deej*deej*deej(ded ej*fd Z# d"dej(dej*deej*deedeef d!Z$xZ%S)%rr(r]Trc\t||tj|j|_t jtjd|j|jj|_ tj|j|_t j |jj|j"j|_|j&r t)j*|j"}nt-j*|j"}|j.%|j.j1|j.|j2%|j2j1|j2||_|j7yr)r.r/r _from_configrrrr4r5rrrr0rrrrrrrr rr!rrrrr)r>r(rr?s r@r/z2InstructBlipVideoForConditionalGeneration.__init__Us3  8EEfFZFZ[LLQ8O8OQWQfQfQrQr)st4AA&BWBWX #%99V-B-B-N-NPVPbPbPnPn#o  1 11==f>P>PQN2>>v?Q?QRN  + + 7  " " ) ).*J*J K  / / ;  & & - -n.R.R S, rAc6|jjSrrrs r@rz>InstructBlipVideoForConditionalGeneration.get_input_embeddingsorrAc:|jj|yrrrs r@rz>InstructBlipVideoForConditionalGeneration.set_input_embeddingsrrrAc:|jj|yr)rset_output_embeddings)r>new_embeddingss r@r0z?InstructBlipVideoForConditionalGeneration.set_output_embeddingsus 11.ArArEc6|jjSr)rget_output_embeddingsrs r@r3z?InstructBlipVideoForConditionalGeneration.get_output_embeddingsxs""88::rAc6|jjSr)r get_encoderrs r@r5z5InstructBlipVideoForConditionalGeneration.get_encoder{""..00rAc6|jjSr)r get_decoderrs r@r8z5InstructBlipVideoForConditionalGeneration.get_decoder~r6rAc|jjs_|jj|jj_|jj|jj _yyrrrs r@rz6InstructBlipVideoForConditionalGeneration._tie_weightsrrAc|j}t|dkDr:d|vr6tjj dkDrt j dt|jdrd|jj_ yyrrr s r@r z@InstructBlipVideoForConditionalGeneration._preprocess_accelerater rArrr\rcy)$ Encodes images into continuous embeddings that can be forwarded to the language model. Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. Nr)r>r]rrr\rs r@get_image_featureszInstructBlipVideoForConditionalGeneration.get_placeholder_maskrrArzrrrrlabelsrrc H| | n|jj} |j|||| d\}}}| s|jn|}| s|jn|}||j |}|t j |}|j|j|j}|j||}|j||}|jjre|jd ||| | | |d|}| r |jn|d}d}| w|jd || |jj j"d|}nB|jd ||||| | | | |d |}| r |j$n|d}| r |jn|d }t'||||| S) a qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)): The sequence used as a prompt to be fed to the Q-Former module. qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): Mask to avoid performing attention on padding token indices. Examples: ```python >>> from transformers import InstructBlipVideoProcessor, InstructBlipVideoForConditionalGeneration >>> import torch >>> from huggingface_hub import hf_hub_download >>> import av >>> import numpy as np >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. ... Args: ... container (`av.container.input.InputContainer`): PyAV container. ... indices (`list[int]`): List of frame indices to decode. ... Returns: ... result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3). ... ''' ... frames = [] ... container.seek(0) ... start_index = indices[0] ... end_index = indices[-1] ... for i, frame in enumerate(container.decode(video=0)): ... if i > end_index: ... break ... if i >= start_index and i in indices: ... frames.append(frame) ... return np.stack([x.to_ndarray(format="rgb24") for x in frames]) >>> model = InstructBlipVideoForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b", device_map="auto") >>> processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b") >>> file_path = hf_hub_download( ... repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset" ... ) >>> container = av.open(file_path) >>> # sample uniformly 4 frames from the videWhy is this video funny?o >>> total_frames = container.streams.video[0].frames >>> indices = np.arange(0, total_frames, total_frames / 4).astype(int) >>> clip = read_video_pyav(container, indices) >>> prompt = "What is happening in the video?" >>> inputs = processor(text=prompt, images=clip, return_tensors="pt").to(model.device) >>> outputs = model.generate( ... **inputs, ... do_sample=False, ... num_beams=5, ... max_length=256, ... repetition_penalty=1.5, ... length_penalty=1.0, ... ) >>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip() >>> print(generated_text) "A person is eating a bowl of pasta, and they are using a fork to eat it. The person is sitting at a table, and the plate of pasta is on the table in front" ```NTrrr\rrrr)rr?r) rrzrrrrrr?rr")rrrrrr)r(rget_video_featuresrrr5rrbrr`rr!rrr loss_functionrrrr)r>r]rrrrzrrrrrr?rr\rrr)rr'rrrrs r@rkz1InstructBlipVideoForConditionalGeneration.forwardsf&1%8C ..0  7D557 BM  !"__Y7N 5 8 89M9M}ObOb c!66yP]6^%445GI^_ ;; 6 6)d))+-"3%9'# G(3W^^ FD!)t))!&T[[=T=T=_=_ci *d)) +-"3'="3%9'#  G$/7<r]rrrrzrr\generate_kwargsrfr)rr' video_tokens start_tokensrinputsrs r@rJz2InstructBlipVideoForConditionalGeneration.generateLsD 4 )  ' ' )!''* ?C?V?V /#9%= @W@ <~}   $ = =>A]A]]`aa +t{{/F/F/S/S.TT !LL,uzzR^ReRef %,,Z; 7D557 BM  !"__Y7N 5 8 89M9M}ObOb c!66yP]6^%445GI^_ #0NS""))<<"+F; .$%%..KK?KrAc|j\}}}} } |j||z|| | }|j||d} | d} tj| j ddtj | j} |jj| jddd}tj|j ddtj | j}|tj|}|j|d}|j|d}tj||gd}|j|||| | d }|dddd|j dddf}|j|}|j||jj |zd}|r|| |fS|S) r<T)r]r\rrNrGrrLr")rrzrrrr)rNrQrr5rrIrrrrerrrVrrr(r)r>r]rrr\rrfr"r#rCrDrr$r%rr&r'r(r)s r@rCz->q-A+A1(DE !% 8 8 F!6 = =j$++JfJfioJoqs t (.-G G$$rA)NFF) NNNNNNNNNNFN)NNNNNF)&rmrnror#rrrrr/rrr0rModuler3r5r8rr r5rrrrrsr=rrrrrrrrrkno_gradrJrCrtrus@r@rrEsz $#$O!+,64:8B;ryy;11R ?0>B38&+  '' !++ !))9)9 :  #+4.  d^  " "e.>.> "uO`O` " >B15598<=A59,0/3-1&*).$(N ''N !,,N !))9)9 : N E--. N !!1!12 N $E$4$45N !))9)9 :N   1 12N $D>N 'tnN ))*N d^N #'N D>N +,!N " uJJ K#N N `U]]_9==A045959).C''C$E$4$45C!))9)9 : C E,,- C !!1!12 C  1 12C#'C   CCR>B38&+ 9%''9%!++9%!))9)9 : 9% #+4. 9% d^ 9%rAr)rrrrr)r)Jr dataclassesrtypingrrrrr5r activationsr generationr modeling_flash_attention_utilsr modeling_layersr modeling_outputsrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrrrutilsrrrrrrautorr r!configuration_instructblipvideor#r$r% get_loggerrmrrPr'rpfloatrrrrrrr1r<rLrSrWrqr~rrrrrr__all__rrAr@rbsq, !11 !)B9 G&lljjII   H %G Gd% II% <<% % << % U\\* %  %%.QQh 299 -$>-`L ryyL ^yyx  , ,^ 299  RYY \$>\~C biiC L00f#-#-#-L;#C;|| $D| ~   ;   : N =N  N b}%0PRa}%}%@  rA