rhl ddlZddlZddlmZmZddlZddlZddlm Z ddl m cm Z ddlm Z ddlmZddlmZddlmZddlmZdd lmZmZmZmZmZmZdd lmZdd lm Z m!Z!m"Z"d d l#m$Z$e"jJe&Z'Gdde jPZ)Gdde jPZ*Gdde jPZ+Gdde jPZ,Gdde jPZ-GddeZ.GddeZ/Gdde jPZ0Gdde jPZ1Gd d!e jPZ2e Gd"d#eZ3Gd$d%eZ4Gd&d'eZ5Gd(d)eZ6Gd*d+e jPZ7Gd,d-e jPZ8Gd.d/e jPZ9 dKd0e:e;e;fd1e;Gd?d@e3ZDe GdAdBe3ZEGdCdDe jPZFGdEdFe jPZGe dG;GdHdIe3ZHgdJZIy)LN)OptionalUnion)CrossEntropyLoss)ACT2FN)is_deepspeed_zero3_enabled)is_fsdp_managed_module)GradientCheckpointingLayer)BaseModelOutputCausalLMOutputSequenceClassifierOutputTokenClassifierOutputWav2Vec2BaseModelOutput XVectorOutput)PreTrainedModel)auto_docstringis_peft_availablelogging) WavLMConfigc$eZdZfdZdZxZS)WavLMSamePadLayercPt||dzdk(rd|_yd|_yNrr)super__init__num_pad_remove)selfnum_conv_pos_embeddings __class__s {/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/wavlm/modeling_wavlm.pyrzWavLMSamePadLayer.__init__&s) #:Q#>!#CacV|jdkDr|ddddd|j f}|SNr)rr hidden_statess r"forwardzWavLMSamePadLayer.forward*s6    ")!Q0F43F3F2F0F*FGMr#__name__ __module__ __qualname__rr( __classcell__r!s@r"rr%sKr#rc$eZdZfdZdZxZS)WavLMPositionalConvEmbeddingct|tj|j|j|j |j dz|j |_tjj}ttjjdr$tjjj}tr(ddl }|jj|jj d5||jdd|_dddt|jdrU|jjj j"}|jjj j$}n,|jj&}|jj(}|jj+|||jj+||n||jdd|_t-|j |_t0|j2|_y#1swY'xYw) Nr) kernel_sizepaddinggroups weight_normr) modifier_rankweight)namedimparametrizations)rrnnConv1d hidden_sizer num_conv_pos_embedding_groupsconvutilsr5hasattrr:r deepspeedzeroGatheredParametersr7 original0 original1weight_gweight_vregister_external_parameterrr3rfeat_extract_activation activation)rconfigr5rBrGrHr!s r"rz%WavLMPositionalConvEmbedding.__init__1s II      6622a777   hh** 288,,m <((33??K % ' 224993C3CST2U I' aH  Ityy"459955<<FF9955<<FF99--99-- NN 6 6tX F NN 6 6tX F#DIIH!DDI()G)GH  !?!?@ I Is IIc|jdd}|j|}|j|}|j|}|jdd}|SNrr) transposer?r3rKr&s r"r(z$WavLMPositionalConvEmbedding.forwardRsV%//15  -0  ]3  6 %//15 r#r)r.s@r"r0r00sABr#r0c$eZdZfdZdZxZS)WavLMFeatureProjectionc4t|tj|jd|j |_tj|jd|j|_ tj|j|_ y)Neps) rrr; LayerNormconv_dimlayer_norm_eps layer_normLinearr= projectionDropoutfeat_proj_dropoutdropoutrrLr!s r"rzWavLMFeatureProjection.__init__^sf ,,vr':@U@UV))FOOB$79K9KLzz&":":; r#cp|j|}|j|}|j|}||fSN)rYr[r^)rr'norm_hidden_statess r"r(zWavLMFeatureProjection.forwardds:!__];(:;  ]3 000r#r)r.s@r"rQrQ]s < 1r#rQceZdZdZ ddedededededef fd Z dd ejd e ejd e ejd ed e eje eje e ejff dZ d ejd eejej fdejd ed ejejff dZdeded ejfdZdejd ejfdZxZS)WavLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paper embed_dim num_headsr^ num_buckets max_distancehas_relative_position_biasct|||_||_||_||z|_|j |z|jk7rt d|jd|d|j dz|_tj|||_ tj|||_ tj|||_ tj|||_ ||_||_tj t#j$d|jdd|_tj|j d|_|r0tj*|j|j|_yy)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gr)rrrerfr^head_dim ValueErrorscalingr;rZk_projv_projq_projout_projrgrh Parametertorchonesgru_rel_pos_constgru_rel_pos_linear Embeddingrel_attn_embed)rrerfr^rgrhrir!s r"rzWavLMAttention.__init__os7 "" !Y. MMI %$.. 8MdnnM]$YKr3 }}d* ii 95 ii 95 ii 95  )Y7 &(!#ejjDNNAq.Q!R"$))DMM1"= %"$,,t/?/?"PD  &r#r'attention_mask position_biasoutput_attentionsreturnc|j\}}}|S|j||}|jdj|dddj ||j z||}|j |j dd|j dfz} | jdddd} |j| } | j | j dddzjd} tj| jdd\} } | | |jzd z zd z} | j ||j zdd|z}|j d||f}|j||||\}}|||fS) z'Attention layer with relative attentionNrrrSrr)rr9?g@)size compute_bias unsqueezerepeatviewrfshapepermuterwsumrtsigmoidchunkrvtorch_multi_head_self_attention)rr'rzr{r|indexbsztgt_len_gated_hidden_statesrelative_position_projgate_agate_b gate_outputgated_position_bias attn_output attn_weightss r"r(zWavLMAttention.forwards(,,.Wa   --gw?M''*11#q!Q?DDS4>>EY[bdkl  ,001D1DSb1IT^^]_L`1`a199!Q1E"&!8!89L!M!7!DDQBDO)?)? ?# EFL *..sT^^/CRKm[166GW7MN$($H$H >+>@Q% ! \L-77r#rcX|jddx}x}}||jdnd}dx} } d} tj||||j|j t jdgt j|jj|jj|jjf| | | |j|jj|jj|j |||d|jj|jj|jj\} } | jdd} | C| dddfj#| j$dd|j fz| j$ddz} | | fS)zCsimple wrapper around torch's multi_head_attention_forward functionrrNFT)use_separate_proj_weight q_proj_weight k_proj_weight v_proj_weight)rOneFmulti_head_attention_forwardrerfrtemptycatrqbiasrorpr^rrr7training broadcast_tor)rr'rzrr|querykeyvaluekey_padding_maskbias_kbias_v add_zero_attnrrs r"rz.WavLMAttention.torch_multi_head_self_attentions,55a;;;e3A3M>,,Q/SW %&$B$B    NN NN KK  IIt{{'')9)94;;;K;KL M    LL MM MM   MM   %)++,,++,,++,,+% ! \2"++Aq1  #(40==""2A&$..)::\=O=OPQPR=SSLL((r# query_length key_lengthctj|tjdddf}tj|tjdddf}||z }|j|}|j |j j j}|j |}|jgd}|S)Ndtype)rrr) rtarangelong_relative_positions_buckettoryr7devicer)rrrcontext_positionmemory_positionrelative_positionrelative_position_bucketvaluess r"rzWavLMAttention.compute_biass << EJJG4P,,zDT1WM+.>>#'#B#BCT#U #;#>#>t?R?R?Y?Y?`?`#a $$%=> * r#relative_positionsc$|jdz}|dkDjtj|z}tj|}|dz}||k}tj |j |z }|tj |j|z z }|||z z}||zjtj}tj|tj||dz }|tj|||z }|Sr) rgrrtrabslogfloatmathrhmin full_likewhere)rrrgrelative_buckets max_exactis_smallrelative_positions_if_largerelative_position_if_larges r"rz)WavLMAttention._relative_positions_buckets &&!+ .266uzzB[P"YY'9:1$ % 1&+ii0B0H0H0JY0V&W#&ADHHTM^M^ajMjDk&k#&A[S\E\&]#&/2M&M%Q%QRWR\R\%]"%*YY &8RT_bcTc(d& " EKK2DF`aar#)i@i TNNFr)r*r+r,__doc__intrboolrrtTensorrtupler( FloatTensorr LongTensor BoolTensorrrrr-r.s@r"rdrdlsG +/"Q"Q"Q "Q  "Q  "Q%)"QN2604"' '8||'8!.'8  - '8  '8 u||Xell3XeELL>Q5RR S'8R5)((5)e..0@0@@A5)#.. 5)  5)   U.. / 5)n#%BSBS U=N=N SXSdSd r#rdc$eZdZfdZdZxZS)WavLMFeedForwardct|tj|j|_tj |j|j|_ t|jtrt|j|_n|j|_tj |j|j|_tj|j |_yra)rrr;r\activation_dropoutintermediate_dropoutrZr=intermediate_sizeintermediate_dense isinstance hidden_actstrrintermediate_act_fn output_densehidden_dropoutoutput_dropoutr_s r"rzWavLMFeedForward.__init__s $&JJv/H/H$I!"$))F,>,>@X@X"Y f'' -'-f.?.?'@D $'-'8'8D $IIf&>&>@R@RS jj)>)>?r#c|j|}|j|}|j|}|j|}|j |}|Sra)rrrrrr&s r"r(zWavLMFeedForward.forward sX// > 00? 11-@ ))-8 ++M: r#r)r.s@r"rrs @r#rc2eZdZddedeffd ZddZxZS)WavLMEncoderLayerrLrict|t|j|j|j |j |j||_tj|j|_ tj|j|j|_t!||_tj|j|j|_yN)rerfr^rgrhrirTrrrdr=num_attention_headsattention_dropoutrgmax_bucket_distance attentionr;r\rr^rVrXrYr feed_forwardfinal_layer_normrrLrir!s r"rzWavLMEncoderLayer.__init__+ '((00,,**33'A  zz&"7"78 ,,v'9'9v?T?TU,V4 " V-?-?VEZEZ [r#c|}|j|||||\}}}|j|}||z}|j|}||j|z}|j |}||f}|r||fz }|S)Nrzr{r|r)rr^rYrr) rr'rzr{r|r attn_residualroutputss r"r(zWavLMEncoderLayer.forward:s% 59^^ )'/ 6D6 2 |] ]3 % 5  6 %(9(9-(HH --m<  -0    &Gr#Trr*r+r,rrrr(r-r.s@r"rr*s \{ \ \r#rc2eZdZddedeffd ZddZxZS) WavLMEncoderLayerStableLayerNormrLrict|t|j|j|j |j |j||_tj|j|_ tj|j|j|_t!||_tj|j|j|_yrrrs r"rz)WavLMEncoderLayerStableLayerNorm.__init__Trr#c|}|j|}|j||||\}}}|j|}||z}||j|j |z}||f}|r||fz }|S)N)rzr{r|)rYrr^rr)rr'rzr{r|rrrs r"r(z(WavLMEncoderLayerStableLayerNorm.forwardcs%  6 59^^ )'/ 6D6 2 |]  ]3 % 5 %(9(9$:O:OP]:^(__  -0    &Gr#r)NNFrr.s@r"rrSs \{ \ \r#rc.eZdZfdZ ddZxZS) WavLMEncoderc t|||_t||_t j |j|j|_ t j|j|_ t jt|jDcgc]}t!||dk(c}|_d|_ycc}wNrTr)riF)rrrLr0pos_conv_embedr;rVr=rXrYr\rr^ ModuleListrangenum_hidden_layersrlayersgradient_checkpointingrrLir!s r"rzWavLMEncoder.__init__ys  :6B,,v'9'9v?T?TUzz&"7"78 mmUZ[a[s[sUt uPQ v16 K u  ',# v!C c|rdnd}|rdnd}|5|jdjdd|jd}d||<|j|} || z}|j |}|j |}t xs t|} d} t|jD]y\} } |r||fz}tjg}|jxr | dkDxr||jjk}|r| r| ||| || }|dd\}} |rd}|sq|dfz}{|r||fz}|std|||fDSt!||| S) NrSrrrrNNNc3&K|] }|| ywrar.0vs r" z'WavLMEncoder.forward..mq_`_lmlast_hidden_stater' attentions)rrrrrYr^rr enumeraterrtrandrrL layerdroprr rr'rzr|output_hidden_states return_dictall_hidden_statesall_self_attentionsexpand_attention_maskposition_embeddings synced_gpusr{rlayerdropout_probabilityskip_the_layer layer_outputss r"r(zWavLMEncoder.forwards#7BD$5b4  %$2$<$.rrr)rrrrr^rr rrrtrrrLrrYrr rs r"r(z#WavLMEncoderStableLayerNorm.forwards#7BD$5b4  %$2$<$rS)meanrtexprr)probsmarginal_probs perplexitys r"_compute_perplexityz.WavLMGumbelVectorQuantizer._compute_perplexity(sR*YY .599^VZEZ;[*[ac ddeiik r#c|j\}}}|j|}|j||z|jzd}|jrt j j|j|jd}|j|}tj|j||z|jdjd}|j|}n}|jd}|j|jj!d|jddd}|j||z|jd}|j|}|j||zd}|j#d|j$z} | j||z|j|j&d} | j)dj||d} | |fS)NrST)tauhardrrr)rr2rr-rr; functionalgumbel_softmaxrr3type_asrtsoftmaxr:argmax new_zerosscatter_rr1r/r) rr' batch_sizesequence_lengthr=codevector_probscodevector_soft_distr9codevector_idxcodevectors_per_groupr1s r"r(z"WavLMGumbelVectorQuantizer.forward.s3@3F3F0 O[((7 %**:+G$//+Y[]^ ==!}};;MAQAQ Q+00o1Mt`d`m`moqr !oob)..z?BO J&&r#) r*r+r,rr staticmethodr:r(r-r.s@r"r*r* s& * "'r#r*ceZdZUeed<dZdZdZdZdZ dZ dZ dde e jefd eefd Z dd ed e jfd Zy)WavLMPreTrainedModelrLwavlm input_valuesTFc zt|tr|jjjj dd|jj jjtjj|jyt|trtjj |jjddtjd|jj d|jj"zz ztjj%|jj dyt|t&rtjd|j(j*z }tjj|j(j| |tjj|j(j | |yt|tj,rm|jjj d|j.j0|j %|j jjyyt|tj2tj4frJ|j jj|jjj7dyt|tj8rtjj;|j|j jtj|j<|j"|j dzz }tjj|j | |yyy) zInitialize the weightsrr)r5stdrr)abNr)rr*r2r7datanormal_rzero_r;inituniform_r1r0r?rsqrtr2 in_channels constant_rQr[ in_featuresrZrLinitializer_rangerV GroupNormfill_r<kaiming_normal_r4)rmoduleks r" _init_weightsz"WavLMPreTrainedModel._init_weights]s f8 9    % % * * 2 2 2 C    # # ( ( . . 0 GG  V// 0  < = GGOO "" !v{{'>'>q'AFKKD[D['["\]]   GG  fkk.. 2  6 7 !f//;;;tj||z |ddzS)Nfloor) rounding_moder)rtdiv input_lengthr2strides r"_conv_out_lengthzOWavLMPreTrainedModel._get_feat_extract_output_lengths.._conv_out_lengths"99\K7wWZ[[ [r#r)rLrfzip conv_kernel conv_stridernum_adapter_layersadapter_stride)rrerfror2rnrs r" _get_feat_extract_output_lengthsz5WavLMPreTrainedModel._get_feat_extract_output_lengths~s2=1Ddkk--+  \ $'t{{'>'> @W@W#X Q K,]KPM Q 4;;99: _ 04;;C]C] ^  _r#feature_vector_lengthrzc|jddddf}|j||}|jtj}|j d}tj ||f|j|j}d|tj|j d|j|dz f<|jdgjdjdgj}|S)NrSrrfr)rrr)r) cumsumrurrtrrzerosrrrflipr)rrvrzrfnon_padded_lengthsoutput_lengthsrFs r""_get_feature_vector_attention_maskz7WavLMPreTrainedModel._get_feature_vector_attention_masks ,22r2:1b5A>>?Q_j>k'**5::6#))!,  . /~7K7KTbTiTi uv ^%9%9!%<^EZEZ[]kno]opq',,bT299"=BBB4HMMOr#ra)r*r+r,r__annotations__base_model_prefixmain_input_namesupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attnrdrrtrrrrrur~rr#r"rNrNSs $O&*# N9DZ^"5#3#3S#89HPQU0Y]%(:?:J:Jr#rNc&eZdZdfd ZdZxZS)WavLMNoLayerNormConvLayercdt||dkDr|j|dz nd|_|j||_t j |j|j|j||j||j|_ t|j|_ y)Nrrr2rnr)rrrW in_conv_dim out_conv_dimr;r<rqrr conv_biasr?rrJrKrrLlayer_idr!s r"rz"WavLMNoLayerNormConvLayer.__init__s rS)r?rOrYrKr&s r"r(zWavLMLayerNormConvLayer.forwardsV -0 %//B7  6 %//B7  6 r#rr)r.s@r"rrs Ar#rc&eZdZdfd ZdZxZS)WavLMGroupNormConvLayerct||dkDr|j|dz nd|_|j||_t j |j|j|j||j||j|_ t|j|_ t j|j|jd|_y)NrrrT)r- num_channelsaffine)rrrWrrr;r<rqrrrr?rrJrKr_rYrs r"rz WavLMGroupNormConvLayer.__init__s .5s#h!$5f$=#hs)rrrr=r;rZprojrVproj_layer_normrrrsrrr_s `r"rzWavLMAdapter.__init__+s   $ $(:(: : &"4"4f6O6OPDI#%<<0I0I#JD /3 3DI,mm#huVMfMfGg#hh ))r#ch|j.|j"|j|}|j|}|jdd}|jD]D}tj j }|j r||jkDs=||}F|jdd}|SrN)rrrOrnprandomrr)rr'rlayerdrop_probs r"r(zWavLMAdapter.forward8s 99 T%9%9%E IIm4M 00?M%//15 [[ 5EYY--/N==^dnn%D %m 4  5 &//15 r#r)r.s@r"rr*s  *r#rr mask_prob mask_lengthrz min_masksr}c|\}dkr tdkDrtdddtjjdj fd}|-|j j djnt|Dcgc]}c}}tj|ft } g} |} | d k(r| S|D]} || } tjjtj| dz z | d }t|d k(rdz }n|d }tj|tj| | z tj |zg}| j#|tj$| } tj&| dddddf|| f} | j)|| z} tjddddf}tj&||| fj)|| z}| |z} | j+dz kDrdz | | dz kD<tj,| | dd| Scc}w) an Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `ct|zz z}t|}|zkDrz}|dz z |krt|dz z d}|S)z;Given input length, compute how many spans should be maskedrr)rmax)rmnum_masked_spanepsilonrrrrGs r"compute_num_masked_spanz6_compute_mask_indices..compute_num_masked_spanosoi,6DwNOoy9 [ (? :-DL'/DL/5/A/A|F+t  r#cXtjdt|jyz Calling this function will disable the gradient computation for the feature encoder so that its parameters will not be updated during training. The method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5. Please use the equivalent `freeze_feature_encoder` method instead.Nwarningswarn FutureWarningfreeze_feature_encoderrs r"freeze_feature_extractorz#WavLMModel.freeze_feature_extractor'   Q  ##%r#c8|jjy Calling this function will disable the gradient computation for the feature encoder so that its parameter will not be updated during training. N)rrrs r"rz!WavLMModel.freeze_feature_encoders 113r#r'mask_time_indicesrzct|jdds|S|j\}}}|)|jj |j ||<n|jj dkDr|jrt||f|jj |jj||jj}tj||jtj}|jj |j ||<|jjdkDr|jrt||f|jj|jj |jj"}tj||jtj}|dddfj%d|d}d||<|S) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://huggingface.co/papers/1904.08779). apply_spec_augmentTNr)rrrzr)rr)rrrrS)getattrrLrrrrrrrmask_time_lengthmask_time_min_masksrttensorrrrmask_feature_lengthmask_feature_min_masksexpand)rr'rrzrFrGr=mask_feature_indicess r"_mask_hidden_stateszWavLMModel._mask_hidden_statesst{{$8$? 4A3E3E3G0 O[  (/3/E/E/H/HI\I\/]M+ , [[ ' '! + 5_-++44 KK88-++99 ! !& ->}G[G[chcmcm n /3/E/E/H/HI\I\/]M+ , ;; ( (1 ,#8[)++77 KK;;++<< $ $)<<0D]MaMainisis#t #74#@#G#GO]_#` 23M. /r#rPr|rrr}cH||n|jj}||n|jj}||n|jj}|j |}|j dd}|!|j |jd|d}|j|\}}|j|||}|j|||||} | d}|j|j|}|s ||f| ddzSt||| j| j S) a/ mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict masked extracted features in *config.proj_codevector_dim* space. NrrFrx)rrzrzr|rrr)rextract_featuresr'r)rLr|ruse_return_dictrrOr~rrrrrWavLMBaseModelOutputr'r) rrPrzrr|rrrr'encoder_outputss r"r(zWavLMModel.forwardsb2C1N-TXT_T_TqTq$8$D $++JjJj &1%uEN+/*A*ABR*S' '00 ->~1 ,, )/!5# ' (* << # LL7M!#34qr7JJ J#+-)77&11   r#)NNNNNNN)r*r+r,rrrrrtrrrrrrrrrrr(r-r.s@r"rrs{( &4:>59 ,((,$E$5$56,!!1!12 ,\269=,0/3&*7 u||,7 !.7 $E$5$56 7 $D> 7 'tn 7 d^7  u** +7 7 r#rrzm WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). ) custom_introceZdZddeeffd ZdZdZdZdZ e ddee jdee jd ee d ee d ee d ee jd eeeffdZxZS) WavLMForCTC target_langct||t||_t j |j |_||_|jtd|jdt|dr|jr |jn |j}t j ||j|_|j%y)a/ target_lang (`str`, *optional*): Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or adapter..bin. Only relevant when using an instance of [`WavLMForCTC`] with adapters. Uses 'eng' by default. NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `WavLMForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.rf)rrrrOr;r\ final_dropoutr^r  vocab_sizermr!rArfrr=rZlm_headr)rrLr rr!s r"rzWavLMForCTC.__init__^s  ' zz&"6"67 &    $00@AHH *1)GFL^L^F % %djdvdv yy!3V5F5FG  r#c|j}|&t|jddtd|d|-t|jddtj dy||j |dyy)a' This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when passing `target_lang=...` to `from_pretrained(...)`. This method is **not** supposed to be called by the user and is prone to be changed in the future. Nadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T) force_load)r rrLrmloggerinfo load_adapter)rr s r" tie_weightszWavLMForCTC.tie_weights{s&&  "wt{{ D 'tn D d^ D &D  un$ %D D r#r z WavLM Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. ceZdZfdZdZdZdZe d dee jdee jdee dee d ee d ee jd e e effd ZxZS)WavLMForSequenceClassificationct||t|dr|jr t dt ||_|jdz}|jr0tjtj||z |_ tj|j|j |_tj|j |j$|_|j)y)Nrfz\Sequence classification does not support the use of WavLM adapters (config.add_adapter=True)r)rrrArfrmrrOruse_weighted_layer_sumr;rsrtru layer_weightsrZr=classifier_proj_size projector num_labels classifierrrrL num_layersr!s r"rz'WavLMForSequenceClassification.__init__s  6= )f.@.@n  ' --1  ( (!#ejj.Dz.Q!RD 6#5#5v7R7RS))F$?$?ARARS r#cXtjdt|jyrrrs r"rz7WavLMForSequenceClassification.freeze_feature_extractor rr#cL|jjjyrrrs r"rz5WavLMForSequenceClassification.freeze_feature_encoderrr#cP|jjD] }d|_ yrrrs r"rz0WavLMForSequenceClassification.freeze_base_modelrr#rPrzr|rrr r}c<||n|jj}|jjrdn|}|j|||||}|jjrr|t}t j |d}tjj|jd} || jdddzjd}n|d}|j|}||jd} n|j|j d|} | j#dj%dd|j d} d || <|jd| jdjddz } |j'| } d}|Ft)}|| jd|jj*|jd}|s| f|tdz}||f|zS|St-|| |j.|j0 S)  input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`WavLMProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NTrrrrSrrrr&)rLrr=rOr4rtstackr;r?rBr>rrr@r5r~rrrrBrrAr r'r)rrPrzr|rrr rr' norm_weights pooled_output padding_maskexpand_padding_maskr(r'loss_fctr9s r"r(z&WavLMForSequenceClassification.forward&s.&1%->r1a-HHMMRSMTM#AJM}5  !)..1.5MBB=CVCVWXCY[ijL"."8"8"<"C"CAq-J]J]^_J`"a 25M.. /)--!-4|7G7GA7G7N7S7STVXY7ZZM/  ')HFKKDKK,B,BCV[[QS_UDY)F)G!HHF)-)9TGf$ Ev E'!//))   r#r)r*r+r,rrrrrrrtrrrrr r(r-r.s@r"r;r;s" &:(26,0/3&*)-B u||,B !.B $D> B 'tn B d^ B &B  u.. /B B r#r;ceZdZfdZdZdZdZe d dee jdee jdee jdee d ee d ee d e e effd ZxZS) WavLMForAudioFrameClassificationct||t|dr|jr t dt ||_|jdz}|jr0tjtj||z |_ tj|j|j |_|j |_|j%y)Nrfz_Audio frame classification does not support the use of WavLM adapters (config.add_adapter=True)r)rrrArfrmrrOrr=r;rsrtrur>rZr=rArB init_weightsrCs r"rz)WavLMForAudioFrameClassification.__init__ns  6= )f.@.@q  ' --1  ( (!#ejj.Dz.Q!RD ))F$6$68I8IJ ++ r#cXtjdt|jyrrrs r"rz9WavLMForAudioFrameClassification.freeze_feature_extractor~rr#cL|jjjyrrrs r"rz7WavLMForAudioFrameClassification.freeze_feature_encoderrr#cP|jjD] }d|_ yrrrs r"rz2WavLMForAudioFrameClassification.freeze_base_modelrr#rPrzr r|rrr}c ||n|jj}|jjrdn|}|j|||||}|jjrr|t}t j |d}tjj|jd} || jdddzjd}n|d}|j|} d} |\t} | | jd|jt j |jd|jd} |s| f|tdz} | St#| | |j$|j& S) rINTrrrrSr)axisr&)rLrr=rOr4rtrJr;r?rBr>rrrBrrArCrr'r)rrPrzr r|rrrr'rKr(r'rOr9s r"r(z(WavLMForAudioFrameClassification.forwardsh.&1%->r1a-HHMMRSMTM#AJM/  ')HFKKDOOtuDY)F)G!HHFM$!//))   r#r)r*r+r,rrrrrrrtrrrrrr(r-r.s@r"rQrQls &:(26)-,0/3&*9 u||,9 !.9 & 9 $D> 9 'tn 9 d^9  u++ ,9 9 r#rQc&eZdZdfd ZdZxZS) AMSoftmaxLossct|||_||_||_t j tj||d|_ t j|_ y)NT)r) rrscalemarginrAr;rsrtrandnr7rr')r input_dimrAr\r]r!s r"rzAMSoftmaxLoss.__init__sQ   $ll5;;y*#EUYZ '') r#c|j}tjj|jd}tjj|d}t j ||}||jz }tjj||j}|jt j|j||z}|j||}|S)Nrrr)flattenr;r? normalizer7rtmmr]one_hotrAr\rrr') rr'r r7 cos_thetapsionehotr(r's r"r(zAMSoftmaxLoss.forwards!((!(< // 1/E HH]F3 $++%&&vt?ekk&++-iHHyy( r#)g>@g?r)r.s@r"rZrZs * r#rZcXeZdZdfd Zdej dej fdZxZS) TDNNLayerct||dkDr|j|dz n|j||_|j||_|j ||_|j||_tj|j|j z|j|_ tj|_ y)Nrr)rrtdnn_dimrr tdnn_kernelr2 tdnn_dilationdilationr;rZkernelReLUrKrs r"rzTDNNLayer.__init__s rZr=rkr@rrrirtdnnxvector_output_dimrrBrZrA objectiverS)rrLrDr tdnn_layersr!s r"rzWavLMForXVector.__init__s  ' --1  ( (!#ejj.Dz.Q!RD 6#5#5vq7IJ5:3v;O5PQy+Q QMM+. !#6??2+>+BFD]D]!^))F$=$=v?X?XY&v'@'@&BSBST Rs>FcXtjdt|jyrrrs r"rz(WavLMForXVector.freeze_feature_extractor&rr#cL|jjjyrrrs r"rz&WavLMForXVector.freeze_feature_encoder2rr#cP|jjD] }d|_ yrrrs r"rz!WavLMForXVector.freeze_base_model9rr#recVd}|jjD] }|||d}|S)z? Computes the output length of the TDNN layers c||z |zdzS)Nrrrls r"rozBWavLMForXVector._get_tdnn_output_lengths.._conv_out_lengthFs!;.69A= =r#r)rLrl)rreror2s r"_get_tdnn_output_lengthsz(WavLMForXVector._get_tdnn_output_lengthsAs:  >  ;;22 LK,]KKM Lr#rPrzr|rrr r}c||n|jj}|jjrdn|}|j|||||}|jjrr|t}t j |d}tjj|jd} || jdddzjd}n|d}|j|}|jD] } | |} |%|jd} |j!d} n|j#|jd} |j%| }g} g} t'|D]U\}}| j)||d|fjd| j)||d|fj!dWt j | } t j | } t j*| | gd}|j-|}|j/|}d}||j1||}|s||f|tdz}||f|zS|St3||||j4|j6S) rINTrrrrSr)r'r( embeddingsr'r)rLrr=rOr4rtrJr;r?rBr>rrr@rxr5rRrurrrrrrBrzrr'r)rrPrzr|rrr rr'rK tdnn_layer mean_features std_featuresfeat_extract_output_lengthstdnn_output_lengthsrlengthstatistic_poolingoutput_embeddingsr(r'r9s r"r(zWavLMForXVector.forwardPs.&1%->r1a-HHMMRSMTM#AJM}5 )) 6J&}5M 6  !)..1.5M(,,,3L*.*O*OP^PbPbghPbPi*j '"&"?"?@["\ ML&':; J 6$$]1gvg:%>%C%C%C%JK##M!WfW*$=$A$Aa$A$HI J"KK 6M ;;|4L!II}l&CL 223DE!23  >>&&1D/07;X;Y3ZZF)-)9TGf$ Ev E(!//))   r#r)r*r+r,rrrrrrtrrrrrrrrrr(r-r.s@r"rvrv s & &:( eE O 'tn O d^ O &O  um# $O O r#rv)rQr r;rvrrNr%)Jrrtypingrrnumpyrrttorch.nnr;torch.nn.functionalr?rr activationsrintegrations.deepspeedrintegrations.fsdpr modeling_layersr modeling_outputsr r r rrrmodeling_utilsrr@rrrconfiguration_wavlmr get_loggerr*rModulerr0rQrdrrrrr%r*rNrrrrrrrrrrndarrayrrrr4r r;rQrZrirv__all__rr#r"rs " %!@79.??,   H % *299*Z 1RYY 1c RYYc Lryy0&2&R"'A"JG 299G TH "))H VC'C'LS?SSl :*8680#"))#L $299F26 t c?tttU--. t  t ZZ tn/N %N N b!" S &S  S l p %9p  p ff ';f f RBII. @ N *N  N b r#