rhZddlmZddlmZmZmZddlZddlmZddl m Z ddl m Z ddl mZmZddlmZdd lmZdd lmZdd lmZdd lmZmZdd lmZmZddlmZm Z ddl!m"Z"ddl#m$Z$m%Z%m&Z&m'Z'm(Z(ddl)m*Z*ddl+m,Z,m-Z-ddl.m/Z/e(j`e1Z2ee&dGdde$Z3edGddejhZ5GddejhZ6Gdd ejhZ7d!Z8dFd"Z9d#ejtd$e;d%ejtfd&Z< dGd'ejhd(ejtd)ejtd*ejtd+eejtd,e=d-e=d.e"e%fd/Z>Gd0d1ejhZ?Gd2d3eZ@e&d4e&Gd5d6e ZAe&Gd7d8eAZBGd9d:ejhZCe&d;Gd<d=eAeZDGd>d?ejhZEe&Gd@dAeAZFe&dBGdCdDeAe/ZGgdEZHy)H) dataclass)CallableOptionalUnionN)check_model_inputs)ACT2FN)Cache DynamicCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack) ModelOutputTransformersKwargsauto_docstringcan_return_tuplelogging) AutoModel) CsmConfigCsmDepthDecoderConfig)CsmGenerationMixinz: Base class for the model autoregressive outputs. ) custom_introceZdZUdZdZeejed<dZ ejed<dZ ee e ejed<dZ ee ejdfed<dZ ee ejdfed<dZeejed <dZejed <dZee e ejed <dZee ejdfed <dZee ejdfed <dZeejed<y)CsmOutputWithPasta loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. depth_decoder_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction) of the depth decoder model. depth_decoder_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the depth decoder (scores for each vocabulary token before SoftMax). depth_decoder_past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) depth_decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. depth_decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. backbone_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction) of the backbone model. Nlosslogitspast_key_values. hidden_states attentionsdepth_decoder_lossdepth_decoder_logitsdepth_decoder_past_key_valuesdepth_decoder_hidden_statesdepth_decoder_attentions backbone_loss)__name__ __module__ __qualname____doc__r%rtorch FloatTensor__annotations__r&r'tupler(r)r*r+r,r-r.r/w/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/csm/modeling_csm.pyr$r$1s'<)-D(5$$ %, $FE  $AEOXeE%*;*;$<=>E=AM8E%"3"3S"89:A:>Ju00#567>6:!2!23:.2%++2OS!8E%8I8I2J,K#LSKO%0A0A30F*G!HOHLhuU->->-C'DEL15M8E--.5r9r$RMSNormc,eZdZdfd ZdZdZxZS) CsmRMSNormct|tjt j ||_||_y)z9 CsmRMSNorm is equivalent to T5LayerNorm N)super__init__nn Parameterr4onesweightvariance_epsilon)self hidden_sizeeps __class__s r:r@zCsmRMSNorm.__init__es1 ll5::k#:; #r9c"|j}|jtj}|j dj dd}|tj ||jzz}|j|j|zS)NrT)keepdim) dtypetor4float32powmeanrsqrtrErD)rFr( input_dtypevariances r:forwardzCsmRMSNorm.forwardmsy#)) %((7  $$Q',,R,>% Ht?T?T4T(UU {{]--k:::r9c^t|jjd|jS)Nz, eps=)r7rDshaperErFs r: extra_reprzCsmRMSNorm.extra_reprts* ))*+6$2G2G1HIIr9)gư>)r0r1r2r@rUrY __classcell__rIs@r:r=r=cs$;Jr9r=c^eZdZddeffd Zej edZxZ S)CsmRotaryEmbeddingconfigct|t|drUt|jt r;|jj d|jj d|_nd|_|j|_ |j|_ ||_ t|j|_ |j|j|\}|_|jd|d|j |_y)N rope_scaling rope_typetypedefaultinv_freqF persistent)r?r@hasattr isinstancer`dictgetramax_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr^r rope_init_fnattention_scalingregister_bufferrdoriginal_inv_freq)rFr^devicerdrIs r:r@zCsmRotaryEmbedding.__init__ys  6> *z&:M:Mt/T#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B! /?+/+<+>@GG HZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk ^^ U C 5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C  5 vvAGGv$cff177f&;;;  5 5s BF%%F.N) r0r1r2rr@r4no_gradrrUrZr[s@r:r]r]xs3/y/"U]]_ <