rhO ddlZddlmZmZddlZddlmZddlm Z m Z m Z ddl m Z ddlmZddlmZe rddlZeeZd d d d d ddde de dide didZeedj/ZGddeZGddZGddeZGddeZGddeZGdd eZGd!d"eZGd#d$eZ Gd%d&eZ!Gd'd(eZ"eeeeeee e!e"e"d) Z#d*Z$ d0d+ee%d,ee&d-e%fd.Z'd1d/Z(y)2N) NamedTupleOptional)tqdm)GGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPING_gguf_parse_value)is_torch_available)is_gguf_available) get_loggerversion tensor_countkv_count)r rr file_typequantization_version)rr)GGUFgeneral tokenizertokenizer_config)ignoreconfigrrrc@eZdZUejed<eed<eed<y) GGUFTensorweightsnamemetadataN)__name__ __module__ __qualname__npndarray__annotations__strdict{/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/modeling_gguf_pytorch_utils.pyrr7s ZZ INr&rceZdZddZdZy)TensorProcessorNc|xsi|_yNr)selfrs r'__init__zTensorProcessor.__init__>s l r&c t||iSr+rr-rrkwargss r'processzTensorProcessor.processAs'4,,r&r+)rrrr.r3r%r&r'r)r)=s #-r&r)c peZdZdfd ZdZ ddej dedeedej fdZ xZ S) LlamaTensorProcessorc&t||yNr,superr.r-r __class__s r'r.zLlamaTensorProcessor.__init__F 'r&c d|vsd|vrx|jjd}|jjd}d||fvr t||iSd|vr|j|||}nd|vr|j|||}t||iS)Nz.attn_k.z.attn_q.num_attention_headsnum_key_value_heads)rgetr_reverse_permute_weights)r-rrr2 num_heads num_kv_headss r'r3zLlamaTensorProcessor.processIs  t!3 (=>I;;??+@AL <00!'444T!77IVt#77LY'4,,r&rn_headrCreturnc|||k7r|}|jd|zdz}|j||dg|jdd}|jddj|jS)Nrr)shapereshapeswapaxes)r-rrDrCdimws r'rAz-LlamaTensorProcessor._reverse_permute_weightsVsr  #,(>!FmmA&(A- GOOFC ?W]]12-> ?zz!Q'' 66r&r+) rrrr.r3r r!intrrA __classcell__r;s@r'r5r5EsE( -OS 7zz 7+. 7>Fsm 7  7r&r5c\eZdZdfd ZdZdej deeefdedefdZ xZ S) Qwen2MoeTensorProcessorc&t||yr7r8r:s r'r.z Qwen2MoeTensorProcessor.__init__dr<r&c d|vrE|jd}|jd}|r!|j||||t|diSd|vrtj|d}t||iS)N_exptensor_key_mappingparsed_parametersffn_gate_inp_shexpraxis)r@_split_moe_expert_tensorrr expand_dims)r-rrr2rUrVs r'r3zQwen2MoeTensorProcessor.processgsz T>!',@!A  & +> ? !--g7H$Pbc!'444 4 'nnW15G'4,,r&rrVrrUc||}|jjdd}td|D]K}|jdd|d}||}t j t j||d|<My)N num_experts<r mlp.experts..tensors)rr@rangereplacetorch from_numpyr copy) r-rrVrrU w_counteri temp_name exp_weights r'rZz0Qwen2MoeTensorProcessor._split_moe_expert_tensorts "$'KKOOM26 q)$ \A ^|A3a5HII J6;6F6FrwwzGZ6[ i ( 3 \r&r+) rrrr.r3r r!r$r#rZrNrOs@r'rQrQcsC( - \zz \6:39o \MP \fj \r&rQcveZdZdfd ZdZdej dedefdZdej dedefdZ xZ S) BloomTensorProcessorc&t||yr7r8r:s r'r.zBloomTensorProcessor.__init__r<r&c d|vrI|jd}|jd}d|vr|j|||}n|j|||}t||iS)Nattn_qkvrD hidden_sizeweight)r_reverse_reshape_weights_reverse_reshape_biasr)r-rrr2rBn_embeds r'r3zBloomTensorProcessor.processse   H-Ikk-0G477GT44WiQ'4,,r&rrDrtc(tj|dd\}}}|j|||z|}|j|||z|}|j|||z|}tj|||gd}|j|dz||zz|S)NrrXr)r array_splitrIstack)r-rrDrtqkv qkv_weightss r'rrz-BloomTensorProcessor._reverse_reshape_weightss..!!41a IIfg/ 9 IIfg/ 9 IIfg/ 9hh1ayq1 ""6A:F1B#CWMMr&ctj|d\}}}|j|||z}|j|||z}|j|||z}tj|||gdj }|S)NrvrrX)r rwrIrxflatten)r-rrDrtq_biask_biasv_biasqkv_biass r'rsz*BloomTensorProcessor._reverse_reshape_biass"$!;6(9:6(9:6(9:88VVV41=EEGr&r+) rrrr.r3r r!rMrrrsrNrOs@r'rlrlsN(- N NC NRU N RZZ  s r&rlc&eZdZdfd ZdZxZS)T5TensorProcessorc&t||yr7r8r:s r'r.zT5TensorProcessor.__init__r<r&c d}|jdD]}|jst|}nt||d|iS)Nr`bid)splitisdigitrMr)r-rrr2rchunks r'r3zT5TensorProcessor.processsHZZ_ E}}%j '4%66r&r+rrrr.r3rNrOs@r'rrs (7r&rc&eZdZdfd ZdZxZS)GPT2TensorProcessorc&t||yr7r8r:s r'r.zGPT2TensorProcessor.__init__r<r&c d|vs d|vsd|vsd|vr |j}|dk(rDd}|jdi}tjt j ||d|<d}t ||iS) Nzattn_qkv.weightzffn_down.weightz ffn_up.weightzattn_output.weight output.weightzlm_head.weightrVra)Tr@rdrer rfr)r-rrr2rVs r'r3zGPT2TensorProcessor.processs  % D($&#t+iiG ? "$D & +> C 161A1A"'''BR1S i ( .D'4,,r&r+rrOs@r'rrs (-r&rc&eZdZdfd ZdZxZS)MambaTensorProcessorc&t||yr7r8r:s r'r.zMambaTensorProcessor.__init__r<r&c d|vrtj|d}d|vrtj| }t||iS)Nzssm_conv1d.weightrrXssm_a)r r[logrr1s r'r3zMambaTensorProcessor.processsD $ &nnW15G d?ffgX&G'4,,r&r+rrOs@r'rrs ( -r&rc&eZdZdfd ZdZxZS)NemotronTensorProcessorc&t||yr7r8r:s r'r.z NemotronTensorProcessor.__init__r<r&c .d|vr|dz }t||iSNz norm.weightrr0r1s r'r3zNemotronTensorProcessor.process" D kG'4,,r&r+rrOs@r'rrs (-r&rc&eZdZdfd ZdZxZS)Gemma2TensorProcessorc&t||yr7r8r:s r'r.zGemma2TensorProcessor.__init__r<r&c .d|vr|dz }t||iSrr0r1s r'r3zGemma2TensorProcessor.processrr&r+rrOs@r'rrs ( -r&r) llamaqwen2moebloomt5 t5encodergpt2mambanemotrongemma2gemma3c||jvrgS|j|}|jDcgc]%}t|j||j'c}Scc}wr+)fieldsdatar partstypes)readerfieldvalue _data_indexs r' read_fieldrsP FMM! MM% EX]XbXb c ekk+6 D cc cs*A model_type num_layers qual_namec trtr ddlm}m}n t j dtd||jjn|}||jjn|}|dk(rd}n|dk(rd }n|d k(rd }d}|jD]\}}||k(s |}n|td |d |||} i} |j} | D]} |d k(rd| vrtjdd| } | d}} | j!ds| j!dr| j#dd\} }d|z}| j%| }|w|| z| ||z<|j'x}rW|D]R\} }t)||||| d}|jDcic] \}}|| vs ||}}}| j+|T| Scc}}w)aY GGUF uses this naming convention for their tensors from HF checkpoint: `blk.N.BB.weight` and `blk.N.BB.bias` where N signifies the block number of a layer, and BB signifies the attention/mlp layer components. See "Standardized tensor names" in https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details. r)MODEL_ARCH_NAMESget_tensor_name_mapLoading a GGUF checkpoint in PyTorch, requires both PyTorch and GGUF>=0.10.0 to be installed. Please see https://pytorch.org/ and https://github.com/ggerganov/llama.cpp/tree/master/gguf-py for installation instructions.KPlease install torch and gguf>=0.10.0 to load a GGUF checkpoint in PyTorch.Ncoherez command-r qwen2_moer gemma3_textrzUnknown gguf model_type: z in gguf-py. This might because you're using an outdated version of gguf-py package, you can install `gguf` package from source refer to https://github.com/ggerganov/llama.cpp/tree/master/gguf-py#developmentr_zmlp.experts.\d+.z.weightz.biasr`r)r)r r ggufrrloggererror ImportErrorrrnum_hidden_layersitemsNotImplementedError state_dictresubendswithrsplitget_namenamed_childrenget_gguf_hf_weights_mapupdate)hf_modelrrrrrarchkeyrname_mapgguf_to_hf_name_maprhf_namersuffix gguf_namerchildsub_maprzr{s r'rr s513>>  A ghh/9/A++zJ6@6H22jJX { " } $ D&,,. U J D  |!' |4U U  #44H$$&JF  #'(Aff0.'JGf   I &'*:*:7*C">>#q1LD&6\F%%d+   2;g2EI./F$"0022~2) 0KD%-eZZcYdeidjjkWlmG(/ X11DW;Wq!tXGX  & &w /  0 Ys  G ,G c\#$trtr ddlm}m}n t j dtd||}|j}t|j}tDcic]}|i} }t|dd} t|d} d} d| vrd | vrd } n"d | vsd | vrd | d d<d | vr dg| d d<d } n| } d| vrd} d| vrShd#d$t#fd|jD} t$fd|jD}| | d d<| | d d<| tvr| tvrt!d| dddg}t#d|jDxs| |v| d d<|jj%D]-\}}|j'| | }|j)d }|d}d j+|d!d}|j,Dcgc]%}t/|j0||j2'}}t5|d!k(r|d}t7|t8r| |vr|j'| | }tj%D]@\}}||vs |||vs|||}|d"k(r!||| ||<||vs0|j;|B||vst j=d#|d$|0| d d%d&k(rd'| d d%<d(| d vr3| d)}d*|vrt5|d*| d d(<nt j?d+|ri| d,<tA|}| jCd i}tDjC| tF}||-}tI|jd./D]}|jJ} ||j,|jL}!|jO|!| || 0}"|"jP}!|"jJ} | |vr^|| } tSjTtWjX|!| d,| <t5|dkDrt j=d1|| Scc}wcc}w)2a Load a GGUF file and return a dictionary of parsed parameters containing tensors, the parsed tokenizer and config attributes. Args: gguf_checkpoint_path (`str`): The path the to GGUF file to load return_tensors (`bool`, defaults to `False`): Whether to read the tensors from the file and return them. Not doing so is faster and only loads the metadata in memory. r) GGUFReader dequantizerrzgeneral.architecturez general.nameNrmistralrrTr is_gated_actT5EncoderModel architecturesrrstablelm> attn_k.bias attn_q.bias attn_v.biasffn_normc3HK|]}D]}||jvywr+r).0tensor bias_nameattn_bias_names r' z'load_gguf_checkpoint..s)mF^lmQZyFKK/m/ms"c3:K|]}|jvywr+r)rr ffn_norm_names r'rz'load_gguf_checkpoint..s#^VMV[[$@#^s use_qkv_biasuse_parallel_residualzGGUF model with architecture z is not supported yet.falconrc3:K|]}d|jk7yw)rNr)rrs r'rz'load_gguf_checkpoint..s HvOv{{ * Hstie_word_embeddingsr`rz1Some keys were not parsed and added into account z | rrr vocab_sizertokenszCan't find a way to retrieve missing config vocab_size from tokenizer parameters. This will use default value from model config class and cause unexpected behavior.rar,z,Converting and de-quantizing GGUF tensors...)desc)rrrUrVz0Some keys of the GGUF file were not considered: )-r r rrrrrrrlistkeysGGUF_TO_TRANSFORMERS_MAPPINGranyraGGUF_SUPPORTED_ARCHITECTURES ValueErrorallrrcrjoinrr rrlen isinstancer#removeinfowarningrr@TENSOR_PROCESSORSr)rr tensor_typer3rrdrer rf)%gguf_checkpoint_pathreturn_tensors model_to_loadrrrr reader_keysrzrV architecture model_nameupdated_architecturerr exceptionsgguf_keyrrprefix config_keyrr parameterparameter_renamesrenamed_config_keytokenizer_parametersrUrProcessorClass processorrrrresultrrs% @@r'load_gguf_checkpointrZs13//  A ghh , -F ]]Fv{{}%K(DE1BEEf&<=a@LFN3J,9 #:(   !<6:(#N3 , &(#N3CX?X(#$;<77z%J"%+%1GL%i01CD{*&&x0 1 { " KKKH:UXY^X_` a7b<"<0H<4A(#L1,X660= + +8;>0^_ TF;;D f.@.@AG&&#5"3 'FnnG;;D--%d+D161A1A"'''BR1S i ( .' T* ;! F{mTU yFhis 5 P$*P))NNr)FN))rtypingrrnumpyr tqdm.autor integrationsrrr utilsr utils.import_utilsr utils.loggingr rdrrrrrrrr)r5rQrlrrrrrrrr#rMrrr%r&r'r#s  ' &1% H  !*" "-F\] "5kBC$&<=O&PQ  $$@$J$O$O$QR --7?7<\o\<$?$N 7 7-/-4 -? - -o- -O -"' ! "  !'## d!% $ K K K K\Ur&