
    rho                        d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ  ej2                  e      Zi dddddddddddddddd
dddddddddddd
dddddddddddd
dddddddddddddd d!ddddddddddd
d"ddddddddddddd d#ddddddddddd
d$d%d&d'd(d)d*ddddddddddd
d+d,dd-dd.d/d0d1d2d3d4d5d6dd.d7d8dd9d:dddddddd;dd<	d=d,d>d?d@d-d.dAdBdddddddCdDdEdddd.ddFdGdHddI	dJdddddddddKdd
dddddddddddLddMdddddddddddLddMdNZdOdPdQdRdSd%d&d'd(dTdU
dVdd%d&d'd(dWdXZdY Z G dZ d[      Z G d\ d]e      Z  G d^ d_e      Z! G d` dae      Z" G db dce      Z# G dd dee      Z$ G df dge      Z%i de de!dhe!d!e!die!d*e"d+e#d#e#d:e#d=e#dBe#d0e$dEe#dJe#dje%dke%Z&dlefdmZ'y)nz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
    )arrayN)	Tokenizerdecodersnormalizerspre_tokenizers
processors)BPEUnigram   )
AddedToken)GemmaConverterGPT2ConverterLlamaConverterQwen2ConverterT5Converter)logging)tqdmgeneral
model_type_model_name_or_path)architecturenamellamamax_position_embeddingsnum_hidden_layersintermediate_sizehidden_sizehead_dim
rope_thetanum_attention_headsnum_key_value_headsrms_norm_eps
vocab_size)
context_lengthblock_countfeed_forward_lengthembedding_lengthrope.dimension_countrope.freq_baseattention.head_countattention.head_count_kv attention.layer_norm_rms_epsilonr#   mistralqwen2qwen2moenum_expertsnum_experts_per_tok)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   expert_countexpert_used_countqwen3qwen3moefalcon	tokenizerbos_token_ideos_token_idunk_token_idpad_token_id)ggml.bos_token_idggml.eos_token_idggml.unknown_token_idggml.padding_token_idphi3bloomn_layern_headlayer_norm_epsilon)r%   r'   r*   r#   attention.layer_norm_epsilont5n_positions
num_layersd_ffd_modeld_kv	num_headsrelative_attention_num_bucketsdecoder_start_token_id)r$   r%   r&   r'   attention.key_lengthr*   r+   rE   z attention.relative_buckets_countrN   r#   stablelmlayer_norm_eps)	r$   r%   r&   r'   r(   r*   r+   rE   r#   gpt2n_ctxn_embdr&   )r%   r$   r'   r&   r*   rE   
starcoder2norm_epsilon)r%   r$   r'   r&   r*   r+   rE   mambaconv_kernel
state_sizetime_step_rank)	r#   r$   r'   r,   r%   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizenemotronnorm_epssliding_window)r$   r%   r&   r'   r(   r)   rO   r*   r+   r,   zattention.sliding_windowr#   )gemma2gemma3tokenizer_typetokensscores
token_typemergesadd_prefix_space)

ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr<   r=   r>   r?   zggml.add_space_prefixchat_template)rg   rf   r<   r=   r>   r?   )r7   tokenizer_configc                    t        |t              s|g}t        |      dk(  r|d   }d }n|d   dk7  rt        d      |\  }}|dv rt	        | d         } | S |dv rt        | d         } | S |dv rt        | d         } | S |dv r3t        d	t        |             j                         j                         } | S |d
v rt        | |      } | S )N   r   	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rj   r            
      )      )   )   B)rk   )
isinstancelistlen
ValueErrorintfloatboolr   tobytesdecode_gguf_parse_value)_value	data_typearray_data_types      q/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/integrations/ggml.pyr   r     s    i&K	
9~aL	Q<1opp%."	?..VAY M 
g	vay! M 
c	fQi
 M	 
c	sDL)113::< M 
c	"6?;M    c                       e Zd Zd Zy)GGUFTokenizerSkeletonc                 p   |j                         D ]  \  }}t        | ||        t        | d      s)t        | d      rt        | d      st        d      | j                  }| j
                  }t        |      D ci c]  \  }}|||    c}}t        j                  d       g }t        j                               D ]k  \  }	}
g }t        dt        |	            D ]*  }|	d | |	|d  }}||v s||v s|j                  |||
f       , t        |fdd	      }|j                  |       m t        |d
 d	      }|D cg c]  }|d   |d   f }}|| _        nt| j                  D 	cg c]  }	t!        |	j#                  d             c}	| _        t        | d      s1t        t        | j                              D cg c]  }d  c}| _        t        | d      sg | _        t        | d      sd | _        t        | d      r| j&                  | j(                  | _        y y y c c}}w c c}w c c}	w c c}w )Nrd   ra   rb   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.rj   c                 $    | d      | d      fS )Nr   rj    )xvocabs    r   <lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>D  s    U1Q4[%!+4N r   T)keyreversec                     | d   S )Nr   r   )vals    r   r   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>F  s
    CF r   r    added_tokensr:   unknown_token_id)itemssetattrhasattrry   ra   rb   	enumerateloggerwarningr   rangerx   appendsortedextendrd   tuplesplitr   r:   r   )selfdict_kvra   rb   itrd   mergepiece_scorelocalindexpiece_lpiece_rr   _r   s                    @r   __init__zGGUFTokenizerSkeleton.__init__/  s    KKM 	 DAqD!Q	  tX&4*'$2I r  [[F[[F.7.?@daQq	\@ENNWXF&*5;;=&9 %"{"1c%j1 FE',Ve}eEFmWG&(W->gw%DEF u*NX\]e$% F(:DIF2893s1vs1v&9F9 DK@DLu5S!12LDK4*-23t{{3C-DEtEt^, "Dt^, $D 4+,1B1B1J $ 5 5D 2K,7 A : MEs   =H#H)+!H. 	H3N)__name__
__module____qualname__r   r   r   r   r   r   .  s    '6r   r   c                   0    e Zd Zd Zd Zd Zd Zd Zd Zy)GGUFLlamaConverterc                     t        |      | _        | j                  | _        i | _        t	        | j                  dd      dk7  | _        y )Nr`   r   )r   protooriginal_tokenizeradditional_kwargsgetattris_llama_3_tokenizerr   tokenizer_dicts     r   r   zGGUFLlamaConverter.__init__Z  s>    *>:
"&**!#$+DJJ8H'$RV]$]!r   c                 T    t        t        |j                  |j                              S Nrw   zipra   rb   r   r   s     r   r   zGGUFLlamaConverter.vocab`      Cell344r   c                     |j                   S r   rd   r   s     r   rd   zGGUFLlamaConverter.mergesc      ||r   c           
         | j                  | j                        }| j                  | j                        }t        |      D ci c]  \  }\  }}|| }}}}|j                  |j
                  |j                     nd }t        |dd       |j
                  |j                     nd }	t        |dd       |j
                  |j                     nd }
t        t        |||dd            }g }t        | j                  d      s^||j                  t        |dd             |	|j                  t        |	dd             |
|j                  t        |
dd             n}t        j                  t        j                  | j                  j                         dk(        d	   }|D ]6  }|j                  t        | j                  j
                  |   dd             8 t#        |      d	k7  r|j%                  |       t#        | j                  j&                        d	k7  r?|j)                  | j                  j&                  D cg c]  }t        |dd       c}       || j*                  d
<   |	| j*                  d<   |
| j*                  d<   | j,                  r>d | j*                  d<   d| j*                  d<   d| j*                  d<   d| j.                  _        |S c c}}}w c c}w )Nr8   r9   T)	unk_tokenfuse_unkbyte_fallbackrc   F
normalizedspecialrl   r   r   	eos_token	bos_tokenre   clean_up_tokenization_spaceslegacy)r   r   rd   r   r:   ra   r   r8   r   r	   r   r   r   npwherer   rc   rx   add_special_tokensr   
add_tokensr   r   r   r   )r   r   vocab_scoresrd   r   word_score	bpe_vocabr   r   r   r7   special_tokensspecial_tokens_idxidxadded_tokens                   r   r7   zGGUFLlamaConverter.tokenizerf  s   zz$**-TZZ(6?6MNN!2NT6T1WN	N8=8J8J8VELL!3!34\`	8?~W[8\8hELL!3!34nr	8?~W[8\8hELL!3!34nr	#"
	 tzz<0$%%juVZ&[\$%%juVZ&[\$%%juVZ&[\ "$"((4::3H3H*IQ*N!OPQ!R) j%%j1B1B31GTYcg&hij ~!#((8tzz&&'1,  ]a]g]g]t]tukKE5Iu /8{+.7{+.7{+$$9=D""#56EID""#AB/4D""8,-2D##*k ON vs   KKc                 0   t        j                         t        j                         t        j                  dd      g}| j                  r|t        j
                  ddd      gz  }|r|t        j                  dd      gz  }t        j                  |      S )N   ▁r   FTre   trim_offsets	use_regexrj   contentleft)r   ByteFallbackFuseReplacer   	ByteLevelStripSequencer   replacementre   sequences       r   decoderzGGUFLlamaConverter.decoder  s    !!#MMOUC(
 $$++UQVbfghhH!<==H  **r   c                     | j                  | j                        }| j                  | j                        }|||_        d}d}t        | j                  d      r| j                  j
                  }| j                  ||      }|||_        | j                  ||      |_        | j                         }|r||_        | j                  r7t        j                  ddd      |_        t        j                  g       |_        |S )Nr   Tre   Fr   )r7   r   
normalizerr   r   re   pre_tokenizerr   post_processorr   r   r   r   r   )r   r7   r   r   re   r   r   s          r   	convertedzGGUFLlamaConverter.converted  s    NN4::.	 __TZZ0
!#-I 4**,>?#66GG**;8HI$&3I# LL6FG	,,.'5I$ $$&4&>&>!&Ud'I#
 $/#7#7#;I r   N)	r   r   r   r   r   rd   r7   r   r   r   r   r   r   r   Y  s"    ^58t+!r   r   c                   *     e Zd Zd Zdef fdZ xZS )GGUFQwen2Converterc                 2    t        |      | _        i | _        y r   r   r   r   r   s     r   r   zGGUFQwen2Converter.__init__      "7"G!#r   returnc           	      4   t        | j                  j                        D ci c]  \  }}||
 }}}| j                  j                  }t        |   ||      }|j                  t        ddd      t        ddd      t        ddd      g       |S c c}}w )N<|endoftext|>FTr   z<|im_start|>z
<|im_end|>)r   r   ra   rd   superr   r   r   r   r   r   r   rd   r7   	__class__s         r   r   zGGUFQwen2Converter.converted  s    (1$2I2I2P2P(QRWQqRR((//G%eV4	$$?udK>eTJ<E4H	
  Ss   Br   r   r   r   r   r   __classcell__r   s   @r   r   r     s    $9  r   r   c                   6    e Zd Zd Zd Zd Zd Zd ZdefdZ	y)	GGUFPhi3Converterc                 T    t        |      | _        | j                  | _        i | _        y r   r   r   r   r   r   s     r   r   zGGUFPhi3Converter.__init__  s"    *>:
"&**!#r   c                 T    t        t        |j                  |j                              S r   r   r   s     r   r   zGGUFPhi3Converter.vocab  r   r   c                     |j                   S r   r   r   s     r   rd   zGGUFPhi3Converter.merges  r   r   c                    | j                  | j                        }| j                  | j                        }t        |      D ci c]  \  }\  }}|| }}}}t	        t        ||            }|j                  t        ddddd      t        ddd      t        dddd      t        d	ddd      t        d
ddd      t        dddd      t        dddd      t        dddd      t        dddd      t        dddd      t        dddd      t        dddd      g       |j                  |j                  |j                     nd | j                  d<   |j                  |j                  |j                     nd | j                  d<   |j                  |j                  |j                     nd | j                  d<   |j                  |j                  |j                     nd | j                  d<   |S c c}}}w )N</s>TF)rstriplstripr   r   r   r   z<|assistant|>)r  r   r   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r   r   r   	pad_token)r   r   rd   r   r   r	   r   r   r:   ra   r   r9   r8   r;   )	r   r   r   rd   r   r   r   r   r7   s	            r   r7   zGGUFPhi3Converter.tokenizer  s    zz$**-TZZ(6?6MNN!2NT6T1WN	Nc)V45	$$6$uX\]?udK?4ESWX-duVZ[-duVZ[-duVZ[-duVZ[<PTU9TeTR-duVZ[-duVZ[:dudS	
$ 160B0B0NELL++,TX 	{+ 160B0B0NELL++,TX 	{+ 160B0B0NELL++,TX 	{+ 160B0B0NELL++,TX 	{+ E Os   G:c                     t        j                         t        j                         t        j                  |d      g}|r|t        j                  dd      gz  }t        j
                  |      S )Nr   rj   r   )r   r   r   r   r   r   r   s       r   r   zGGUFPhi3Converter.decoder  s\    !!#MMO[#.
 !<==H  **r   r   c                     | j                  | j                        }d}d}t        | j                  d      r| j                  j                  }| j                  ||      |_        |S )Nr   Tre   )r7   r   r   r   re   r   )r   r7   r   re   s       r   r   zGGUFPhi3Converter.converted$  s\    NN4::.	4**,>?#66GG LL6FG	r   N)
r   r   r   r   r   rd   r7   r   r   r   r   r   r   r  r    s(    $
5%N	+
9 
r   r  c                   *     e Zd Zd Zdef fdZ xZS )GGUFGPTConverterc                 2    t        |      | _        i | _        y r   r   r   s     r   r   zGGUFGPTConverter.__init__2  r   r   r   c                     t        | j                  j                        D ci c]  \  }}||
 }}}| j                  j                  }t        |   ||      }|S c c}}w r   )r   r   ra   rd   r   r   r   s         r   r   zGGUFGPTConverter.converted6  s]    (1$2I2I2P2P(QRWQqRR((//G%eV4	 Ss   Ar   r   s   @r   r  r  1  s    $9  r   r  c                   0    e Zd Zd Zd Zd Zd ZdefdZy)GGUFT5Converterc                     dg|d<   t        |      | _        t        | j                  j                        D ci c]  \  }}||
 c}}| _        | j                  | _        i | _        y c c}}w Nz
dummy textrd   )r   r   r   ra   token2idr   r   )r   r   r   r   s       r   r   zGGUFT5Converter.__init__>  s]    $0>x *>:
*3DJJ4E4E*FG$!QAG"&**!# Hs   A(c                 T    t        t        |j                  |j                              S r   r   r   s     r   r   zGGUFT5Converter.vocabG  r   r   c                     t        | j                  dd      rcg }t        | j                  dd      r|t        j                  d      gz  }|t        j                  dd      gz  }t        j
                  |      S y )Nr   Tre   r   )prependr   )patternr   )r   r   r   Prependr   r   )r   r   r   s      r   r   zGGUFT5Converter.normalizerJ  sr    4**Hd;Ht..0BDI[00?@@,,S%HIIH''11r   c                 Z    t        j                  ddgg dd| j                  d   fg      S )N$Ar  )r  r  z$Br  )singlepairr   )r   TemplateProcessingr  )r   s    r   r   zGGUFT5Converter.post_processorS  s5    ,,&>-v./
 	
r   r   c                    | j                  | j                        }t        t        || j                  j                  d            }| j                  | j                        }|||_        d}d}t        | j                  d      r| j                  j                  }| j                  ||      }|||_	        | j                  ||      |_
        | j                         }|r||_        |S )NFunk_idr   r   Tre   )r   r   r   r
   r:   r   r   r   re   r   r   r   )r   r   r7   r   r   re   r   r   s           r   r   zGGUFT5Converter.converted\  s    zz$**-zz..#
	 __TZZ0
!#-I 4**,>?#66GG**;8HI$&3I# LL6FG	,,.'5I$r   N)	r   r   r   r   r   r   r   r   r   r   r   r   r  r  =  s"    $5
9 r   r  c                   0    e Zd Zd Zd Zd Zd ZdefdZy)GGUFGemmaConverterc                 `    dg|d<   t        |      | _        | j                  | _        i | _        y r  r  r   s     r   r   zGGUFGemmaConverter.__init__}  s.    $0>x *>:
"&**!#r   c                 L   t        t        |j                  |j                              }g }|D ]s  \  }}|dk(  r|j	                  d|f       d|v r>t        |j                               dk(  r"dt        |      z  }|j	                  ||f       a|j	                  ||f       u |S )Nz<0x09>	r   r   r   )rw   r   ra   rb   r   rx   strip)r   r   original_vocabupdated_vocabtokenscoreunderscoress          r   r   zGGUFGemmaConverter.vocab  s    c%,,=>* 	5LE5 $$dE]3#ekkm"4"9#c%j0$$k5%9:$$eU^4	5 r   c                 .    t        j                  dd      S )Nr   r   )r   r   r   s     r   r   zGGUFGemmaConverter.normalizer  s    ""3..r   c                     t        j                  dd      t        j                         t        j                         g}|r|t        j                  dd      gz  }t        j
                  |      S )Nr   r   rj   r   )r   r   r   r   r   r   r   s       r   r   zGGUFGemmaConverter.decoder  s\    UC(!!#MMO
 !<==H  **r   r   c                    | j                  | j                        }t        t        || j                  j                  | j
                              }| j                  | j                        }|||_        d}d}t        | j                  d      r| j                  j                  }| j                  ||      |_
        | j                  ||      }|||_        |S )Nr!  r   Tre   )r   r   r   r
   r:   handle_byte_fallbackr   r   r   re   r   r   )r   r   r7   r   r   re   r   s          r   r   zGGUFGemmaConverter.converted  s    zz$**-zz.."77
	 __TZZ0
!#-I 4**,>?#66GG LL6FG	**;8HI$&3I#r   N)	r   r   r   r   r   r   r   r   r   r   r   r   r$  r$  |  s"    $/	+9 r   r$  	qwen2_moe	qwen3_moer^   gemma3_textr   c                 ^    | }t        |   |      }|j                         }||j                  fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )GGUF_TO_FAST_CONVERTERSr   r   )r   r   tokenizer_class_name	converterfast_tokenizers        r   convert_gguf_tokenizerr:    s9     ('(<=nMI((*N96666r   )(__doc__r   numpyr   
tokenizersr   r   r   r   r   tokenizers.modelsr	   r
    r   convert_slow_tokenizerr   r   r   r   r   utilsr   utils.loggingr   
get_loggerr   r   GGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPINGr   r   r   r   r  r  r  r$  r6  r:  r   r   r   <module>rF     s   
   S S *  o o    
		H	%X$%X
 3*2) *& 5#8,:"X$ 3*2) *& 5#8,:"%X> 3*2) $& 5#8,:"?XV 3*2) $& 5#8,:"%2WXr 3*2) $& 5#8,:"sXJ 3*2) $& 5#8,:"%2KXf 3*2) $& 5#8,:"gX~ ++!/!/	XJ 3*2) $& 5#8,:"KXb  ) ("(<cXp 	'#%% & +#8(<,L":"qXJ 3*2) $ 5#8(8"
KX`  !$4 ((<aXp *3)2 5#8(6qXB "3),@*(&.-
CXX 3*2) $& 5#8,6"YXr 4*2) $& !+ 5#8,:$4"" 4*2) $& !+ 5#8,:$4"QX x ''++!/!/!3 )"++!/!/ 00(6 (6Vv vr (H HV	} 	<k <~> >B # 	
 #       " 	/       %! (7I 7r   