
    rhS"                        d dl mZ 	 d dlmZ d dlZd dlmZ d dl	mc m
Z d dlmZ  G d dej                        Z G d de      Zy# e$ r	 d dlmZ Y Gw xY w)	    )annotations)SelfN)Modulec                  L     e Zd Zd fdZddZedd       Zedd       Z xZS )TiedTransposec                0    t         |           || _        y N)super__init__linear)selfr   	__class__s     /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/sentence_transformers/sparse_encoder/models/SparseAutoEncoder.pyr   zTiedTranspose.__init__   s        c                    | j                   j                  J t        j                   || j                   j                  j	                         d       S r	   )r   biasFweightt)r   xs     r   forwardzTiedTranspose.forward   s>    {{'''xx4;;--//1488r   c                J    | j                   j                  j                         S r	   )r   r   r   r   s    r   r   zTiedTranspose.weight   s    {{!!##%%r   c                .    | j                   j                  S r	   )r   r   r   s    r   r   zTiedTranspose.bias   s    {{r   )r   z	nn.Linearr   torch.Tensorreturnr   )r   r   )	__name__
__module____qualname__r   r   propertyr   r   __classcell__r   s   @r   r   r      s5    9 & &    r   r   c                       e Zd ZdZg dZdhZ	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZddZdddZddZ	dddZ
ddd	Z	 d	 	 	 	 	 dd
ZdddZe	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 dd       Zd ZddZ xZS )SparseAutoEncodera  
    This module implements the Sparse AutoEncoder architecture based on the paper:
    Beyond Matryoshka: Revisiting Sparse Coding for Adaptive Representation, https://arxiv.org/abs/2503.01776

    This module transforms dense embeddings into sparse representations by:

    1. Applying a multi-layer feed-forward network
    2. Applying top-k sparsification to keep only the largest values
    3. Supporting auxiliary losses for training stability (via k_aux parameter)

    Args:
        input_dim: Dimension of the input embeddings.
        hidden_dim: Dimension of the hidden layers. Defaults to 512.
        k: Number of top values to keep in the final sparse representation. Defaults to 8.
        k_aux: Number of top values to keep for auxiliary loss calculation. Defaults to 512.
        normalize: Whether to apply layer normalization to the input embeddings. Defaults to False.
        dead_threshold: Threshold for dead neurons. Neurons with non-zero activations below this threshold are considered dead. Defaults to 30.
    )	input_dim
hidden_dimkk_aux	normalizedead_thresholdmax_active_dimsc                $    t                    | _        | _         _        t        j                  t        j                  |             _	        t        j                  ||d       _        t        j                  t        j                  |             _        t         j                         _        | _        | _        | _           j%                  dt        j                  |t        j&                                fd}| _        y )NF)r   stats_last_nonzero)dtypec                P    j                   kD  }| xj                  |z  c_        | S r	   )r.   data)r   	dead_maskr+   r   s     r   auxk_mask_fnz0SparseAutoEncoder.__init__.<locals>.auxk_mask_fnQ   s&    //.@IFFiFHr   )r
   r   r&   r'   r+   nn	Parametertorchzerospre_biasLinearencoderlatent_biasr   decoderr(   r)   r*   register_bufferlongr3   )	r   r&   r'   r(   r)   r*   r+   r3   r   s	   `     ` r   r   zSparseAutoEncoder.__init__9   s     	"$,U[[%;<"$))Iz"N<<J(?@&3DLL&A
"15;;zQVQ[Q[3\]	
 )r   c                    || j                   z
  }t        j                  || j                  j                  | j
                        }|S )a&  
        :param x: input data (shape: [batch, input_dim])
        :param latent_slice: slice of latents to compute
            Example: latent_slice = slice(0, 10) to compute only the first 10 latents.
        :return: autoencoder latents before activation (shape: [batch, hidden_dim])
        )r8   r   r   r:   r   r;   )r   r   latents_pre_acts      r   encode_pre_actz SparseAutoEncoder.encode_pre_actX   s:     ((1dll&9&94;K;KLr   c                r    |j                  dd      }||z
  }|j                  dd      }|||z   z  }|||fS )NT)dimkeepdim)meanstd)r   r   epsmurG   s        r   LNzSparseAutoEncoder.LNc   sH    VVDV)FeeDe)sO"czr   c                z    | j                   s|t               fS | j                  |      \  }}}|t        ||      fS )N)rI   rG   )r*   dictrJ   )r   r   rI   rG   s       r   
preprocesszSparseAutoEncoder.preprocessj   s:    ~~df9WWQZ
2s$"#&&&r   c                   || j                   }t        j                  ||d      }t        j                  |      }|j	                  d|j
                  |j                         t        j                  |      }t        j                  | j                        }|j                  d|j
                  j                  d      |j                  dkD  j                  |j                        j                  d             | xj                  d|j                  d      z
  z  c_	        | xj                  dz  c_	        d}| j                  r|rt        j                  | j!                  |      | j                        }	t        j                  |      }
|
j	                  d|	j
                  |	j                         t        j                  |
      }||fS )	z
        :param x: input data (shape: [batch, input_dim])
        :return: autoencoder latents (shape: [batch, hidden_dim])
        NrC   )r(   rD   r   h㈵>   )max)inputr(   )r(   r6   topk
zeros_likescatter_indicesvaluesr   relur.   scatter_add_reshapetor/   clampr)   r3   )r   r   r(   compute_auxrS   z_topk	latents_ktmplatents_auxkaux_topkz_auxks              r   top_kzSparseAutoEncoder.top_kp   sc   
 9Azz!qb)!!!$DLL$++6FF6N	t667LL  $[[4##CII.66r:	

 	1syyQy'7#771$ ::+zz''***H %%a(FOOB 0 0(//B66&>L,&&r   c                ~    | j                  |      | j                  z   }| j                  r|J ||d   z  |d   z   }|S )z
        :param latents: autoencoder latents (shape: [batch, hidden_dim])
        :return: reconstructed data (shape: [batch, n_inputs])
        rG   rI   )r<   r8   r*   )r   latentsinforets       r   decodezSparseAutoEncoder.decode   sJ     ll7#dmm3>>###U#d4j0C
r   c                F   ||n| j                   }|d   }t        j                         rC| j                  |      \  }}| j	                  |      }| j                  ||d      \  }}||d<   |S | j                  |      \  }}| j	                  |      }| j                  ||      \  }}	| j                  |d|z        \  }
}| j                  ||      }| j                  |
|      }| j                  |	|      }|j                  |||
|	||||| j                  z
  d       ||d<   |S )Nsentence_embeddingF)r]      )sentence_embedding_backbonesentence_embedding_encodedsentence_embedding_encoded_4kauxiliary_embeddingdecoded_embedding_kdecoded_embedding_4kdecoded_embedding_auxdecoded_embedding_k_pre_bias)	r(   r6   is_inference_mode_enabledrM   rA   rd   ri   updater8   )r   featuresr,   r(   r   rg   r@   r_   _ra   
latents_4krecons_k	recons_4k
recons_auxs                 r   r   zSparseAutoEncoder.forward   s<     /:O)* **,ooa(GAt"11!4O::oqe:LLIq-6H)*O//!$4--a0"&**_a"@	<

?AE:
A;;y$/KK
D1	[[t4
 	/0.=1;'3'/(1)3084==0H		
 *3%&r   c                L    | j                  |       | j                  ||       y )N)safe_serialization)save_configsave_torch_weights)r   output_pathr~   s      r   savezSparseAutoEncoder.save   s$    %@RSr   c                x    |||||d} | j                   dd|i|}	 | di |	}
 | j                  d||
d|}
|
S )N)	subfoldertokencache_folderrevisionlocal_files_onlymodel_name_or_path)r   model )load_configload_torch_weights)clsr   r   r   r   r   r   kwargs
hub_kwargsconfigr   s              r   loadzSparseAutoEncoder.load   sd     #(  0

 !U4FU*Uf&&&h:LTYh]ghr   c                *    d| j                          dS )NzSparseAutoEncoder())get_config_dictr   s    r   __repr__zSparseAutoEncoder.__repr__   s    #D$8$8$:#;1==r   c                    | j                   S )z
        Get the dimension of the sentence embedding. Warning: the number of non-zero elements in the embedding is only k out of the hidden_dim.

        Returns:
            int: Dimension of the sentence embedding
        )r'   r   s    r    get_sentence_embedding_dimensionz2SparseAutoEncoder.get_sentence_embedding_dimension   s     r   )      r   F   )r&   intr'   r   r(   r   r)   r   r*   boolr+   r   r   Noner   )rO   )r   r   rH   float)r   r   )NT)r   r   r(   
int | Noner]   r   r   r   r	   )rf   r   r   r   )rw   dict[str, torch.Tensor]r,   r   r   r   )T)r~   r   r   r   ) NNNF)r   strr   r   r   zbool | str | Noner   
str | Noner   r   r   r   r   r   )r   r   )r   r   r    __doc__config_keysforward_kwargsr   rA   rJ   rM   rd   ri   r   r   classmethodr   r   r   r"   r#   s   @r   r%   r%   !   s*   & [K'(N
  )) ) 	)
 ) ) ) 
)>	''B PT'/'BL'	 'RT  #'#'#!&  !	
 !   
 ,>r   r%   )
__future__r   typingr   ImportErrortyping_extensionsr6   torch.nnr4   torch.nn.functional
functionalr   #sentence_transformers.models.Moduler   r   r%   r   r   r   <module>r      sS    "'     6 BII  $K K9  '&'s   A AA