
    rh                    $   d Z ddlmZ ddlZddlZddlmZ ddlZ	ddl
mZ ddlmZmZmZmZ ddlmZmZmZmZmZmZ dd	lmZmZ dd
lmZmZmZmZm Z m!Z! ddl"m#Z#  e jH                  e%      Z&dZ'dZ(g dZ)dZ*dZ+e G d de             Z, G d dejZ                  j\                        Z/ G d dejZ                  j\                        Z0 G d dejZ                  j\                        Z1 G d dejZ                  j\                        Z2 G d dejZ                  j\                        Z3 G d dejZ                  j\                        Z4 G d d ejZ                  j\                        Z5 G d! d"ejZ                  j\                        Z6 G d# d$ejZ                  j\                        Z7e G d% d&ejZ                  j\                               Z8 G d' d(e      Z9d)Z:d*Z; ed+e:       G d, d-e9             Z< G d. d/ejZ                  j\                        Z= G d0 d1ejZ                  j\                        Z> G d2 d3ejZ                  j\                        Z? ed4e:       G d5 d6e9             Z@ ed7e:       G d8 d9e9e             ZA ed:e:       G d; d<e9             ZBg d=ZCy)>zTensorFlow DeiT model.    )annotationsN)	dataclass   )get_tf_activation)TFBaseModelOutputTFBaseModelOutputWithPoolingTFImageClassifierOutputTFMaskedImageModelingOutput)TFPreTrainedModelTFSequenceClassificationLossget_initializerkeraskeras_serializableunpack_inputs)
shape_liststable_softmax)ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )
DeiTConfigr   z(facebook/deit-base-distilled-patch16-224)r      i   ztabby, tabby catc                  X    e Zd ZU dZdZded<   dZded<   dZded<   dZded<   dZ	ded	<   y)
-TFDeiTForImageClassificationWithTeacherOutputa  
    Output type of [`DeiTForImageClassificationWithTeacher`].

    Args:
        logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
            Prediction scores as the average of the cls_logits and distillation logits.
        cls_logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
            Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
            class token).
        distillation_logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
            Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
            distillation token).
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus
            the initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
            the self-attention heads.
    Ntf.Tensor | Nonelogits
cls_logitsdistillation_logitsztuple[tf.Tensor] | Nonehidden_states
attentions)
__name__
__module____qualname____doc__r   __annotations__r    r!   r"   r#        |/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/transformers/models/deit/modeling_tf_deit.pyr   r   B   sA    ,  $F##'J ',0)0-1M*1*.J'.r*   r   c                  X     e Zd ZdZdd fdZddZd	dZ	 	 	 d
	 	 	 	 	 	 	 	 	 ddZ xZS )TFDeiTEmbeddingszv
    Construct the CLS token, distillation token, position and patch embeddings. Optionally, also the mask token.
    c                    t        |   di | || _        || _        t	        |d      | _        t        j                  j                  |j                  d      | _
        y )Npatch_embeddings)confignamedropoutr1   r)   )super__init__r0   use_mask_tokenTFDeiTPatchEmbeddingsr/   r   layersDropouthidden_dropout_probr2   )selfr0   r6   kwargs	__class__s       r+   r5   zTFDeiTEmbeddings.__init__f   sS    "6", 5VJ\ ]||++F,F,FY+Wr*   c                   | j                  dd| j                  j                  ft        j                  j                         dd      | _        | j                  dd| j                  j                  ft        j                  j                         dd      | _        d | _        | j                  rM| j                  dd| j                  j                  ft        j                  j                         dd      | _        | j                  j                  }| j                  d|dz   | j                  j                  ft        j                  j                         dd      | _        | j                  ry d| _        t        | d	d       Mt        j                   | j                  j"                        5  | j                  j%                  d        d d d        t        | d
d       Nt        j                   | j&                  j"                        5  | j&                  j%                  d        d d d        y y # 1 sw Y   exY w# 1 sw Y   y xY w)Nr   T	cls_token)shapeinitializer	trainabler1   distillation_token
mask_token   position_embeddingsr/   r2   )
add_weightr0   hidden_sizer   initializerszerosr?   rC   rD   r6   r/   num_patchesrF   builtgetattrtf
name_scoper1   buildr2   )r;   input_shaperK   s      r+   rP   zTFDeiTEmbeddings.buildm   s   a001**002	 ) 
 #'//a001**002%	 #2 #
 "oo!T[[445!..446!	 . DO ++77#'??kAot{{'>'>?**002&	 $3 $
  ::
4+T2>t4499: 2%%++D124D)5t||001 )""4() ) 62 2) )s   +H+H7+H47I c           
        |j                   d   dz
  }| j                  j                   d   dz
  }||k(  r||k(  r| j                  S | j                  d d dd d f   }| j                  d d dd d f   }| j                  d d dd d d f   }|j                   d   }	|| j                  j                  z  }
|| j                  j                  z  }|
dz   |dz   }}
t	        j
                  |dt        t        j                  |            t        t        j                  |            |	f      }t        j                  j                  |t        |
      t        |      fd      }t	        j                  |g d	      }t	        j
                  |dd|	f      }t	        j                  t	        j                  |d
      t	        j                  |d
      |gd
      S )Nr   rE   r   g?bicubic)sizemethodr   rE   r   r   permaxis)r@   rF   r0   
patch_sizerN   reshapeintmathsqrtimageresize	transposeconcatexpand_dims)r;   
embeddingsheightwidthrK   num_positionsclass_pos_embeddist_pos_embedpatch_pos_embeddimh0w0s               r+   interpolate_pos_encodingz)TFDeiTEmbeddings.interpolate_pos_encoding   s    &&q)A-0066q9A=-'FeO+++221a7;11!Q':221ab!8<r"t{{---dkk,,, c28B**aTYY}%=!>DIImD\@]_bc
 ((///R#b'@R[d/e,,\J**_q"clCyy^^O!4bnn^Z[6\^mnuv
 	
r*   c                l   |j                   \  }}}}| j                  |      }t        |      \  }	}
}|it        j                  | j
                  |	|
dg      }t        j                  |d      }t        j                  ||j                        }|d|z
  z  ||z  z   }t        j                  | j                  |	d      }t        j                  | j                  |	d      }t        j                  |||fd      }| j                  }|r| j                  |||      }||z   }| j                  ||      }|S )	Nr   rS   rZ   dtypeg      ?r   )repeatsr[   training)r@   r/   r   rN   tilerD   re   castrs   repeatr?   rC   rd   rF   rp   r2   )r;   pixel_valuesbool_masked_posrv   rp   _rg   rh   rf   
batch_size
seq_lengthmask_tokensmask
cls_tokensdistillation_tokensposition_embeddings                   r+   callzTFDeiTEmbeddings.call   s)    +0065!**<8
$.z$:!
J&''$//J
A3NOK>>/;D774{'8'89D#sTz2[45GGJYYt~~zJ
 ii(?(?Z[\YY
,?LSTU
!55#!%!>!>z6SX!Y"44
\\*x\@
r*   F)r0   r   r6   boolreturnNoneN)rf   	tf.Tensorrg   r^   rh   r^   r   r   )NFF)
rz   r   r{   r   rv   r   rp   r   r   r   )	r$   r%   r&   r'   r5   rP   rp   r   __classcell__r=   s   @r+   r-   r-   a   sY    X%)N
< -1). * 	
 #' 
r*   r-   c                  4     e Zd ZdZd fdZddZddZ xZS )r7   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                   t        |   di | |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        t        j                  j                  |||d      | _        y )Nr   r   
projection)kernel_sizestridesr1   r)   )r4   r5   
image_sizer\   num_channelsrH   
isinstancecollectionsabcIterablerK   r   r8   Conv2Dr   )	r;   r0   r<   r   r\   r   rH   rK   r=   s	           r+   r5   zTFDeiTPatchEmbeddings.__init__   s    "6"!'!2!2F4E4EJ
$*$7$79K9Kk#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$$(&,,--Z, . 
r*   c                    t        |      \  }}}}t        j                         r|| j                  k7  rt	        d      | j                  |      }t        |      \  }}}}t        j                  ||||z  |f      }|S )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.)r   rN   executing_eagerlyr   
ValueErrorr   r]   )r;   rz   r}   rg   rh   r   xs          r+   r   zTFDeiTPatchEmbeddings.call   s    2<\2J/
FE<!ld6G6G&Gw  OOL)2<Q-/
FE<JJq:v~|DEr*   c                   | j                   ry d| _         t        | dd       \t        j                  | j                  j
                        5  | j                  j                  d d d | j                  g       d d d        y y # 1 sw Y   y xY w)NTr   )rL   rM   rN   rO   r   r1   rP   r   r;   rQ   s     r+   rP   zTFDeiTPatchEmbeddings.build   s}    ::
4t,8t334 M%%tT49J9J&KLM M 9M Ms   *A??Br0   r   r   r   )rz   r   r   r   r   r$   r%   r&   r'   r5   r   rP   r   r   s   @r+   r7   r7      s    
"
Mr*   r7   c                  N     e Zd Zd fdZddZ	 d	 	 	 	 	 	 	 	 	 ddZd	dZ xZS )
TFDeiTSelfAttentionc                   t        |   d
i | |j                  |j                  z  dk7  r&t	        d|j                   d|j                   d      |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _        t        j                  | j                        | _
        t        j                  j                  | j                  t        |j                        d      | _        t        j                  j                  | j                  t        |j                        d      | _        t        j                  j                  | j                  t        |j                        d      | _        t        j                  j'                  |j(                  	      | _        || _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()queryunitskernel_initializerr1   keyvaluerater)   )r4   r5   rH   num_attention_headsr   r^   attention_head_sizeall_head_sizer_   r`   sqrt_att_head_sizer   r8   Denser   initializer_ranger   r   r   r9   attention_probs_dropout_probr2   r0   r;   r0   r<   r=   s      r+   r5   zTFDeiTSelfAttention.__init__   s   "6" : ::a?#F$6$6#7 8''-'A'A&B!E 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PP"&))D,D,D"E\\''$$IaIa9bip ( 

 <<%%$$IaIa9bin & 
 \\''$$IaIa9bip ( 

 ||++1T1T+Ur*   c                    t        j                  ||d| j                  | j                  f      }t        j                  |g d      S )NrS   tensorr@   r   rE   r   r   rX   )rN   r]   r   r   rc   )r;   r   r}   s      r+   transpose_for_scoresz(TFDeiTSelfAttention.transpose_for_scores  s;    6*b$BZBZ\`\t\t1uv ||F66r*   c                   t        |      d   }| j                  |      }| j                  |      }| j                  |      }| j	                  ||      }	| j	                  ||      }
| j	                  ||      }t        j                  |	|
d      }t        j                  | j                  |j                        }t        j                  ||      }t        |d      }| j                  ||      }|t        j                  ||      }t        j                  ||      }t        j                  |g d	
      }t        j                  ||d| j                   f      }|r||f}|S |f}|S )Nr   inputsT)transpose_brr   rS   )r   r[   r   rv   r   rX   r   )r   r   r   r   r   rN   matmulrx   r   rs   divider   r2   multiplyrc   r]   r   )r;   r"   	head_maskoutput_attentionsrv   r}   mixed_query_layermixed_key_layermixed_value_layerquery_layer	key_layervalue_layerattention_scoresdkattention_probsattention_outputoutputss                    r+   r   zTFDeiTSelfAttention.call  se     .q1
 JJmJ<((-(8 JJmJ<//0A:N--ozJ	//0A:N 99[)NWWT,,4D4J4JK99%5r: )0@rJ ,,o,Q   kk/9EO99_kB<<(8|L ::-=jRTVZVhVhEij9J#_5 RbPcr*   c                   | j                   ry d| _         t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)NTr   r   r   )rL   rM   rN   rO   r   r1   rP   r0   rH   r   r   r   s     r+   rP   zTFDeiTSelfAttention.buildG  s9   ::
4$'3tzz/ H

  $dkk.E.E!FGH4%1txx}}- FdDKK,C,CDEF4$'3tzz/ H

  $dkk.E.E!FGH H 4H HF FH Hs$   3E*<3E6-3F*E36E?Fr0   r   )r   r   r}   r^   r   r   r   
r"   r   r   r   r   r   rv   r   r   tuple[tf.Tensor]r   )r$   r%   r&   r5   r   r   rP   r   r   s   @r+   r   r      sN    47 ' ' '  	'
 ' 
'RHr*   r   c                  6     e Zd ZdZd fdZdddZddZ xZS )	TFDeiTSelfOutputz
    The residual connection is defined in TFDeiTLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    c                   t        |   di | t        j                  j	                  |j
                  t        |j                        d      | _        t        j                  j                  |j                        | _        || _        y Ndenser   r   r)   r4   r5   r   r8   r   rH   r   r   r   r9   r:   r2   r0   r   s      r+   r5   zTFDeiTSelfOutput.__init__]  o    "6"\\''$$IaIa9bip ( 

 ||++1K1K+Lr*   c                P    | j                  |      }| j                  ||      }|S Nr   r   r   r2   r;   r"   input_tensorrv   s       r+   r   zTFDeiTSelfOutput.callf  s*    

-
8MHMr*   c                (   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   y xY wNTr   	rL   rM   rN   rO   r   r1   rP   r0   rH   r   s     r+   rP   zTFDeiTSelfOutput.buildl  }    ::
4$'3tzz/ H

  $dkk.E.E!FGH H 4H H   3BBr   r   r"   r   r   r   rv   r   r   r   r   r   r   s   @r+   r   r   W  s    
Hr*   r   c                  L     e Zd Zd fdZd Z	 d	 	 	 	 	 	 	 	 	 ddZddZ xZS )	TFDeiTAttentionc                l    t        |   di | t        |d      | _        t	        |d      | _        y )N	attentionr3   outputr)   )r4   r5   r   self_attentionr   dense_outputr   s      r+   r5   zTFDeiTAttention.__init__w  s1    "6"1&{K,V(Cr*   c                    t         r   NotImplementedError)r;   headss     r+   prune_headszTFDeiTAttention.prune_heads}  s    !!r*   c                p    | j                  ||||      }| j                  |d   ||      }|f|dd  z   }|S )Nr"   r   r   rv   r   r"   r   rv   r   )r   r   )r;   r   r   r   rv   self_outputsr   r   s           r+   r   zTFDeiTAttention.call  sb     **&)O`ks + 
  ,,&q/x - 
 $%QR(88r*   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   exY w# 1 sw Y   y xY w)NTr   r   )rL   rM   rN   rO   r   r1   rP   r   r   s     r+   rP   zTFDeiTAttention.build  s    ::
4)40<t22778 0##))$/04.:t00556 .!!''-. . ;0 0. .   C%CCC r   r   )
r   r   r   r   r   r   rv   r   r   r   r   )r$   r%   r&   r5   r   r   rP   r   r   s   @r+   r   r   v  sM    D"    	
  
"	.r*   r   c                  0     e Zd Zd fdZddZddZ xZS )TFDeiTIntermediatec                T   t        |   di | t        j                  j	                  |j
                  t        |j                        d      | _        t        |j                  t              r"t        |j                        | _        || _        y |j                  | _        || _        y )Nr   r   r)   )r4   r5   r   r8   r   intermediate_sizer   r   r   r   
hidden_actstrr   intermediate_act_fnr0   r   s      r+   r5   zTFDeiTIntermediate.__init__  s    "6"\\''**vOgOg?hov ( 

 f''-'89J9J'KD$  (.'8'8D$r*   c                L    | j                  |      }| j                  |      }|S )Nr   )r   r   )r;   r"   s     r+   r   zTFDeiTIntermediate.call  s(    

-
800?r*   c                (   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   y xY wr   r   r   s     r+   rP   zTFDeiTIntermediate.build  r   r   r   r"   r   r   r   r   r$   r%   r&   r5   r   rP   r   r   s   @r+   r   r     s    Hr*   r   c                  2     e Zd Zd fdZdddZddZ xZS )TFDeiTOutputc                   t        |   di | t        j                  j	                  |j
                  t        |j                        d      | _        t        j                  j                  |j                        | _        || _        y r   r   r   s      r+   r5   zTFDeiTOutput.__init__  r   r*   c                Z    | j                  |      }| j                  ||      }||z   }|S r   r   r   s       r+   r   zTFDeiTOutput.call  s4    

-
8MHM%4r*   c                (   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   y xY wr   )	rL   rM   rN   rO   r   r1   rP   r0   r   r   s     r+   rP   zTFDeiTOutput.build  s}    ::
4$'3tzz/ N

  $dkk.K.K!LMN N 4N Nr   r   r   r   r   r  r   s   @r+   r  r    s    Nr*   r  c                  J     e Zd ZdZd fdZ	 d	 	 	 	 	 	 	 	 	 ddZddZ xZS )	TFDeiTLayerz?This corresponds to the Block class in the timm implementation.c                ^   t        |   di | t        |d      | _        t	        |d      | _        t        |d      | _        t        j                  j                  |j                  d      | _        t        j                  j                  |j                  d      | _        || _        y )	Nr   r3   intermediater   layernorm_beforeepsilonr1   layernorm_afterr)   )r4   r5   r   r   r   r
  r  deit_outputr   r8   LayerNormalizationlayer_norm_epsr  r  r0   r   s      r+   r5   zTFDeiTLayer.__init__  s    "6"(kB.vNK'X> % ? ?H]H]dv ? w$||>>vG\G\ct>ur*   c                    | j                  | j                  ||      |||      }|d   }||z   }| j                  ||      }| j                  ||      }| j	                  |||      }|f|dd  z   }	|	S )Nr   )r   r   r   rv   r   )r"   rv   r   r   )r   r  r  r
  r  )
r;   r"   r   r   rv   attention_outputsr   layer_outputintermediate_outputr   s
             r+   r   zTFDeiTLayer.call  s     !NN..mh.W/ + 
 -Q/ )=8 ++=8+T"//lU]/^ ''-MT\ ( 
  /$5ab$99r*   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   xY w# 1 sw Y   UxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)NTr   r
  r  r  r  )rL   rM   rN   rO   r   r1   rP   r
  r  r  r0   rH   r  r   s     r+   rP   zTFDeiTLayer.build  s   ::
4d+7t~~223 +$$T*+4.:t00556 .!!''-.4-9t//445 -  &&t,-4+T2>t4499: S%%++T49P9P,QRS4*D1=t33889 R$$**D$8O8O+PQR R >+ +. .- -S SR Rs<   H%H?H!3H.
3H:HH!H+.H7:Ir   r   r   r   r   r   s   @r+   r  r    sL    I	      	
  
@Rr*   r  c                  N     e Zd Zd fdZ	 d	 	 	 	 	 	 	 	 	 	 	 	 	 ddZddZ xZS )TFDeiTEncoderc                    t        |   di | t        |j                        D cg c]  }t	        |d|        c}| _        y c c}w )Nzlayer_._r3   r)   )r4   r5   rangenum_hidden_layersr  layer)r;   r0   r<   ir=   s       r+   r5   zTFDeiTEncoder.__init__  s@    "6"HMfNfNfHgh1k&!~>h
hs   Ac                    |rdnd }|rdnd }t        | j                        D ]-  \  }	}
|r||fz   } |
|||	   ||      }|d   }|s%||d   fz   }/ |r||fz   }|st        d |||fD              S t        |||      S )Nr)   r   r   r   c              3  &   K   | ]	  }||  y wr   r)   ).0vs     r+   	<genexpr>z%TFDeiTEncoder.call.<locals>.<genexpr>@  s     hqZ[Zghs   )last_hidden_stater"   r#   )	enumerater  tupler   )r;   r"   r   r   output_hidden_statesreturn_dictrv   all_hidden_statesall_attentionsr  layer_modulelayer_outputss               r+   r   zTFDeiTEncoder.call   s     #7BD0d(4 	FOA|#$58H$H!(+#A,"3!	M *!,M !/=3C2E!E	F    1]4D Dh]4E~$Vhhh +;LYg
 	
r*   c                    | j                   ry d| _         t        | dd       K| j                  D ];  }t        j                  |j
                        5  |j                  d        d d d        = y y # 1 sw Y   IxY w)NTr  )rL   rM   r  rN   rO   r1   rP   )r;   rQ   r  s      r+   rP   zTFDeiTEncoder.buildF  sp    ::
4$'3 &]]5::. &KK%& && 4& &s   A..A7	r   r   )r"   r   r   r   r   r   r&  r   r'  r   rv   r   r   z$TFBaseModelOutput | tuple[tf.Tensor]r   r  r   s   @r+   r  r    s]    i $
 $
 $
  	$

 #$
 $
 $
 
.$
L&r*   r  c                       e Zd ZeZ	 d	 	 	 	 	 	 	 d fdZd	dZd Zd Ze		 	 	 	 	 	 	 	 d
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd       Z
ddZ xZS )TFDeiTMainLayerc                   t        |   di | || _        t        ||d      | _        t        |d      | _        t        j                  j                  |j                  d      | _        |rt        |d      | _        y d | _        y )	Nrf   )r6   r1   encoderr3   	layernormr  poolerr)   )r4   r5   r0   r-   rf   r  r0  r   r8   r  r  r1  TFDeiTPoolerr2  r;   r0   add_pooling_layerr6   r<   r=   s        r+   r5   zTFDeiTMainLayer.__init__T  st     	"6"*6.Wcd$V)<88AVAV]h8i=Nl69TXr*   c                .    | j                   j                  S r   )rf   r/   )r;   s    r+   get_input_embeddingsz$TFDeiTMainLayer.get_input_embeddings`  s    ///r*   c                    t         )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        r   )r;   heads_to_prunes     r+   _prune_headszTFDeiTMainLayer._prune_headsc  s
    
 "!r*   c                J    |t         d g| j                  j                  z  }|S r   )r   r0   r  )r;   r   s     r+   get_head_maskzTFDeiTMainLayer.get_head_maskj  s*     %%!>!>>Ir*   c	                <   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      t        j                  |d      }| j                  |      }| j                  ||||      }	| j                  |	|||||      }
|
d   }| j                  ||      }| j                  | j                  ||      nd }|s|||fn|f}||
dd  z   S t        |||
j                  |
j                        S )	Nz You have to specify pixel_valuesrW   )r{   rv   rp   )r   r   r&  r'  rv   r   ru   r   )r#  pooler_outputr"   r#   )r0   r   r&  use_return_dictr   rN   rc   r<  rf   r0  r1  r2  r   r"   r#   )r;   rz   r{   r   r   r&  r'  rp   rv   embedding_outputencoder_outputssequence_outputpooled_outputhead_outputss                 r+   r   zTFDeiTMainLayer.callr  sZ    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@ ||L,? &&y1	??+%=	 + 
 ,,/!5# ' 
 *!,..8.LKO;;KbOhGhl?L?XO];_n^pL/!""555+-')77&11	
 	
r*   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   1xY w# 1 sw Y   xY w# 1 sw Y   ~xY w# 1 sw Y   y xY w)NTrf   r0  r1  r2  )rL   rM   rN   rO   rf   r1   rP   r0  r1  r0   rH   r2  r   s     r+   rP   zTFDeiTMainLayer.build  sb   ::
4t,8t334 ,%%d+,4D)5t||001 )""4()4d+7t~~223 L$$dD$++2I2I%JKL44(4t{{//0 (!!$'( ( 5, ,) )L L( (s0   F%F#?3F/0F;F #F,/F8;GTFr0   r   r5  r   r6   r   r   r   )r   r7   NNNNNNFF)rz   r   r{   r   r   r   r   bool | Noner&  rI  r'  rI  rp   r   rv   r   r   z4TFBaseModelOutputWithPooling | tuple[tf.Tensor, ...]r   )r$   r%   r&   r   config_classr5   r7  r:  r<  r   r   rP   r   r   s   @r+   r.  r.  P  s    L Z_
Y 
Y59
YRV
Y	
Y0"  *.,0&*)-,0#').;
&;
 *;
 $	;

 ';
 *;
 !;
 #';
 ;
 
>;
 ;
z(r*   r.  c                      e Zd ZdZeZdZdZy)TFDeiTPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    deitrz   N)r$   r%   r&   r'   r   rJ  base_model_prefixmain_input_namer)   r*   r+   rL  rL    s    
 L$Or*   rL  aR  
    This model is a TensorFlow
    [keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer). Use it as a regular
    TensorFlow Module and refer to the TensorFlow documentation for all matter related to general usage and behavior.

    Parameters:
        config ([`DeiTConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`DeiTImageProcessor.__call__`] for details.

        head_mask (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z^The bare DeiT Model transformer outputting raw hidden-states without any specific head on top.c            	           e Zd Z	 d	 	 	 	 	 	 	 d fdZe ee       eee	e
de      	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	d                     Zd
dZ xZS )TFDeiTModelc                N    t        |   |fi | t        |||d      | _        y )NrM  r5  r6   r1   )r4   r5   r.  rM  r4  s        r+   r5   zTFDeiTModel.__init__  s.     	*6*#&7]c
	r*   vision)
checkpointoutput_typerJ  modalityexpected_outputc	           
     8    | j                  ||||||||      }	|	S )N)rz   r{   r   r   r&  r'  rp   rv   )rM  )
r;   rz   r{   r   r   r&  r'  rp   rv   r   s
             r+   r   zTFDeiTModel.call   s6    ( ))%+/!5#%=  	
 r*   c                    | j                   ry d| _         t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   y xY w)NTrM  )rL   rM   rN   rO   rM  r1   rP   r   s     r+   rP   zTFDeiTModel.build   se    ::
4&2tyy~~. &		%& & 3& &s   A11A:rF  rG  rH  )rz   r   r{   r   r   r   r   rI  r&  rI  r'  rI  rp   r   rv   r   r   z$tuple | TFBaseModelOutputWithPoolingr   )r$   r%   r&   r5   r   r   DEIT_INPUTS_DOCSTRINGr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr   rP   r   r   s   @r+   rQ  rQ    s     Z_
 
59
RV
	
 *+@A&0$. *.,0&*)-,0#').& * $	
 ' * ! #'  
. B .&r*   rQ  c                  0     e Zd Zd fdZddZddZ xZS )r3  c                    t        |   di | t        j                  j	                  |j
                  t        |j                        |j                  d      | _	        || _
        y )Nr   )r   r   
activationr1   r)   )r4   r5   r   r8   r   pooler_output_sizer   r   
pooler_actr   r0   r   s      r+   r5   zTFDeiTPooler.__init__+  sZ    "6"\\''++.v/G/GH((	 ( 

 r*   c                <    |d d df   }| j                  |      }|S )Nr   r   )r   )r;   r"   first_token_tensorrC  s       r+   r   zTFDeiTPooler.call6  s*     +1a40

*<
=r*   c                (   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   y xY wr   r   r   s     r+   rP   zTFDeiTPooler.build>  r   r   r   r   r   r  r   s   @r+   r3  r3  *  s    	Hr*   r3  c                  ,     e Zd ZdZd fdZddZ xZS )TFDeitPixelShufflez0TF layer implementation of torch.nn.PixelShufflec                x    t        |   di | t        |t              r|dk  rt	        d|       || _        y )NrE   z1upscale_factor must be an integer value >= 2 got r)   )r4   r5   r   r^   r   upscale_factor)r;   rj  r<   r=   s      r+   r5   zTFDeitPixelShuffle.__init__J  sA    "6".#..12DPQ_P`abb,r*   c           
        |}t        |      \  }}}}| j                  dz  }t        ||z        }t        j                  t        |      D 	cg c]  }t        |      D ]
  }	||	|z  z     c}	}g      }
t        j                  |t        j                  |
|dg      d      }t        j                  j                  || j                  d      }|S c c}	}w )NrE   r   rS   )paramsindices
batch_dimsNHWC)
block_sizedata_format)
r   rj  r^   rN   constantr  gatherrw   nndepth_to_space)r;   r   r"   r}   r|   num_input_channelsblock_size_squaredoutput_depthr  jpermutations              r+   r   zTFDeitPixelShuffle.callP  s    /9-/H,
Aq,!00!3-0BBC
 kk278J2KiQUZ[gUhiPQa!(((i(ij
 		V`bcUd@ertu,,]tGZGZhn,o	 js   C
)rj  r^   r   r   )r   r   r   r   )r$   r%   r&   r'   r5   r   r   r   s   @r+   rh  rh  G  s    :-r*   rh  c                  2     e Zd Zd fdZdddZddZ xZS )TFDeitDecoderc                    t        |   di | t        j                  j	                  |j
                  dz  |j                  z  dd      | _        t        |j
                  d      | _	        || _
        y )NrE   r   0)filtersr   r1   1r3   r)   )r4   r5   r   r8   r   encoder_strider   conv2drh  pixel_shuffler0   r   s      r+   r5   zTFDeitDecoder.__init__b  sk    "6"ll))))1,v/B/BBPQX[ * 
 00E0ECPr*   c                N    |}| j                  |      }| j                  |      }|S r   )r  r  )r;   r   rv   r"   s       r+   r   zTFDeitDecoder.callj  s+    M2**=9r*   c                   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d d | j                  j                  g       d d d        t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   exY w# 1 sw Y   y xY w)NTr  r  )
rL   rM   rN   rO   r  r1   rP   r0   rH   r  r   s     r+   rP   zTFDeitDecoder.buildp  s    ::
44(4t{{//0 O!!4tT[[5L5L"MNO4$/;t11667 /""((./ / <O O/ /s   4C#=C/#C,/C8r   r   )r   r   rv   r   r   r   r   r  r   s   @r+   r|  r|  a  s    	/r*   r|  z~DeiT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://huggingface.co/papers/2111.09886).c                       e Zd Zd fdZe ee       eee	      	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd                     Z
ddZ xZS )	TFDeiTForMaskedImageModelingc                p    t         |   |       t        |ddd      | _        t	        |d      | _        y )NFTrM  rS  decoderr3   )r4   r5   r.  rM  r|  r  r;   r0   r=   s     r+   r5   z%TFDeiTForMaskedImageModeling.__init__  s2     #FeTX_ef	$V)<r*   rV  rJ  c	           
        ||n| j                   j                  }| j                  ||||||||      }	|	d   }
|
ddddf   }
t        |
      \  }}}t	        |dz        x}}t        j                  |
||||f      }
| j                  |
|      }t        j                  |d      }d}|| j                   j                  | j                   j                  z  }t        j                  |d||f      }t        j                  || j                   j                  d      }t        j                  || j                   j                  d	      }t        j                  |d      }t        j                  |t
        j                        }t        j                   j#                  t        j                  |d
      t        j                  |d
            }t        j                  |d      }t        j$                  ||z        }t        j$                  |      dz   | j                   j&                  z  }||z  }t        j                  |d      }|s|f|	dd z   }||f|z   S |S t)        |||	j*                  |	j,                        S )a  
        bool_masked_pos (`tf.Tensor` of type bool and shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Returns:

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, TFDeiTForMaskedImageModeling
        >>> import tensorflow as tf
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
        >>> model = TFDeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="tf").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = tf.cast(tf.random.uniform((1, num_patches), minval=0, maxval=2, dtype=tf.int32), tf.bool)

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 224, 224]
        ```N)r{   r   r   r&  r'  rp   rv   r   r   rS   g      ?ru   )r   r   r   rE   rE   )r   rE   r   r   gh㈵>)r   )lossreconstructionr"   r#   )r0   r?  rM  r   r^   rN   r]   r  rc   r   r\   ry   re   rx   float32r   lossesmean_absolute_error
reduce_sumr   r
   r"   r#   )r;   rz   r{   r   r   r&  r'  rp   rv   r   rB  r}   sequence_lengthr   rg   rh   reconstructed_pixel_valuesmasked_im_lossrU   r   reconstruction_loss
total_lossnum_masked_pixelsr   s                           r+   r   z!TFDeiTForMaskedImageModeling.call  sU   V &1%<k$++B]B]))+/!5#%=  	
 "!* *!QrT'24>4O1
O\_c122**_z65R^6_` &*\\/H\%U" &(\\2Ll%["&;;))T[[-C-CCD jj2tT:JKO99_dkk.D.DaHD99T4;;#9#91=D>>$*D774,D"',,"B"B\<87F#
 #%..1Da"H':T'ABJ!#t!4t!;t{{?W?W W'*;;NZZ=N02WQR[@F3A3M^%.YSYY*5!//))	
 	
r*   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   exY w# 1 sw Y   y xY w)NTrM  r  )rL   rM   rN   rO   rM  r1   rP   r  r   s     r+   rP   z"TFDeiTForMaskedImageModeling.build  s    ::
4&2tyy~~. &		%&4D)5t||001 )""4() ) 6& &) )r   r   rH  )rz   r   r{   r   r   r   r   rI  r&  rI  r'  rI  rp   r   rv   r   r   z#tuple | TFMaskedImageModelingOutputr   )r$   r%   r&   r5   r   r   r[  r   r
   r]  r   rP   r   r   s   @r+   r  r  |  s    = *+@A+FUde *.,0&*)-,0#').a
&a
 *a
 $	a

 'a
 *a
 !a
 #'a
 a
 
-a
 f B a
F	)r*   r  z
    DeiT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                       e Zd Zd fdZe ee       eee	      	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd                     Z
ddZ xZS )	TFDeiTForImageClassificationc                :   t         |   |       |j                  | _        t        |dd      | _        |j                  dkD  r+t
        j                  j                  |j                  d      n t
        j                  j                  dd      | _	        || _
        y )NFrM  r5  r1   r   
classifierr3   linear)r4   r5   
num_labelsr.  rM  r   r8   r   
Activationr  r0   r  s     r+   r5   z%TFDeiTForImageClassification.__init__  s      ++#Fe&Q	
   1$ LLv00|D(((E 	
 r*   r  c	           	     B   ||n| j                   j                  }| j                  |||||||      }	|	d   }
| j                  |
dddddf         }|dn| j	                  ||      }|s|f|	dd z   }||f|z   S |S t        |||	j                  |	j                        S )a  
        labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, TFDeiTForImageClassification
        >>> import tensorflow as tf
        >>> from PIL import Image
        >>> import requests

        >>> keras.utils.set_random_seed(3)  # doctest: +IGNORE_RESULT
        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> # note: we are loading a TFDeiTForImageClassificationWithTeacher from the hub here,
        >>> # so the head will be randomly initialized, hence the predictions will be random
        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
        >>> model = TFDeiTForImageClassification.from_pretrained("facebook/deit-base-distilled-patch16-224")

        >>> inputs = image_processor(images=image, return_tensors="tf")
        >>> outputs = model(**inputs)
        >>> logits = outputs.logits
        >>> # model predicts one of the 1000 ImageNet classes
        >>> predicted_class_idx = tf.math.argmax(logits, axis=-1)[0]
        >>> print("Predicted class:", model.config.id2label[int(predicted_class_idx)])
        Predicted class: little blue heron, Egretta caerulea
        ```Nr   r   r&  r'  rp   rv   r   r   )r  r   r"   r#   )r0   r?  rM  r  hf_compute_lossr	   r"   r#   )r;   rz   r   labelsr   r&  r'  rp   rv   r   rB  r   r  r   s                 r+   r   z!TFDeiTForImageClassification.call  s    ^ &1%<k$++B]B]))/!5#%=  
 "!*Aq!9: ~t4+?+?+OY,F)-)9TGf$EvE&!//))	
 	
r*   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   |xY w# 1 sw Y   y xY w)NTrM  r  )
rL   rM   rN   rO   rM  r1   rP   r  r0   rH   r   s     r+   rP   z"TFDeiTForImageClassification.build]  s    ::
4&2tyy~~. &		%&4t,8t334 M%%tT4;;3J3J&KLM M 9& &M Ms   C"%3C."C+.C7r   rH  )rz   r   r   r   r  r   r   rI  r&  rI  r'  rI  rp   r   rv   r   r   z#tf.Tensor | TFImageClassifierOutputr   )r$   r%   r&   r5   r   r   r[  r   r	   r]  r   rP   r   r   s   @r+   r  r    s     *+@A+BQ`a *.&*#')-,0#').H
&H
 $H
 !	H

 'H
 *H
 !H
 #'H
 H
 
-H
 b B H
T	Mr*   r  a  
    DeiT Model transformer with image classification heads on top (a linear layer on top of the final hidden state of
    the [CLS] token and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet.

    .. warning::

            This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
            supported.
    c                       e Zd Zd fdZe ee       eee	e
e      	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd                     ZddZ xZS )	'TFDeiTForImageClassificationWithTeacherc                   t         |   |       |j                  | _        t        |dd      | _        |j                  dkD  r+t
        j                  j                  |j                  d      n t
        j                  j                  dd      | _	        |j                  dkD  r+t
        j                  j                  |j                  d      n t
        j                  j                  dd      | _
        || _        y )	NFrM  r  r   cls_classifierr3   r  distillation_classifier)r4   r5   r  r.  rM  r   r8   r   r  r  r  r0   r  s     r+   r5   z0TFDeiTForImageClassificationWithTeacher.__init__v  s      ++#Fe&Q	
   1$ LLv007GH((8H(I 	   1$ LLv007PQ((8Q(R 	$
 r*   )rU  rV  rJ  rX  c           	     R   ||n| j                   j                  }| j                  |||||||      }|d   }	| j                  |	d d dd d f         }
| j	                  |	d d dd d f         }|
|z   dz  }|s||
|f|dd  z   }|S t        ||
||j                  |j                        S )Nr  r   r   rE   )r   r    r!   r"   r#   )r0   r?  rM  r  r  r   r"   r#   )r;   rz   r   r   r&  r'  rp   rv   r   rB  r    r!   r   r   s                 r+   r   z,TFDeiTForImageClassificationWithTeacher.call  s    $ &1%<k$++B]B]))/!5#%=  
 "!*((Aq)AB
"::?1aQR7;ST 22a7j*=>LFM<! 3!//))
 	
r*   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)NTrM  r  r  )rL   rM   rN   rO   rM  r1   rP   r  r0   rH   r  r   s     r+   rP   z-TFDeiTForImageClassificationWithTeacher.build  s3   ::
4&2tyy~~. &		%&4)40<t22778 Q##))4t{{7N7N*OPQ42D9Et;;@@A Z,,22D$@W@W3XYZ Z F& &Q QZ Zs$   E%3E3E+EE(+E4r   )NNNNNFF)rz   r   r   r   r   rI  r&  rI  r'  rI  rp   r   rv   r   r   z5tuple | TFDeiTForImageClassificationWithTeacherOutputr   )r$   r%   r&   r5   r   r   r[  r   _IMAGE_CLASS_CHECKPOINTr   r]  _IMAGE_CLASS_EXPECTED_OUTPUTr   rP   r   r   s   @r+   r  r  i  s    & *+@A*A$4	 *.&*)-,0#').(
&(
 $(
 '	(

 *(
 !(
 #'(
 (
 
?(
 B (
TZr*   r  )r  r  r  rQ  rL  )Dr'   
__future__r   collections.abcr   r_   dataclassesr   
tensorflowrN   activations_tfr   modeling_tf_outputsr   r   r	   r
   modeling_tf_utilsr   r   r   r   r   r   tf_utilsr   r   utilsr   r   r   r   r   r   configuration_deitr   
get_loggerr$   loggerr]  r\  r^  r  r  r   r8   Layerr-   r7   r   r   r   r   r  r  r  r.  rL  DEIT_START_DOCSTRINGr[  rQ  r3  rh  r|  r  r  r  __all__r)   r*   r+   <module>r     s    "   !  /   3  + 
		H	%  A &  E 1  /K / /<ju||)) jZ*MELL.. *M\WH%,,,, WHvHu||)) H>$.ell(( $.PH++ H<N5<<%% N4@R%,,$$ @RH3&ELL&& 3&l n(ell(( n( n(d%- %	  2 d0&' 0&	0&hH5<<%% H:++ 4/ELL&& /6 ;
v)#8 v)
v)r  eM#8:V eMeMP  RZ.C RZRZjr*   