
    rh                     
	   U d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZm	Z	m
Z
mZmZmZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
l m!Z! er"d dl"Z"d dl#Z#d dl$m%Z& d dl'Zd dl(Zd dl)Zd dl*Zd dl+Zda,ee-   e.d<   g dZ/de-fdZ0i Z1e2e3e4f   e.d<   	 dRde5deejl                  jn                     de3fdZ8dee3   fdZ9de3fdZ:d Z;de3fdZ< ejz                  e>      Z? G d de      Z@dejl                  jn                  ddfdZAdeBe3df   fdZCdejl                  jn                  deBedf   fdZDdejl                  jn                  defd ZEdejl                  jn                  deBe3df   fd!ZFd"eBe3df   deBe3df   fd#ZGd$eBeej                  ej                  e4ej                  eKej                  e-f   df   deBd%   fd&ZMd'eBej                  df   d(eBd%   deBej                  df   fd)ZNd*edej                  fd+ZOd,eej                  e4eKe-f   d-d.dej                  fd/ZPd*ej                  d0eej                  ej                  e4ej                  eKej                  e-f   deej                  e4eKe-f   fd1ZQd2d3d4eBe3df   d5eBej                  df   d6eBd%   d7eBe3df   d8eBej                  df   d9eBd%   d:e-d;eBd<   d=eBeej                  ej                  e4ej                  eKej                  e-f   df   deBeej                  e4eKe-f   df   fd>ZRd2d3d4eBe3df   d5eBej                  df   d6eBd%   d7eBe3df   d8eBej                  df   d9eBd%   d:e-d;eBd<   d=eBeej                  ej                  e4ej                  eKej                  e-f   df   deBeej                  e4eKe-f   df   fd?ZSd@eTfdAZU G dB dC      ZVe j                   G dD dE             ZXee3eBe3ee3ef   f   f   ZYee.dF<   	  e j                  dGH       edIJ       G dK dL                    ZZ edIJ       G dM dN             Z[ edIJ      ddOdejl                  jn                  dPeeeZee3ef   f      fdQ       Z\y)S    N)MappingSequence)AnyCallableFinalOptionalTYPE_CHECKINGUnion)	TypeAlias)
FakeTensor)compatibility)FakeTensorProp)OperatorSupport)CALLABLE_NODE_OPS)_pytree_pybind_state_SUPPORT_ONNXRT)is_onnxrt_backend_supportedtorch_compile_backendOrtExecutionProviderOrtBackendOptions
OrtBackendreturnc                     t         `	 t        j                  d       t        j                  d       t        j                  d       ddl} ddl} ddl} ddlm}m}m	}m
} da t         S t         S # t        $ r
 da Y t         S w xY w)	a!  Returns ``True`` if ONNX Runtime dependencies are installed and usable
    to support TorchDynamo backend integration; ``False`` otherwise.

    Example::

        # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ONNX)
        >>> import torch
        >>> if torch.onnx.is_onnxrt_backend_supported():
        ...     @torch.compile(backend="onnxrt")
        ...     def f(x):
        ...             return x * x
        ...     print(f(torch.randn(10)))
        ... else:
        ...     print("pip install onnx onnxscript onnxruntime")
        ...
    Nonnxruntimezonnxruntime.capi._pybind_state
onnxscriptr   )decomposition_tablefx_onnx_interpreterpasses
type_utilsTF)r   	importlibimport_module
torch.onnxtorch.onnx._internal%torch.onnx._internal._exporter_legacytorch.onnx._internal.fxr   r   r    r!   ImportError)torchr   r   r    r!   s        s/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/torch/onnx/_internal/onnxruntime.pyr   r   /   s    & 	$##M2##$DE ##L1'8  #O ?  	$#O	$s   AA- -B ?B _dumped_onnx_modelmodel_stringgraph_modulec                    t         j                  j                  dd      }|syt        j                  |d      dz   }| | d}t	        |d      5 }|j                  |        ddd       |t        |<   |D| | d}t	        |d	d
      5 }|j                  t        |j                               ddd       |S |S # 1 sw Y   ZxY w# 1 sw Y   |S xY w)a  Stores the onnx model into a file.
    The name is "{ONNXRT_DUMP_PATH}{N}.onnx"
    where *N* is the number of files already stored with
    this prefix.
    If graph_module is not None, the graph is stored as a string with
    the same filename except the extension (.txt).
    ONNXRT_DUMP_PATHN    z.onnxwbz.txtwzutf-8)encoding)osenvirongetr+   openwritestrgraph)r,   r-   prefixnfilenameffilename_txts          r*   _dump_onnx_modelrB   b   s     ZZ^^.5Fvr*Q.A!E"H	h	 	!"v !D),g6 	-!GGC**+,	-O8O 
	-Os   B;
%C;CCc                      dgS )NCPUExecutionProvider rE       r*   _infer_default_epsrG   {   s     ###rF   namec                     t         j                  j                         r*t         j                  j                  j	                  |        yy)zIf PyTorch is installed with CUDA support, this starts NVTX range.

    Check torch.cuda.nvtx.range_push's document for more details.
    N)r)   cudais_availablenvtx
range_pushrH   s    r*   _nvtx_range_pushrO      s/    
 zz 

""4( !rF   c                      t         j                  j                         r)t         j                  j                  j	                          yy)zIf PyTorch is installed with CUDA support, this terminates NVTX range.

    Check torch.cuda.nvtx.range_pop's document for more details.
    N)r)   rJ   rK   rL   	range_poprE   rF   r*   _nvtx_range_poprR      s-    
 zz 

!!# !rF   device_typec                     ddl m} | dk(  r|j                  j                         S | dk(  r|j                  j	                         S | dk(  r|j                  j                         S t        d| z         )Nr   r   rJ   cpumaiazUnsupported device type: )onnxruntime.capir   	OrtDevicerJ   rU   npu
ValueError)rS   ORTCs     r*   _get_ort_device_typer\      sg    6f~~""$$e~~!!##f~~!!##
0;>
??rF   c                        e Zd ZdZdee   deeef   f fdZde	ee
j                  j                  f   de
j                  j                  def fdZ xZS )	OrtOperatorSupporta0  Operator support for ONNXRuntime backend.

    It has two-level of support decision. One is via support_dict and the other one
    is via extra_support_dict. The logic of using support_dict is implemented in
    OrtOperatorSupport and extra_support_dict is used by OperatorSupport.is_node_supported.
    support_dictextra_support_dictc                 2    t         |   |       || _        y N)super__init___onnx_support_dict)selfr_   r`   	__class__s      r*   rd   zOrtOperatorSupport.__init__   s    
 	+,".rF   
submodulesnoder   c                    |j                   t        vry|j                   dk(  rM|j                  | j                  v r5t        j                  d|j                  t        |j                               yt        | !  ||      r5t        j                  d|j                  t        |j                               yt        j                  d|j                  t        |j                               y)NFcall_functionz0support_dict supports node.target: %s (type: %s)Tz6extra_support_dict supports node.target: %s (type: %s)zLsupport_dict and extra_support_dict don't support node.target: %s (type: %s))
opr   targetre   loggerinfotyperc   is_node_supportedwarning)rf   rh   ri   rg   s      r*   rq   z$OrtOperatorSupport.is_node_supported   s     77++77o%$++9P9P*PKKBT[[!
  7$Z6KKHT[[!
 ZKK	

 rF   )__name__
__module____qualname____doc__setr   dictr;   rd   r   r)   nnModulefxNodeboolrq   __classcell__)rg   s   @r*   r^   r^      sc    /SX /4S> /!#uxx"67?Dxx}}	 rF   r^   c                     | j                   }g }d}|j                  D ]7  }|j                  dk(  r|j                  |       |&|j                  dk7  s6|}9 |y|D ]  }|j	                  |        y)z
    In torch.fx.Graph, placeholder is a special assignment node. If it's not
    executed in the beginning, it could overwrite values computed by upstream
    nodes.
    Nplaceholder)r<   nodesrl   appendprepend)r-   r<   placeholdersfirst_not_placeholderri   r   s         r*   _move_placeholder_to_frontr      s     EL  )77m#% (TWW-E$(!	)
 $# 3%%k23rF   .c                      g }| D ]]  }t        |d      s|j                  }|j                  dk(  r|j                  d       =|j                  dk(  sM|j                  d       _ t	        |      S )zBReturn the first valid device (i.e., GPU or CPU) in argument list.devicerJ   CUDAExecutionProviderrU   rD   )hasattrr   rp   r   tuple)argsepsargr   s       r*   _infer_ep_from_devicer      sg    
C 33!ZZF{{f$

23%

123 :rF   c                    g }| j                   j                  D ]f  }|j                  dk(  st        |d      r7d|j                  v r)t        |j                  d   t        j                        sJ |j                  |       h t        |      S )Nr   metaval)
r<   r   rl   r   r   
isinstancer)   Tensorr   r   )r-   r   ri   s      r*   _extract_graph_module_inputsr      s{    L""(( &77m#tV$$)));!$))E"2ELLAAA%	&
 rF   c                     | j                   j                  D ]"  }|j                  dk(  s|j                  d   c S  t	        d      )zHCollect "val" fields from outputs metadata in this torch.fx.GraphModule.outputr   z2No output node found in this torch.fx.GraphModule.)r<   r   rl   r   rZ   )r-   ri   s     r*   _extract_graph_module_outputsr     sG    ""((  77h 99Q<	 
 I
JJrF   c                     t        j                  t        |             \  }}|D cg c]+  }t        |d      rd|j                  v r|j                  d   - }}t        | S c c}w )z[Return the all valid devices (i.e., GPU or CPU) among outputs of this torch.fx.GraphModule.r   r   )r   tree_flattenr   r   r   r   )r-   flattened_output_args_
output_argselected_output_argss        r*   _infer_ep_from_graph_moduler     ss    &33%l3 1 0 J'EZ__,D	 	  !"677s   0A r   c                 f    dt         dt        fd}t        |       }t        t	        ||d            S )z:Sort execution providers in eps based on pre-set priority.epr   c                     | dk(  ry| dk(  ryy)NrD      r   r2   r   rE   )r   s    r*   get_execution_provider_priorityz2_sort_eps.<locals>.get_execution_provider_priority!  s    ''(( rF   T)keyreverse)r;   intrw   r   sorted)r   r   
unique_epss      r*   	_sort_epsr     s6    	C 	C 	 SJ
(GQUVWWrF   valueszORTC.OrtDevice.c           	      P   ddl m dt        dt        fddt        t        j
                  t        j                  t        t        j                  t        t        j                  t        f   dt        ffdt        |       dkD  rt        fd| D              }|S  d	      fS )
Nr   r   	device_idr   c                     | xs dS )Nr   rE   )r   s    r*   _device_id_or_zeroz-_get_onnx_devices.<locals>._device_id_or_zero:  s    ~ArF   valuec           	      8   t        | t        j                        rc j                  t	        | j
                  j                        j                  j                          | j
                  j                              S t        | t        j                  t        t        j                  t        t        j                  t        f      r5 j                  t	        d      j                  j                         d      S t        dt!        t        |             z         )NrU   r   zUnsupported value type: )r   r)   r   rX   r\   r   rp   default_memoryindexSymIntr   SymFloatfloatSymBoolr}   rZ   r;   )r   r[   r   s    r*   _map_tensor_or_sym_to_devicez7_get_onnx_devices.<locals>._map_tensor_or_sym_to_device=  s    
 eU\\*!4>>$U\\%6%67--/"5<<#5#56 
 ELL#u~~uemmTR
 "4>>$U+T^^-J-J-La  7#d5k:JJKKrF   c              3   .   K   | ]  } |        y wrb   rE   ).0r   r   s     r*   	<genexpr>z$_get_onnx_devices.<locals>.<genexpr>R  s     TE8?Ts   r2   )rW   r   r   r
   r)   r   r   r   r   r   r}   lenr   )r   ort_devicesr[   r   r   s     @@@r*   _get_onnx_devicesr   0  s     7c c LLL%,,U^^UEMMSWW
L 
	L( 6{QTVTT,Q/11rF   tensorsdevicesc                 D   dd l }ddlm} t        j                  |j                  t        j
                  |j
                  t        j                  |j                  t        j                  |j                  t        j                  |j                  t        j                  |j                  t        j                  |j                  t        j                  |j                  t        j                  |j                  i	} |j                         }|j!                  t#        |              g }g }g }| D ]^  }	|j%                  ||	j&                            |j%                  |	j)                                |j%                  |	j+                                ` |j-                  | ||||       |S )Nr   r   )numpyrW   r   r)   float16float32float64uint8int8int16int32int64longlongr}   bool_OrtValueVectorreserver   r   dtypesizedata_ptrpush_back_batch)
r   r   npr[   torch_dtype_to_numpy_dtype	ortvaluesdtypesshapes	data_ptrstensors
             r*   !_get_ortvalues_from_torch_tensorsr   X  s&    6 	rzzrzzrzzRXX

BGGRXXRXXR[[

BHH
" $##%Ic'l#FFI ,0>?fkkm$*+, gy&&'JrF   r   c                     | j                   rt        d      t        j                  | j	                         | j
                  | j                        }|S )Nz#sparse tensor is not yet supported.)r   r   )	is_sparserZ   r)   emptyr   r   r   )r   outs     r*   _to_real_tensorr   x  s<    >??
++fkkm6<<
NCJrF   dynamo_value
value_infoonnx.ValueInfoProtoc                    t        | t        j                        rZt        |j                  j
                  j                  j                        dk(  r$| j                  dk(  rt        j                  |       S t        | t              r%t        j                  | t        j                        S t        | t              r%t        j                  | t        j                        S t        | t              r%t        j                  | t        j                        S t        | t        j                        sJ | j                         S )z9Helper function to wrap PyTorch variables as torch.Tensorr   )r2   )r   )r   r)   r   r   rp   tensor_typeshapedimsqueezer   r   r   r   r   r}   
contiguous)r   r   s     r*   _adjust_scalar_from_fx_to_onnxr     s     	<.
++11556!;$& }}\**	L#	&||L<<	L%	(||L>>	L$	'||L

;;,555&&((rF   
prim_valuec           	          t        | t        j                        sJ d       t        |t        j                  t        t        j
                  t        t        j                  t        f      r| j                         S | S )zFHelper function to wrap ORT-produced torch.Tensor as PyTorch variableszORT's output must be tensor.)
r   r)   r   r   r   r   r   r   r}   item)r   r   s     r*   _adjust_scalar_from_onnx_to_fxr     sX    $ fell+K-KK+	sENNE5==$G
 {{}MrF   sessonnxruntime.InferenceSessioninput_namesinputsinput_devicesoutput_namesoutputsoutput_devicespreallocate_outputinput_value_infosr   .normalized_prim_outputsc
                    dd l }
ddlm} t        d       t	        d t        ||      D              }t                t        d       t        ||      }|rt	        d |D              }t        ||      }n |j                         }t                t        d        |
j                         }|j                  dd	       | j                  ||||||       t                |r3t        d
       t	        d t        |	      D              }t                |S dd l}
t        d
       |
j                  j                  j                  j!                  |      }t	        d t        ||	      D              }t                |S )Nr   r   r   c              3   :   K   | ]  \  }}t        ||        y wrb   r   r   r   r   s      r*   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  #      C 	'sJ7   r   c              3   V   K   | ]!  }t        |t              rt        |      n| # y wrb   )r   r   r   )r   ts     r*   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  s(      
GH*Q
";OAB
s   ')run_with_ortvaluevector'disable_synchronize_execution_providers1zafter run_with_ortvaluevectorc              3   :   K   | ]  \  }}t        ||        y wrb   r   r   onnx_outputprim_outputs      r*   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  #      
([ +;D
r   c              3   :   K   | ]  \  }}t        ||        y wrb   r  r  s      r*   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  r  r   )r   rW   r   rO   r   ziprR   r   r   
RunOptionsadd_run_config_entryr  onnxruntime.trainingtraining	ortmodule_utils_ortvalues_to_torch_tensor)r   r   r   r   r   r   r   r   r   r   r   r[   
ort_inputspth_outputsort_outputsrun_optionss                   r*   %_run_onnx_session_with_ortvaluevectorr    s{   " 6\" "6+<= F &'26=IJ
  
LS
 
 8^T)d))+./(+((*K$$%NPST  [*lK  89  
,/=T,U
 
 	# 	89!**44;;VV
  
,/=T,U
 
 	rF   c
           	      V   dd l }
t        d t        ||      D              }t        ||      D ci c]=  \  }}||
j                  j	                  |j                         j                               ? }}}| j                  ||      }t        d t        ||	      D              }|S c c}}w )Nr   c              3   :   K   | ]  \  }}t        ||        y wrb   r   r   s      r*   r   z/_run_onnx_session_with_fetch.<locals>.<genexpr>!  r   r   c              3   `   K   | ]&  \  }}t        t        j                  |      |       ( y wrb   )r   r)   
from_numpy)r   r   r
  s      r*   r   z/_run_onnx_session_with_fetch.<locals>.<genexpr>*  s4      
 E;	 	'U#	
s   ,.)r   r   r  OrtValueortvalue_from_numpyrU   r   run)r   r   r   r   r   r   r   r   r   r   r   rH   r   feedr  r  s                   r*   _run_onnx_session_with_fetchr"    s    "  "6+<= F  V4D& 	k""66vzz|7I7I7KLLD  ((<.K 
 #&k3J"K K s   AB%rp   c                     ddl }t        |j                  j                  t        |j                  j
                  t        |j                  j                  i}|j                  |       S )a=  
    Converts a Python type to the corresponding ONNX tensor element type.
    For example, `_from_python_type_to_onnx_tensor_element_type(float)` returns
    `onnx.TensorProto.FLOAT`.

    Args:
      type (type): The Python type to convert.

    Returns:
      int: The corresponding ONNX tensor element type.

    r   N)	onnxr   TensorProtoFLOATr   INT64r}   BOOLr8   )rp   r$  (_PYTHON_TYPE_TO_ONNX_TENSOR_ELEMENT_TYPEs      r*   -_from_python_type_to_onnx_tensor_element_typer*  4  sW      	t%%T##d##0,
 477==rF   c                       e Zd ZdZdddeedf   ded   deedf   d	ed   d
ed   ded   deeej                  df   ej                  f   fdZ	d Z
y)OrtExecutionInfoPerSessionzWInformation required to execute torch.fx.GraphModule using onnxruntime.InferenceSessionsessionr   r   .r   r   r   output_value_infosr   r   r   example_outputsc	                 t    || _         || _        || _        || _        || _        || _        || _        || _        y rb   r-  r   r   r   r.  r   r   r/  )	rf   r-  r   r   r   r.  r   r   r/  s	            r*   rd   z#OrtExecutionInfoPerSession.__init__N  sM     6= -8BS-9CU :G:H  	rF   c                    dd l }|j                  j                  t        j                  |j                  j
                  t        j                  |j                  j                  t        j                  |j                  j                  t        j                  |j                  j                  t        j                  |j                  j                  t        j                  |j                  j                  t        j                   |j                  j"                  t        j$                  |j                  j&                  t        j(                  |j                  j*                  t        j,                  |j                  j.                  t        j0                  |j                  j2                  t        j4                  |j                  j6                  t        j8                  i}|j;                         D ci c]  \  }}||
 }}}t=        |      t=        | j>                        k7  rytA        || j>                        D ]  \  }}tC        |t        jD                  tF        tH        f      s ytC        |tH        tF        t$        f      rrtK        tM        |            }	|	|jL                  jN                  jP                  k7  r yt=        |jL                  jN                  jR                  jT                        dk7  r y||jV                     }	|	|jL                  jN                  jP                  k7  r ytA        |jR                  |jL                  jN                  jR                  jT                        D ]Z  \  }
}tC        |
tH              r|jX                  |
k(  s|jZ                  r2tC        |
t        j\                        r|jZ                  rY  y  yc c}}w )Nr   FT)/r$  r%  r&  r)   r   FLOAT16r   
FLOAT8E5M2float8_e5m2FLOAT8E5M2FNUZfloat8_e5m2fnuzFLOAT8E4M3FNfloat8_e4m3fnFLOAT8E4M3FNUZfloat8_e4m3fnuzDOUBLEr   r(  r}   UINT8r   INT8r   INT16r   INT32r   r'  r   itemsr   r   r  r   r   r   r   r*  rp   r   	elem_typer   r   r   	dim_value	dim_paramr   )rf   r   r$  (_onnx_tensor_element_type_to_torch_dtyper   r   (_torch_dtype_to_onnx_tensor_element_typer   r   
onnx_dtyper   onnx_dims               r*   is_supportedz'OrtExecutionInfoPerSession.is_supportedo  s    ""EMM$$emm''):):++U-B-B))5+>+>++U-B-B##U]]!!5::""EKK!!5::""EKK""EKK""EKK4
0" GLLN4
U 3J4
0 4
 t9D2233"4)?)?@ 	!OCcELL%#=> #UD12J4PS9U
!<!<!F!FF z2288<<=B  B#))LJZ__88BBB!$SYY
0K0K0Q0Q0U0U!V !Xc3'&&#-1C1CU\\2x7I7I !%	!6 I4
s   .ON)rs   rt   ru   rv   r   r;   r
   r)   r   rd   rI  rE   rF   r*   r,  r,  K  s    a
/
 38_
 !!;<	

 CHo
 ""<=
 23
 34
 uU\\3%67EF
B7rF   r,  c                       e Zd ZddZdej
                  j                  fdZdej
                  j                  defdZ	y)"OrtExecutionInfoForAllGraphModulesNc                     i | _         y rb   )execution_info_per_graph_module)rf   s    r*   rd   z+OrtExecutionInfoForAllGraphModules.__init__  s    
  	,rF   r-   c                 t    || j                   vry | j                   |   }|D ]  } |j                  | s|c S  y rb   )rM  rI  )rf   r-   r   
candidates	candidates        r*   &search_reusable_session_execution_infozIOrtExecutionInfoForAllGraphModules.search_reusable_session_execution_info  sQ     tCCC 99,G
# 	!I%y%%t,  	!
 rF   ro   c                 ~    || j                   vr|g| j                   |<   y | j                   |   j                  |       y rb   )rM  r   )rf   r-   ro   s      r*   cache_session_execution_infoz?OrtExecutionInfoForAllGraphModules.cache_session_execution_info  s=     tCCCBFD00>00>EEdKrF   )r   N)
rs   rt   ru   rd   r)   r{   GraphModulerQ  r,  rS  rE   rF   r*   rK  rK    s@    !HH00 L!HH00L8RLrF   rK  r   T)frozenF)is_backward_compatiblec                       e Zd ZU dZdZeee      ed<   	 dZ	e
ed<   	 dZeee      ed<   	 dZe
ed<   	 dZe
ed	<   	 dZed
   ed<   	 dZeeedgdf         ed<   y)r   aJ  Options for constructing an ``OrtBackend``, the ONNX Runtime
    backend (``"onnxrt"``) for ``torch.compile``.

    Example::

        >>> @torch.compile(
        ...     backend="onnxrt",
        ...     options=torch.onnx._OrtBackendOptions(...),
        ... )
        ... def ort_function(x):
        ...     return x ** x
    Npreferred_execution_providersTinfer_execution_providersdefault_execution_providersFr   use_aot_autogradzonnxruntime.SessionOptionsort_session_optionszonnx.ModelProtopre_ort_model_transforms)rs   rt   ru   rv   rX  r   r   r   __annotations__rY  r}   rZ  r   r[  r\  r]  r   rE   rF   r*   r   r     s     OS!8H5I,J#KR '+t*mLP(3G*H!IP  %$V!d! CG">?FV 	 h,-t345 1rF   r   c            	          e Zd ZU dZddee   fdZdej                  j                  de
eeeeef   f      fdZdej                  j                  fdZdej                  j                  dej                  j                  fd	Zdej                  j                  dej                  j                  fd
ZdZeed<   g Zeed       ed<   e	 ddeeeeeef   f      dd fd       Zed        Zed        Zy)r   a	  A backend compiles (sub-)graphs in torch.fx.GraphModule to onnxruntime.InferenceSession calls.

    The compiler entry point is OrtBackend.compile, which
        1. partitions the original graph into supported sub-graphs (type: torch.fx.GraphModule) and unsupported
           sub-graphs.
        2. For each supported sub-graph, it replaces its _wrapped_call function with _ort_accelerated_call.
        3. Inside _ort_accelerated_call, it creates onnxruntime.InferenceSession and calls it to execute the sub-graph.
    Noptionsc                 "   ddl m} dd l}dd l}dd l}|
t               n|| _        |j                  j                  j                  j                         | _        |j                  j                  j                  j                  j                  | j                  j                        }d d d d d d}t!        ||      | _        i | _        t'               | _        d| _        d| _        t/        |j0                  d      rt2        | _        y t4        | _        y )Nr   r   )getattrz_operator.getitemz_operator.mulz_operator.addz_operator.subFr   )rW   r   r$   r&   +torch.onnx._internal.fx.decomposition_tabler   _optionsr$  	_internal_exporter_legacyResolvedExportOptions_resolved_onnx_exporter_optionsr{   r   '_create_onnx_supports_op_overload_tableonnx_registryr^   _supported_ops_partitioner_cacherK  _all_ort_execution_info_assert_allclose_to_baselineexecution_countr   r   r  r"  r   )rf   r`  r[   r)   r_   r`   s         r*   rd   zOrtBackend.__init__!  s    :4:6=o027  JJ  11GGI 	, zz++..BBjj00>>

  "& "!!
.
 1?QR UW (J'K$,1) 
 t**,=> 2 	 . 	rF   r-   r   c                    d}| j                   j                  rt        | x}r|}nt        |      x}r|}g }g | j                   j                  xs g t        |      | j                   j                  xs
 t               D ]L  }t        |t              r|i f}nt        |t              r|d   |d   i f}|7||vs<|j                  |       N |S )NrE   r2   r   )rd  rY  r   r   rX  r   rZ  rG   r   r;   r   r   )rf   r-   r   inferred_epseps_from_argseps_from_graph_moduleselected_epsr   s           r*   _select_epszOrtBackend._select_epsu  s     )+==22 5t <<}<
  -*El*SS&S  5
mm99?R
|$
 mm77O;M;O
 
	(B
 "c""XB&2a5=eR[~"L"8##B'
	( rF   c                 	   ddl }ddlm}m}  | j                  j
                  |g| }|rb|j                  }|j                  }	|j                  }
|j                  }|j                  }|j                  }|j                  }|j                  }n|j                  |      j                         }| j                   j"                  r,d| _        t'        |      }d }t)        j*                  ||      }n	  t-        |      j.                  |i |}|j7                         }|j9                  |      j                         }|j                  || j                   j:                        }|j=                  | j                   j>                  j@                        }| jB                  jD                  r#| jB                  jD                  D ]
  } ||        |jG                         }tH        jJ                  jM                  d	d      rtO        ||
        |jP                  || jB                  jR                   | jT                  |g|       }tW        d |jX                  jZ                  D              }	tW        d |jX                  j\                  D              }
t_        |      }ta        |tV              rt_        |      }nt_        |f      }tW        d |jX                  jZ                  D              }tW        d |jX                  j\                  D              }tc        ||	||
||||      }| j                  je                  ||       | xjf                  dz  c_3        ta        |th        jj                        }|r|fn|}ta        |tV              sJ tm        d |D              sJ to        d       | j                  ||	|||
||| jB                  j$                  ||
      }tq                | jr                  rhti        jt                  jv                  jx                  |g|ddi}|r|fn|}t{        ||      D ]%  \  }}th        j|                  j                  ||       ' |r|d   S |S # t0        $ r t2        j5                  d|       d| _         w xY w)a  This function replaces GraphModule._wrapped_call in compiled model.

        The _wrapped_call is the underlying implementation of forward method. Replacing
        it means we delegate the computation to _ort_acclerated_call and therefore
        onnxruntime.InferenceSession.
        r   N)r   r    Fc                 X    t        | d      rd| j                  v r| j                  d   S | S )Nr   r   )r   r   )r   s    r*   maybe_map_to_meta_valz>OrtBackend._ort_acclerated_call.<locals>.maybe_map_to_meta_val  s-    uf-%5::2E  %zz%00$rF   zFakeTensorProb failed for %s)fx_graph_moduleonnxfunction_dispatcher)opset_versionr/   )r-   )path_or_bytessess_options	providersc              3   4   K   | ]  }|j                     y wrb   rN   r   inputs     r*   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s     Ou

O   c              3   4   K   | ]  }|j                     y wrb   rN   r   r   s     r*   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s      S Sr  c              3       K   | ]  }|  y wrb   rE   r  s     r*   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s     %Pe%P   c              3       K   | ]  }|  y wrb   rE   r  s     r*   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s     &T&v&Tr  r1  r2   c              3   z   K   | ]3  }t        |t        j                  t        j                  t        f       5 y wrb   )r   r)   r   r   r   )r   elems     r*   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>7  s.      
 tellELL#>?
s   9;$run_onnx_session_with_ortvaluevectorexecutoraten)@r   r'   r   r    rm  rQ  r-  r   r   r   r.  r   r   r/  MovePlaceholderToFrontr   rh  dynamic_shapesr   r   r   tree_mapr   	propagate	Exceptionrn   rr   FxOnnxInterpreterInsertTypePromotionrz  to_model_protorj  r{  rd  r]  SerializeToStringr6   r7   r8   rB   InferenceSessionr\  ru  r   r<   r  r   r   r   r,  rS  ro  r)   r   allrO   rR   rn  _primsr  executer  testingassert_close)rf   r-   r   kwargsr   r   r    !cached_execution_info_per_sessiononnx_sessionr   r   r   r.  r   r   prim_outputsextracted_outputsrx  fx_interpreterexported
onnx_model	transformonnx_model_bytesexecution_info_per_sessionis_single_tensor_outputr   onnx_outputsbaseline_outputsnormalized_baseline_ouptutsr	  baseline_outputs                                  r*   _ort_acclerated_callzOrtBackend._ort_acclerated_call  s    	G PD((OO# 	*
 -<DDL;GGK<IIL A S S!B!U!U=KKM>MMN<LLL
 "88ce  33BB*/'$A,$O!%  '//)+< #I>,#?#I#I$!'$L 1BBDN "55lCGGIL &)) ,(,(L(L(d(d * H
 "00"BBPP^^ 1 J }}55!%!G!G *Ij)*  *;;=zz~~0$7 !!1M 8;77.!]]>>*$**<?$?L  O
8H8H8N8NOOK  S:;K;K;R;R SSL-d3M ,.!2<!@!2L?!C %%P9I9I9O9O%P P!&&TJ<L<L<S<S&T!T)C$'"3)#5+- ,	*& ((EE8 	!
 #-\5<<"H6\OL 	  15999 
/
 
 	
 

 	?@xx#MM,,#
 	,,$||44<< # .4  (?!#DT ( 1491 I,_ **;HI #:|AK|KO ! NN#A<P /4D+ s   0Q (Q?c                 ~   ddl m} || j                  v r| j                  |   }|S |} ||| j                  d      }|j	                         }|| j                  |<   |j
                  j                  D ]H  }|j                  dk(  sd|j                  v s"t        ||j                        }| j                  |_        J |S )Nr   )CapabilityBasedPartitionerT)allows_single_node_partitioncall_modulefused_)!torch.fx.passes.infra.partitionerr  rl  rk  partition_and_fuser<   r   rl   rH   rb  r  _wrapped_call)	rf   r-   r   r  partitioned_prim_graph_moduleprim_graph_modulepartitionerri   fused_modules	            r*   compilezOrtBackend.compileZ  s    
 	Q, 4222,0,C,CL,Q)0 -,- !-4!##-1K
 -8,J,J,L)4QD##L1 6;;AA K 77m+DII0E#*+H$))#TL 261J1JL.K -,rF   c                     | j                   j                  r<ddlm} ddlm}   || j                  || j                  j                        ||      S | j                  ||      S )zIf ``OrtBackendOptions.use_aot_autograd`` is ``True``, the `auto_autograd` compiler
        will be invoked, wrapping this ``OrtBackend`` instance's ``compile`` method. Otherwise,
        the ``compile`` method is invoked directly.r   )#min_cut_rematerialization_partition)aot_autograd)fw_compilerpartition_fndecompositions)	rd  r[  functorch.compiler  torch._dynamo.backends.commonr  r  rh  r   )rf   r-   r   r  r  s        r*   __call__zOrtBackend.__call__  s`     ==))MB< LL@#CCWW D	" " ||L$//rF      %_OrtBackend__instance_cache_max_count_OrtBackend__instance_cachec                     dt         dt         fdt         t               st        d
i  xs i  t         fdt        j                  D        d      }|{t        t        j                        t        j                  k  s'J dt        j                   dt         dt         d	       t        j                  j                  t               x}       |S )a  Returns a possibly cached instance of an ``OrtBackend``. If an existing
        backend was created previously through this function with the same options,
        it will be returned. Otherwise a new backend will be created, cached, and
        returned.

        Note: if ``options`` sets ``ort_session_options``, a new ``OrtBackend``
        will always be returned, since ``onnxruntime.SessionOptions`` cannot
        participate in caching.abc                 d   | j                   |j                   k7  s}| j                  |j                  k7  sd| j                  |j                  k7  sK| j                  |j                  k7  s2| j                  |j                  k7  s| j
                  |j
                  k7  ry| j                  |j                  yy)NFT)rX  rY  rZ  r   r[  r]  r\  )r  r  s     r*   reusablez<OrtBackend.get_cached_instance_for_options.<locals>.reusable  s    //13R3RR..!2M2MM00A4Q4QQ''1+?+??%%););;--1K1KK $$0A4I4I4UrF   c              3   J   K   | ]  } |j                         s|  y wrb   )rd  )r   r  r`  r  s     r*   r   z=OrtBackend.get_cached_instance_for_options.<locals>.<genexpr>  s     U1x

G7TQUs   ##NzNo more than z instances of z allowed. Please instantiate `z` explicitly to pass to `torch.compile`. See https://github.com/pytorch/pytorch/pull/107973#discussion_r1306144795 for discussion.rE   )r   r   nextr   r  r   r  r   )r`  backendr  s   ` @r*   get_cached_instance_for_optionsz*OrtBackend.get_cached_instance_for_options  s    	) 	.? 	( '#45':7=b:GU
33U

 ?J//0:3X3XX  
 E EFn,<ZL I""X ''..*W:M/MwNrF   c                  @    t         j                  j                          y rb   )r   r  clearrE   rF   r*   clear_cached_instancesz!OrtBackend.clear_cached_instances  s    ##))+rF   c                  4    t        t        j                        S rb   )r   r   r  rE   rF   r*   get_cached_instanceszOrtBackend.get_cached_instances  s    Z0011rF   rb   )rs   rt   ru   rv   r   r   rd   r)   r{   rT  r   r   r;   r   r   ru  r  r  r  r  r   r^  r  liststaticmethodr
   r  r  r  rE   rF   r*   r   r     sI   R
): ; R
h!HH00	%WS#X../	0BBL1E1E BLH4-EHH$8$8 4-588CWCW 4-l0!HH000			0$ )*)24eD./4IM4% 17383D DEF4	4 4l , , 2 2rF   r   )r`  r`  c                8    t        j                  |      | |      S rb   )r   r  )r-   r   r`  s      r*   r   r     s     55g>|TRRrF   rb   )]dataclassesr"   loggingr6   collections.abcr   r   typingr   r   r   r   r	   r
   typing_extensionsr   r)   torch._C
torch._opstorch._prims.executortorch.fx!torch.onnx._internal._lazy_importtorch._subclasses.fake_tensorr   torch.fx._compatibilityr    torch.fx.passes.fake_tensor_propr    torch.fx.passes.operator_supportr   torch.fx.passes.tools_commonr   torch.utilsr   r$  r   rW   r   r[   r$   r%   r&   rc  torch.onnx._internal.fx.passesr   r}   r^  __all__r   r+   rx   r;   r   bytesr{   rT  rB   rG   rO   rR   r\   	getLoggerrs   rn   r^   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r"  rp   r*  r,  	dataclassrK  r   r   r   r   rE   rF   r*   <module>r     sf      	 - G G '      ( 4 1 ; < :  606) #'$ &-T -` &( DcN ' IM'/0D0D'E2$HSM $)3 )$
@c 
@ 
		8	$/ /d3UXX-A-A 3d 3*
E#s(O 
uxx/C/C cSVh K0D0D K K8ehh.B.B 8uSRUX 8 X5c? XuS#X X$%2LL%,,U^^UEMMSWW	
 		%2  !%2P5<<$%056K0L
5<<@J 5<< )	) &) \\)@LL	 	LL	
8P
(PsCxP %,,#$P ./	P
 S/P 5<<$%P /0P P 78P #LL%,,U^^UEMMSWW	
 		P  5sE4/0#56!Pf#
(#sCx# %,,#$# ./	#
 S/# 5<<$%# /0# # 78# #LL%,,U^^UEMMSWW	
 		#  5sE4/0#56!#L> >.[ [| L L LB #(U3S8I3I-J(J"K i K	 d#e,91 91 - $91x e,K2 K2 -K2\ e,
 FJ	S((&&S e-wsCx/@@AB	S -SrF   