
    ¯wg(                     	   U d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d d	l"m#Z# er&d dl$Z$d dl%Z%d d
l&m'Z( d dl)Zd dl*Zd dl+Zd dl,Zd dl-Zd dl.Zda/ee0   e1d<   g dZ2de0fdZ3i Z4ee5e6f   e1d<   	 dOde7deejp                  jr                     de5fdZ:dee5   fdZ;de5fdZ<d Z=de5fdZ> ej~                  e@      ZA G d de      ZBdejp                  jr                  ddfdZCdee5df   fdZDdejp                  jr                  deedf   fdZEdejp                  jr                  defdZFdejp                  jr                  dee5df   fd ZGd!ee5df   dee5df   fd"ZHd#eeej                  ej                  e6ej                  eLej                  e0f   df   ded$   fd%ZNd&eej                  df   d'ed$   deej                  df   fd(ZOd)edej                  fd*ZPd+eej                  e6eLe0f   d,d-dej                  fd.ZQd)ej                  d/eej                  ej                  e6ej                  eLej                  e0f   deej                  e6eLe0f   fd0ZRd1d2d3ee5df   d4eej                  df   d5ed$   d6ee5df   d7eej                  df   d8ed$   d9e0d:ed;   d<eeej                  ej                  e6ej                  eLej                  e0f   df   deeej                  e6eLe0f   df   fd=ZSd1d2d3ee5df   d4eej                  df   d5ed$   d6ee5df   d7eej                  df   d8ed$   d9e0d:ed;   d<eeej                  ej                  e6ej                  eLej                  e0f   df   deeej                  e6eLe0f   df   fd>ZT G d? d@      ZUe j                   G dA dB             ZWee5ee5e
e5ef   f   f   ZXee1dC<   	  e j                  dDE       edFG       G dH dI                    ZY edFG       G dJ dK             ZZ edFG      ddLdejp                  jr                  dMeeeYe
e5ef   f      fdN       Z[y)P    N)AnyCallableDictFinalListMappingOptionalSequenceSetTupleTYPE_CHECKINGUnion)	TypeAlias)
FakeTensor)compatibility)FakeTensorProp)OperatorSupport)CALLABLE_NODE_OPS)_pytree_pybind_state_SUPPORT_ONNXRT)is_onnxrt_backend_supportedtorch_compile_backendOrtExecutionProviderOrtBackendOptions
OrtBackendreturnc                     t         d	 t        j                  d       t        j                  d       t        j                  d       ddl} ddl} ddl} ddl} ddlm}m	}m
}m} da t         S t         S # t        $ r
 da Y t         S w xY w)	a!  Returns ``True`` if ONNX Runtime dependencies are installed and usable
    to support TorchDynamo backend integration; ``False`` otherwise.

    Example::

        # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ONNX)
        >>> import torch
        >>> if torch.onnx.is_onnxrt_backend_supported():
        ...     @torch.compile(backend="onnxrt")
        ...     def f(x):
        ...             return x * x
        ...     print(f(torch.randn(10)))
        ... else:
        ...     print("pip install onnx onnxscript onnxruntime")
        ...
    Nonnxruntimezonnxruntime.capi._pybind_state
onnxscriptr   )decomposition_tablefx_onnx_interpreterpasses
type_utilsTF)r   	importlibimport_module
torch.onnxtorch.onnx._internal%torch.onnx._internal._exporter_legacy torch.onnx._internal.diagnosticstorch.onnx._internal.fxr"   r#   r$   r%   ImportError)torchr"   r#   r$   r%   s        e/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/torch/onnx/_internal/onnxruntime.pyr   r   ;   s    & 	$##M2##$DE ##L1'83  #O ?  	$#O	$s   AA1 1BB_dumped_onnx_modelmodel_stringgraph_modulec                    t         j                  j                  dd      }|syt        j                  |d      dz   }| | d}t	        |d      5 }|j                  |        ddd       |t        |<   |D| | d}t	        |d	d
      5 }|j                  t        |j                               ddd       |S |S # 1 sw Y   ZxY w# 1 sw Y   |S xY w)a  Stores the onnx model into a file.
    The name is "{ONNXRT_DUMP_PATH}{N}.onnx"
    where *N* is the number of files already stored with
    this prefix.
    If graph_module is not None, the graph is stored as a string with
    the same filename except the extension (.txt).
    ONNXRT_DUMP_PATHN    z.onnxwbz.txtwzutf-8)encoding)osenvirongetr0   openwritestrgraph)r1   r2   prefixnfilenameffilename_txts          r/   _dump_onnx_modelrG   o   s     ZZ^^.5Fvr*Q.A!E"H	h	 	!"v !D),g6 	-!GGC**+,	-O8O 
	-Os   B;
%C;CCc                      dgS )NCPUExecutionProvider rJ       r/   _infer_default_epsrL      s     ###rK   namec                     t         j                  j                         r*t         j                  j                  j	                  |        yy)zIf PyTorch is installed with CUDA support, this starts NVTX range.

    Check torch.cuda.nvtx.range_push's document for more details.
    N)r.   cudais_availablenvtx
range_pushrM   s    r/   _nvtx_range_pushrT      s/    
 zz 

""4( !rK   c                      t         j                  j                         r)t         j                  j                  j	                          yy)zIf PyTorch is installed with CUDA support, this terminates NVTX range.

    Check torch.cuda.nvtx.range_pop's document for more details.
    N)r.   rO   rP   rQ   	range_poprJ   rK   r/   _nvtx_range_poprW      s-    
 zz 

!!# !rK   device_typec                     ddl m} | dk(  r|j                  j                         S | dk(  r|j                  j	                         S | dk(  r|j                  j                         S t        d| z         )Nr   r   rO   cpumaiazUnsupported device type: )onnxruntime.capir   	OrtDevicerO   rZ   npu
ValueError)rX   ORTCs     r/   _get_ort_device_typera      sg    6f~~""$$e~~!!##f~~!!##
0;>
??rK   c                        e Zd ZdZdee   deeef   f fdZde	ee
j                  j                  f   de
j                  j                  def fdZ xZS )	OrtOperatorSupporta0  Operator support for ONNXRuntime backend.

    It has two-level of support decision. One is via support_dict and the other one
    is via extra_support_dict. The logic of using support_dict is implemented in
    OrtOperatorSupport and extra_support_dict is used by OperatorSupport.is_node_supported.
    support_dictextra_support_dictc                 2    t         |   |       || _        y N)super__init___onnx_support_dict)selfrd   re   	__class__s      r/   ri   zOrtOperatorSupport.__init__   s    
 	+,".rK   
submodulesnoder   c                    |j                   t        vry|j                   dk(  rM|j                  | j                  v r5t        j                  d|j                  t        |j                               yt        | !  ||      r5t        j                  d|j                  t        |j                               yt        j                  d|j                  t        |j                               y)NFcall_functionz0support_dict supports node.target: %s (type: %s)Tz6extra_support_dict supports node.target: %s (type: %s)zLsupport_dict and extra_support_dict don't support node.target: %s (type: %s))
opr   targetrj   loggerinfotyperh   is_node_supportedwarning)rk   rm   rn   rl   s      r/   rv   z$OrtOperatorSupport.is_node_supported   s     77++77o%$++9P9P*PKKBT[[!
  7$Z6KKHT[[!
 ZKK	

 rK   )__name__
__module____qualname____doc__r   r   r   r@   ri   r   r.   nnModulefxNodeboolrv   __classcell__)rl   s   @r/   rc   rc      sc    /SX /4S> /!#uxx"67?Dxx}}	 rK   rc   c                     | j                   }g }d}|j                  D ]7  }|j                  dk(  r|j                  |       |&|j                  dk7  s6|}9 |y|D ]  }|j	                  |        y)z
    In torch.fx.Graph, placeholder is a special assignment node. If it's not
    executed in the beginning, it could overwrite values computed by upstream
    nodes.
    Nplaceholder)rA   nodesrq   appendprepend)r2   rA   placeholdersfirst_not_placeholderrn   r   s         r/   _move_placeholder_to_frontr      s     EL  )77m#% (TWW-E$(!	)
 $# 3%%k23rK   .c                      g }| D ]]  }t        |d      s|j                  }|j                  dk(  r|j                  d       =|j                  dk(  sM|j                  d       _ t	        |      S )zBReturn the first valid device (i.e., GPU or CPU) in argument list.devicerO   CUDAExecutionProviderrZ   rI   )hasattrr   ru   r   tuple)argsepsargr   s       r/   _infer_ep_from_devicer      sg    
C 33!ZZF{{f$

23%

123 :rK   c                    g }| j                   j                  D ]f  }|j                  dk(  st        |d      r7d|j                  v r)t        |j                  d   t        j                        sJ |j                  |       h t        |      S )Nr   metaval)
rA   r   rq   r   r   
isinstancer.   Tensorr   r   )r2   r   rn   s      r/   _extract_graph_module_inputsr     s{    L""(( &77m#tV$$)));!$))E"2ELLAAA%	&
 rK   c                     | j                   j                  D ]"  }|j                  dk(  s|j                  d   c S  t	        d      )zHCollect "val" fields from outputs metadata in this torch.fx.GraphModule.outputr   z2No output node found in this torch.fx.GraphModule.)rA   r   rq   r   r_   )r2   rn   s     r/   _extract_graph_module_outputsr     sG    ""((  77h 99Q<	 
 I
JJrK   c                     t        j                  t        |             \  }}|D cg c]+  }t        |d      rd|j                  v r|j                  d   - }}t        | S c c}w )z[Return the all valid devices (i.e., GPU or CPU) among outputs of this torch.fx.GraphModule.r   r   )r   tree_flattenr   r   r   r   )r2   flattened_output_args_
output_argselected_output_argss        r/   _infer_ep_from_graph_moduler     ss    &33%l3 1 0 J'EZ__,D	 	  !"677s   0A r   c                 f    dt         dt        fd}t        |       }t        t	        ||d            S )z:Sort execution providers in eps based on pre-set priority.epr   c                     | dk(  ry| dk(  ryy)NrI      r   r7   r   rJ   )r   s    r/   get_execution_provider_priorityz2_sort_eps.<locals>.get_execution_provider_priority.  s    ''(( rK   T)keyreverse)r@   intsetr   sorted)r   r   
unique_epss      r/   	_sort_epsr   +  s6    	C 	C 	 SJ
(GQUVWWrK   valueszORTC.OrtDevice.c           	      P   ddl m dt        dt        fddt        t        j
                  t        j                  t        t        j                  t        t        j                  t        f   dt        ffdt        |       dkD  rt        fd| D              }|S  d	      fS )
Nr   r   	device_idr   c                     | xs dS )Nr   rJ   )r   s    r/   _device_id_or_zeroz-_get_onnx_devices.<locals>._device_id_or_zeroG  s    ~ArK   valuec           	      8   t        | t        j                        rc j                  t	        | j
                  j                        j                  j                          | j
                  j                              S t        | t        j                  t        t        j                  t        t        j                  t        f      r5 j                  t	        d      j                  j                         d      S t        dt!        t        |             z         )NrZ   r   zUnsupported value type: )r   r.   r   r]   ra   r   ru   default_memoryindexSymIntr   SymFloatfloatSymBoolr   r_   r@   )r   r`   r   s    r/   _map_tensor_or_sym_to_devicez7_get_onnx_devices.<locals>._map_tensor_or_sym_to_deviceJ  s    
 eU\\*!4>>$U\\%6%67--/"5<<#5#56 
 ELL#u~~uemmTR
 "4>>$U+T^^-J-J-La  7#d5k:JJKKrK   c              3   .   K   | ]  } |        y wrg   rJ   ).0r   r   s     r/   	<genexpr>z$_get_onnx_devices.<locals>.<genexpr>_  s     TE8?Ts   r7   )r\   r   r   r   r.   r   r   r   r   r   r   lenr   )r   ort_devicesr`   r   r   s     @@@r/   _get_onnx_devicesr   =  s     7c c LLL%,,U^^UEMMSWW
L 
	L( 6{QTVTT,Q/11rK   tensorsdevicesc                 p   ddl m} ddlm}  |j                         }|j                  t        |              g }g }g }| D ]^  }|j                  ||j                            |j                  |j                                |j                  |j                                ` |j                  | ||||       |S )Nr   r   )_TORCH_DTYPE_TO_NUMPY_DTYPE)r\   r   "torch.onnx._internal.fx.type_utilsr   OrtValueVectorreserver   r   dtypesizedata_ptrpush_back_batch)	r   r   r`   r   	ortvaluesdtypesshapes	data_ptrstensors	            r/   !_get_ortvalues_from_torch_tensorsr   e  s     7N###%Ic'l#FFI ,1&,,?@fkkm$*+, gy&&'JrK   r   c                     | j                   rt        d      t        j                  | j	                         | j
                  | j                        }|S )Nz#sparse tensor is not yet supported.)r   r   )	is_sparser_   r.   emptyr   r   r   )r   outs     r/   _to_real_tensorr   z  s<    >??
++fkkm6<<
NCJrK   dynamo_value
value_infoonnx.ValueInfoProtoc                    t        | t        j                        rZt        |j                  j
                  j                  j                        dk(  r$| j                  dk(  rt        j                  |       S t        | t              r%t        j                  | t        j                        S t        | t              r%t        j                  | t        j                        S t        | t              r%t        j                  | t        j                        S t        | t        j                        sJ | j                         S )z9Helper function to wrap PyTorch variables as torch.Tensorr   )r7   )r   )r   r.   r   r   ru   tensor_typeshapedimsqueezer   r   int64r   float32r   
contiguous)r   r   s     r/   _adjust_scalar_from_fx_to_onnxr     s     	<.
++11556!;$& }}\**	L#	&||L<<	L%	(||L>>	L$	'||L

;;,555&&((rK   
prim_valuec           	          t        | t        j                        sJ d       t        |t        j                  t        t        j
                  t        t        j                  t        f      r| j                         S | S )zFHelper function to wrap ORT-produced torch.Tensor as PyTorch variableszORT's output must be tensor.)
r   r.   r   r   r   r   r   r   r   item)r   r   s     r/   _adjust_scalar_from_onnx_to_fxr     sX    $ fell+K-KK+	sENNE5==$G
 {{}MrK   sessonnxruntime.InferenceSessioninput_namesinputsinput_devicesoutput_namesoutputsoutput_devicespreallocate_outputinput_value_infosr   .normalized_prim_outputsc
                    dd l }
ddlm} t        d       t	        d t        ||      D              }t                t        d       t        ||      }|rt	        d |D              }t        ||      }n |j                         }t                t        d        |
j                         }|j                  dd	       | j                  ||||||       t                |r3t        d
       t	        d t        |	      D              }t                |S t        d
       |
j                  j                  j                  j                  |      }t	        d t        ||	      D              }t                |S )Nr   r   r   c              3   :   K   | ]  \  }}t        ||        y wrg   r   r   r   r   s      r/   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  #      C 	'sJ7   r   c              3   V   K   | ]!  }t        |t              rt        |      n| # y wrg   )r   r   r   )r   ts     r/   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  s(      
GH*Q
";OAB
s   ')run_with_ortvaluevector'disable_synchronize_execution_providers1zafter run_with_ortvaluevectorc              3   :   K   | ]  \  }}t        ||        y wrg   r   r   onnx_outputprim_outputs      r/   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  #      
([ +;D
r   c              3   :   K   | ]  \  }}t        ||        y wrg   r  r  s      r/   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  r  r   )r    r\   r   rT   r   ziprW   r   r   
RunOptionsadd_run_config_entryr   training	ortmodule_utils_ortvalues_to_torch_tensor)r   r   r   r   r   r   r   r   r   r   r    r`   
ort_inputspth_outputsort_outputsrun_optionss                   r/   %_run_onnx_session_with_ortvaluevectorr    sx   " 6\" "6+<= F &'26=IJ
  
LS
 
 8^T)d))+./(+((*K$$%NPST  [*lK  89  
,/=T,U
 
 	 	89!**44;;VV
  
,/=T,U
 
 	rK   c
           	      V   dd l }
t        d t        ||      D              }t        ||      D ci c]=  \  }}||
j                  j	                  |j                         j                               ? }}}| j                  ||      }t        d t        ||	      D              }|S c c}}w )Nr   c              3   :   K   | ]  \  }}t        ||        y wrg   r   r   s      r/   r   z/_run_onnx_session_with_fetch.<locals>.<genexpr>!  r   r   c              3   `   K   | ]&  \  }}t        t        j                  |      |       ( y wrg   )r   r.   
from_numpy)r   r   r  s      r/   r   z/_run_onnx_session_with_fetch.<locals>.<genexpr>*  s4      
 E;	 	'U#	
s   ,.)r    r   r  OrtValueortvalue_from_numpyrZ   numpyrun)r   r   r   r   r   r   r   r   r   r   r    rM   r   feedr  r  s                   r/   _run_onnx_session_with_fetchr    s    "  "6+<= F  V4D& 	k""66vzz|7I7I7KLLD  ((<.K 
 #&k3J"K K s   AB%c                       e Zd ZdZdddeedf   ded   deedf   d	ed   d
ed   ded   deeej                  df   ej                  f   fdZ	d Z
y)OrtExecutionInfoPerSessionzWInformation required to execute torch.fx.GraphModule using onnxruntime.InferenceSessionsessionr   r   .r   r   r   output_value_infosr   r   r   example_outputsc	                 t    || _         || _        || _        || _        || _        || _        || _        || _        y rg   r   r   r   r   r!  r   r   r"  )	rk   r   r   r   r   r!  r   r   r"  s	            r/   ri   z#OrtExecutionInfoPerSession.__init__7  sM     6= -8BS-9CU :G:H  	rK   c                    ddl m}m} t        |      t        | j                        k7  ryt        || j                        D ]  \  }}t        |t        j                  t        t        f      s yt        |t        t        t        f      ro |t        |            }||j                  j                  j                  k7  r yt        |j                  j                  j                  j                         dk7  r y||j"                     }||j                  j                  j                  k7  r yt        |j                  |j                  j                  j                  j                         D ]Z  \  }}t        |t              r|j$                  |k(  s|j&                  r2t        |t        j(                        r|j&                  rY  y  y)Nr   )(_TORCH_DTYPE_TO_ONNX_TENSOR_ELEMENT_TYPE,from_python_type_to_onnx_tensor_element_typeFT)r   r&  r'  r   r   r  r   r.   r   r   r   r   ru   r   	elem_typer   r   r   	dim_value	dim_paramr   )	rk   r   r&  r'  r   r   
onnx_dtyper   onnx_dims	            r/   is_supportedz'OrtExecutionInfoPerSession.is_supportedX  sd   	
 t9D2233"4)?)?@ 	!OCcELL%#=> #UD12I$s)T
!<!<!F!FF z2288<<=B  B#))LJZ__88BBB!$SYY
0K0K0Q0Q0U0U!V !Xc3'&&#-1C1CU\\2x7I7I !%	!6 rK   N)rx   ry   rz   r{   r   r@   r   r.   r   ri   r-  rJ   rK   r/   r  r  4  s    a
/
 38_
 !!;<	

 CHo
 ""<=
 23
 34
 uU\\3%67EF
B%rK   r  c                       e Zd ZddZdej
                  j                  fdZdej
                  j                  defdZ	y)"OrtExecutionInfoForAllGraphModulesNc                     i | _         y rg   )execution_info_per_graph_module)rk   s    r/   ri   z+OrtExecutionInfoForAllGraphModules.__init__  s    
  	,rK   r2   c                 t    || j                   vry | j                   |   }|D ]  } |j                  | s|c S  y rg   )r1  r-  )rk   r2   r   
candidates	candidates        r/   &search_reusable_session_execution_infozIOrtExecutionInfoForAllGraphModules.search_reusable_session_execution_info  sQ     tCCC 99,G
# 	!I%y%%t,  	!
 rK   rt   c                 ~    || j                   vr|g| j                   |<   y | j                   |   j                  |       y rg   )r1  r   )rk   r2   rt   s      r/   cache_session_execution_infoz?OrtExecutionInfoForAllGraphModules.cache_session_execution_info  s=     tCCCBFD00>00>EEdKrK   )r   N)
rx   ry   rz   ri   r.   r~   GraphModuler5  r  r7  rJ   rK   r/   r/  r/    s@    !HH00 L!HH00L8RLrK   r/  r   T)frozenF)is_backward_compatiblec                       e Zd ZU dZdZeee      ed<   	 dZ	e
ed<   	 dZeee      ed<   	 dZe
ed<   	 dZe
ed	<   	 dZed
   ed<   	 dZed   ed<   	 dZeeedgdf         ed<   y)r   aJ  Options for constructing an ``OrtBackend``, the ONNX Runtime
    backend (``"onnxrt"``) for ``torch.compile``.

    Example::

        >>> @torch.compile(
        ...     backend="onnxrt",
        ...     options=torch.onnx._OrtBackendOptions(...),
        ... )
        ... def ort_function(x):
        ...     return x ** x
    Npreferred_execution_providersTinfer_execution_providersdefault_execution_providersFr   use_aot_autogradztorch.onnx.ExportOptionsexport_optionszonnxruntime.SessionOptionsort_session_optionszonnx.ModelProtopre_ort_model_transforms)rx   ry   rz   r{   r<  r	   r
   r   __annotations__r=  r   r>  r   r?  r@  rA  rB  r   rJ   rK   r/   r   r     s     OS!8H5I,J#KR '+t*mLP(3G*H!IP  %$V!d! <@NH78?UBF">?FV 	 h,-t345 1rK   r   c            	          e Zd ZU dZddee   fdZdej                  j                  de
eeeeef   f      fdZdej                  j                  fdZdej                  j                  dej                  j                  fd	Zdej                  j                  dej                  j                  fd
ZdZeed<   g Zeed       ed<   e	 ddeeeeeef   f      dd fd       Zed        Zed        Zy)r   a	  A backend compiles (sub-)graphs in torch.fx.GraphModule to onnxruntime.InferenceSession calls.

    The compiler entry point is OrtBackend.compile, which
        1. partitions the original graph into supported sub-graphs (type: torch.fx.GraphModule) and unsupported
           sub-graphs.
        2. For each supported sub-graph, it replaces its _wrapped_call function with _ort_accelerated_call.
        3. Inside _ort_accelerated_call, it creates onnxruntime.InferenceSession and calls it to execute the sub-graph.
    Noptionsc                    ddl m} dd l}dd l}dd l}|
t               n|| _        |j                  j                  j                  j                  | j                  j                  |j                  j                         n| j                  j                        | _        |j                  j                  j                  j                  j!                  | j                  j"                        }d d d d d d}t%        ||      | _        i | _        t+               | _        d| _        d| _        t3        |j4                  d      rt6        | _        y t8        | _        y )Nr   r   )getattrz_operator.getitemz_operator.mulz_operator.addz_operator.subFr   )r\   r   r(   r*   +torch.onnx._internal.fx.decomposition_tabler   _optionsonnx	_internal_exporter_legacyResolvedExportOptionsr@  ExportOptions_resolved_onnx_exporter_optionsr~   r"   '_create_onnx_supports_op_overload_tableonnx_registryrc   _supported_ops_partitioner_cacher/  _all_ort_execution_info_assert_allclose_to_baselineexecution_countr   r   r  r  r  )rk   rE  r`   r.   rd   re   s         r/   ri   zOrtBackend.__init__  s1   :4:6=o027  JJ  11GG==//7 

((*]]11 	,& zz++..BBjj00>>

  "& "!!
.
 1?QR UW (J'K$,1) 
 t**,=> 2 	 . 	rK   r2   r   c                    d}| j                   j                  rt        | x}r|}nt        |      x}r|}g }g | j                   j                  xs g t        |      | j                   j                  xs
 t               D ]L  }t        |t              r|i f}nt        |t              r|d   |d   i f}|7||vs<|j                  |       N |S )NrJ   r7   r   )rI  r=  r   r   r<  r   r>  rL   r   r@   r   r   )rk   r2   r   inferred_epseps_from_argseps_from_graph_moduleselected_epsr   s           r/   _select_epszOrtBackend._select_epsS  s     )+==22 5t <<}<
  -*El*SS&S  5
mm99?R
|$
 mm77O;M;O
 
	(B
 "c""XB&2a5=eR[~"L"8##B'
	( rK   c                 ,
   ddl }ddlm}m}  | j                  j
                  |g| }|rb|j                  }|j                  }	|j                  }
|j                  }|j                  }|j                  }|j                  }|j                  }n|j                  | j                  j                   |      j#                         }| j                  j$                  r,d| _        t)        |      }d }t+        j,                  ||      }n	  t/        |      j0                  |i |}|j9                  | j                  j                         }|j;                  | j                  j                   |      j#                         }|j#                  || j                  j<                        }|j?                  | j                  j@                  jB                  	      }	 dd
l"m#} ddl$m } |jK                  |      }|jM                  |      }| jP                  jR                  r#| jP                  jR                  D ]
  } ||        |jU                         }tV        jX                  j[                  dd      rt]        ||        |j^                  || jP                  j`                   | jb                  |g|       }te        d |jf                  jh                  D              }	te        d |jf                  jj                  D              }
tm        |      }to        |td              rtm        |      }ntm        |f      }te        d |jf                  jh                  D              }te        d |jf                  jj                  D              }tq        ||	||
||||      }| j                  js                  ||       | xjt                  dz  c_:        to        |tv        jx                        }|r|fn|}to        |td              sJ t{        d |D              sJ t}        d       | j#                  ||	|||
||| jP                  j&                  ||
      }t                | j                  rhtw        j                  j                  j                  |g|ddi}|r|fn|}t        ||      D ]%  \  }} tv        j                  j                  ||        ' |r|d   S |S # t2        $ r t4        j7                  d|       d| _         w xY w# tN        $ r t4        j7                  d       Y w xY w)a  This function replaces GraphModule._wrapped_call in compiled model.

        The _wrapped_call is the underlying implementation of forward method. Replacing
        it means we delegate the computation to _ort_acclerated_call and therefore
        onnxruntime.InferenceSession.
        r   N)r#   r$   Fc                 X    t        | d      rd| j                  v r| j                  d   S | S )Nr   r   )r   r   )r   s    r/   maybe_map_to_meta_valz>OrtBackend._ort_acclerated_call.<locals>.maybe_map_to_meta_val  s-    uf-%5::2E  %zz%00$rK   zFakeTensorProb failed for %s)diagnostic_context)fx_graph_moduleonnxfunction_dispatcher)opset_version)	optimizer)r    zONNXScript optimizer is not available. Skipping optimization. Please `pip install onnxscript -U` to enable post-export optimization.r4   )r2   )path_or_bytessess_options	providersc              3   4   K   | ]  }|j                     y wrg   rS   r   inputs     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s     Ou

O   c              3   4   K   | ]  }|j                     y wrg   rS   r   r   s     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s      S Srk  c              3       K   | ]  }|  y wrg   rJ   ri  s     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s     %Pe%P   c              3       K   | ]  }|  y wrg   rJ   rm  s     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s     &T&v&Tro  r$  r7   c              3   z   K   | ]3  }t        |t        j                  t        j                  t        f       5 y wrg   )r   r.   r   r   r   )r   elems     r/   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>(  s.      
 tellELL#>?
s   9;$run_onnx_session_with_ortvaluevectorexecutoraten)Gr    r,   r#   r$   rT  r5  r   r   r   r   r!  r   r   r"  MovePlaceholderToFrontrO  r`  r  dynamic_shapesr   r   r   tree_mapr   	propagate	Exceptionrs   rw   FxOnnxInterpreterInsertTypePromotionrb  to_model_protorQ  rc  r!   rd  onnxscript.rewriteroptimizerewriter-   rI  rB  SerializeToStringr;   r<   r=   rG   InferenceSessionrA  r\  r   rA   rj  r   r   r   r  r7  rV  r.   r   allrT   rW   rU  _primsrt  executer  testingassert_close)!rk   r2   r   kwargsr    r#   r$   !cached_execution_info_per_sessiononnx_sessionr   r   r   r!  r   r   prim_outputsextracted_outputsr_  fx_interpreterexported
onnx_modelrd  ort_rewriter	transformonnx_model_bytesexecution_info_per_sessionis_single_tensor_outputr   onnx_outputsbaseline_outputsnormalized_baseline_ouptutsr  baseline_outputs!                                    r/   _ort_acclerated_callzOrtBackend._ort_acclerated_callt  s    	G PD((OO# 	*
 -<DDL;GGK<IIL A S S!B!U!U=KKM>MMN<LLL
 "8844GG ce  33BB*/'$A,$O!%  '//)+< #I>,#?#I#I$!'$L 1BB#'#G#G#Z#Z C N "5544GGce 
 &)) ,(,(L(L(d(d * H
 "00"BBPP^^ 1 J0 '//
;
)11*=
 }}55!%!G!G *Ij)*  *;;=zz~~0$7 !!1M 8;77.!]]>>*$**<?$?L  O
8H8H8N8NOOK  S:;K;K;R;R SSL-d3M ,.!2<!@!2L?!C %%P9I9I9O9O%P P!&&TJ<L<L<S<S&T!T)C$'"3)#5+- ,	*& ((EE8 	!
 #-\5<<"H6\OL 	  15999 
/
 
 	
 

 	?@xx#MM,,#
 	,,$||44<< # .4  (?!#DT ( 1491 I,_ **;HI #:|AK|Ks ! NN#A<P /4D+ N  ]s   S .S1 (S.1TTc                 ~   ddl m} || j                  v r| j                  |   }|S |} ||| j                  d      }|j	                         }|| j                  |<   |j
                  j                  D ]H  }|j                  dk(  sd|j                  v s"t        ||j                        }| j                  |_        J |S )Nr   )CapabilityBasedPartitionerT)allows_single_node_partitioncall_modulefused_)!torch.fx.passes.infra.partitionerr  rS  rR  partition_and_fuserA   r   rq   rM   rG  r  _wrapped_call)	rk   r2   r   r  partitioned_prim_graph_moduleprim_graph_modulepartitionerrn   fused_modules	            r/   compilezOrtBackend.compileK  s    
 	Q, 4222,0,C,CL,Q)0 -,- !-4!##-1K
 -8,J,J,L)4QD##L1 6;;AA K 77m+DII0E#*+H$))#TL 261J1JL.K -,rK   c                     | j                   j                  r<ddlm} ddlm}   || j                  || j                  j                        ||      S | j                  ||      S )zIf ``OrtBackendOptions.use_aot_autograd`` is ``True``, the `auto_autograd` compiler
        will be invoked, wrapping this ``OrtBackend`` instance's ``compile`` method. Otherwise,
        the ``compile`` method is invoked directly.r   )#min_cut_rematerialization_partition)aot_autograd)fw_compilerpartition_fndecompositions)	rI  r?  functorch.compiler  torch._dynamo.backends.commonr  r  rO  r"   )rk   r2   r   r  r  s        r/   __call__zOrtBackend.__call__  s`     ==))MB< LL@#CCWW D	" " ||L$//rK      %_OrtBackend__instance_cache_max_count_OrtBackend__instance_cachec                     dt         dt         fdt         t               st        d
i  xs i  t         fdt        j                  D        d      }|{t        t        j                        t        j                  k  s'J dt        j                   dt         dt         d	       t        j                  j                  t               x}       |S )a  Returns a possibly cached instance of an ``OrtBackend``. If an existing
        backend was created previously through this function with the same options,
        it will be returned. Otherwise a new backend will be created, cached, and
        returned.

        Note: if ``options`` sets ``ort_session_options``, a new ``OrtBackend``
        will always be returned, since ``onnxruntime.SessionOptions`` cannot
        participate in caching.abc                 6   | j                   |j                   k7  s}| j                  |j                  k7  sd| j                  |j                  k7  sK| j                  |j                  k7  s2| j                  |j                  k7  s| j
                  |j
                  k7  ry| j                  |j                  y| j                  |j                  u ry| j                  |j                  | j                  j                  |j                  j                  k(  xr | j                  j                  |j                  j                  k(  xrZ | j                  j                  |j                  j                  u xr, | j                  j                  |j                  j                  u S y)NFT)r<  r=  r>  r   r?  rB  rA  r@  rw  diagnostic_optionsrQ  fake_context)r  r  s     r/   reusablez<OrtBackend.get_cached_instance_for_options.<locals>.reusable  sl   //13R3RR..!2M2MM00A4Q4QQ''1+?+??%%););;--1K1KK $$0A4I4I4U1#3#33 +0@0@0L$$33q7G7G7V7VV W((;;''::;W ((66!:J:J:X:XXW ((559I9I9V9VV rK   c              3   J   K   | ]  } |j                         s|  y wrg   )rI  )r   r  rE  r  s     r/   r   z=OrtBackend.get_cached_instance_for_options.<locals>.<genexpr>  s     U1x

G7TQUs   ##NzNo more than z instances of z allowed. Please instantiate `z` explicitly to pass to `torch.compile`. See https://github.com/pytorch/pytorch/pull/107973#discussion_r1306144795 for discussion.rJ   )r   r   nextr   r  r   r  r   )rE  backendr  s   ` @r/   get_cached_instance_for_optionsz*OrtBackend.get_cached_instance_for_options  s    !	) !	.? !	F '#45':7=b:GU
33U

 ?J//0:3X3XX  
 E EFn,<ZL I""X ''..*W:M/MwNrK   c                  @    t         j                  j                          y rg   )r   r  clearrJ   rK   r/   clear_cached_instancesz!OrtBackend.clear_cached_instances  s    ##))+rK   c                  4    t        t        j                        S rg   )r   r   r  rJ   rK   r/   get_cached_instanceszOrtBackend.get_cached_instances  s    Z0011rK   rg   )rx   ry   rz   r{   r	   r   ri   r.   r~   r8  r
   r   r@   r   r   r\  r  r  r  r  r   rC  r  r   staticmethodr   r  r  r  rJ   rK   r/   r   r     sM   V
): ; V
p!HH00	%WS#X../	0BUL1E1E ULn4-EHH$8$8 4-588CWCW 4-l0!HH000			0$ )*)24eD./4IMC% 17383D DEFC	C CJ , , 2 2rK   r   )rE  rE  c                8    t        j                  |      | |      S rg   )r   r  )r2   r   rE  s      r/   r   r     s     55g>|TRRrK   rg   )\dataclassesr&   loggingr;   typingr   r   r   r   r   r   r	   r
   r   r   r   r   typing_extensionsr   r.   torch._C
torch._opstorch._prims.executortorch.fxtorch._subclasses.fake_tensorr   torch.fx._compatibilityr    torch.fx.passes.fake_tensor_propr    torch.fx.passes.operator_supportr   torch.fx.passes.tools_commonr   torch.utilsr   rJ  r    r\   r   r`   r(   r)   r*   r+   rH  torch.onnx._internal.fx.passesr   r   rC  __all__r   r0   r@   r   bytesr~   r8  rG   rL   rT   rW   ra   	getLoggerrx   rs   rc   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  	dataclassr/  r   r   r   r   rJ   rK   r/   <module>r     sf      	    (      4 1 ; < :  60+6) #'$ &.T .b &( DcN ' IM'/0D0D'E2$HSM $)3 )$
@c 
@ 
		8	$/ /d3UXX-A-A 3d 3*
E#s(O 
uxx/C/C cSVh K0D0D K K8ehh.B.B 8uSRUX 8 X5c? XuS#X X$%2LL%,,U^^UEMMSWW	
 		%2  !%2P5<<$%056K0L
5<<*J 5<< )	) &) \\)@LL	 	LL	
8N
(NsCxN %,,#$N ./	N
 S/N 5<<$%N /0N N 78N #LL%,,U^^UEMMSWW	
 		N  5sE4/0#56!Nb#
(#sCx# %,,#$# ./	#
 S/# 5<<$%# /0# # 78# #LL%,,U^^UEMMSWW	
 		#  5sE4/0#56!#LI IX L L LB #(U3S8I3I-J(J"K i K	 d#e,<1 <1 - $<1~ e,q2 q2 -q2h e,
 FJ	S((&&S e-wsCx/@@AB	S -SrK   