¯wg  2dZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlmZmZmZmZmZmZmZddlZddlZddlmcmZddlmZddlmZm Z m!Z!m"Z"ddl#m$Z$ddl%m&Z&dd l'm(Z(d Z)ee(ejTejVfZ,eejZj\ej^j`fZ1eejTeed ffZ2ee3efZ4eee,efZ5Gd d ejlZ7ejpGddZ9dZ:d6dZ;d7dZe)f d:dZ?d;dZ@ddZCdZDdZE d?dZFdZG d@dZHGd d!ZI dAd"ZJ dBd#ZK dBd$ZL dC dDd%ZM dEd&ZN dC dFd'ZOdddddejjddd(d(d(ddf dGd)ZR dH dId*ZSGd+d,ZTGd-d.ZUejpGd/d0ZVdJd1ZWdKd2ZXdLd3ZYdKd4ZZdejjddd(df dMd5Z[y)NzFunctions to verify exported ONNX model is functionally equivalent to original PyTorch model. ONNX Runtime is required, and is used as the ONNX backend for export verification. ) annotationsN)AnyCallable CollectionMappingSequenceTupleUnion)_C) _constants _experimental_exporter_statesutils)GLOBALS)onnx_proto_utils)Number)CPUExecutionProvider.ceZdZdZdZdZdZy) OnnxBackendz9Enum class for ONNX backend used for export verification.ONNXReferenceEvaluatorrCUDAExecutionProviderN)__name__ __module__ __qualname____doc__ REFERENCEONNX_RUNTIME_CPUONNX_RUNTIME_CUDA\/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/torch/onnx/verification.pyrr+sC(I-/r rceZdZUdZdZded<dZded<dZded<dZded<e jZ ded <d Z d ed <d Z d ed<dZded<dZded<y)VerificationOptionsaOptions for ONNX export verification. Attributes: flatten: If True, unpack nested list/tuple/dict inputs into a flattened list of Tensors for ONNX. Set this to False if nested structures are to be preserved for ONNX, which is usually the case with exporting ScriptModules. Default True. ignore_none: Whether to ignore None type in torch output, which is usually the case with tracing. Set this to False, if torch output should keep None type, which is usually the case with exporting ScriptModules. Default to True. check_shape: Whether to check the shapes between PyTorch and ONNX Runtime outputs are exactly the same. Set this to False to allow output shape broadcasting. Default to True. check_dtype: Whether to check the dtypes between PyTorch and ONNX Runtime outputs are consistent. Default to True. backend: ONNX backend for verification. Default to OnnxBackend.ONNX_RUNTIME_CPU. rtol: relative tolerance in comparison between ONNX and PyTorch outputs. atol: absolute tolerance in comparison between ONNX and PyTorch outputs. remained_onnx_input_idx: If provided, only the specified inputs will be passed to the ONNX model. Supply a list when there are unused inputs in the model. Since unused inputs will be removed in the exported ONNX model, supplying all inputs will cause an error on unexpected inputs. This parameter tells the verifier which inputs to pass into the ONNX model. acceptable_error_percentage: acceptable percentage of element mismatches in comparison. It should be a float of value between 0.0 and 1.0. Tboolflatten ignore_none check_shape check_dtyperbackendgMbP?floatrtolgHz>atolNSequence[int] | Noneremained_onnx_input_idxz float | Noneacceptable_error_percentage)rrrrr%__annotations__r&r'r(rrr)r+r,r.r/rr r!r#r#3sl4GTKKK&77G[7D%D%4818044r r#cg}|D]>}t|tr|jt|.|j |@|SN) isinstancetupleextend_flatten_tuplesappend)elem flattenedts r!r6r6ZsII   a    _Q/ 0   Q   r c4t|tjrV|jr,|j j j S|j j St|ttfr|Dcgc] }t|c}St|tttfrtj|St|tr3g}|D]*}|j!t|t||g,|S|Scc}wr2)r3torchTensor requires_graddetachcpunumpylistr4 _to_numpyr$intr*nparraydictr5)r8inpr9ks r!rCrCes$ %   ;;=$$&,,. .88:##% % D4- (*./3 #// D4e, -xx~ D$   AA   ilId1g,>? @ A K0s Dcz|D]5}t|ttfs|j|n t ||7|Sr2)r3rBr4r7_inline_flatten_list)inputsres_listis r!rKrKws? 1", e}# #Ax01 Or Tcg}|D](}|jtj||*|Dcgc] }t|c}Scc}w)Ncast_onnx_accepted)r5runpack_quantized_tensorrC)valuesrQvalue_unpackedvaluevs r!_unpack_to_numpyrWsON   ) )%DV W  #1 1QIaL 11 1sA c Xi}|rt|dtr |d}|dd}tt|}i}|j D]\}}t |||<t |}t |dr(|jDcgc]}|j}}n1t |dr |j}ntdt|dt|D]7\}}|t|k(s|||vrtd|d|d|d||||<9|jd|}|Scc}w) N get_inputs input_nameszUnknown ONNX backend type: .z(got too many positional inputs. inputs: z . kw_inputs: z. input names: )r3rGrWr6itemsrChasattrrZnamer[ ValueErrortype enumeratelenrun) onnx_sessionrL kw_inputs ort_inputs input_nameinputrNr[ onnx_outss r! _run_onnxrksZI *VBZ.2J  of5 6FJ&__.2 E!*5!1 :2 v F|\*'3'>'>'@A!qvvA A } -".. 6tL7I6J!LMMf%+5 K KNj$@:6(-PY{[ +}A/ &+ ;q>" +  z2I BsD'c ddl}|t}|j}d|_|j t |tr|n|j||}|S#t$r}td|d}~wwxYw)Nrz0onnxruntime is required for export verification.) providers) onnxruntime ImportError_ORT_PROVIDERSSessionOptionslog_severity_levelInferenceSessionr3strgetvalue)model ort_providersroesession_options ort_sessions r! _ort_sessionr|sU& !002O*+O&..E3'U^^-=/K  ULMSTTUsA A5$ A00A5c ddl}ddlm}t|tr|j |n|j |j}|j|}|S#t$r}td|d}~wwxYw)Nr) referencez1onnx >= 1.13 is required for reference evaluator.) onnxr~rpr3ruloadload_model_from_stringrvReferenceEvaluator)rwronnx_referenceexcprotores r!!_onnx_reference_evaluator_sessionrs}X4 eS ! %  ( ()9 : "44U;L  XMNTWWXs A A9( A44A9c|tjk(r t|}|S|tjtjhvrt ||j f}|Std|)NzUnsupported backend: )rrrrrr|rUr`)rwr)res r!_onnx_backend_sessionrsm+'''8?  [11;3P3PQ Q#EGMM+;<  0 :;;r c 2t|t|k(s Jdt|dt|d|j}|r|dkDs|dkr tdt||D]n\}} |jst j ||\}}tjj|||j|j|jdpy#t$r3}|rd t jt j|||j|j t j |j"z z }||kr%t%j&d |d |d |dYd}~ |j(t j*k(s|j(t j,k(rt%j&d|j(t j*k(s|j(t j,k(rt%j&dd}~wwxYw)Nz(Number of outputs differ ONNX runtime: (z ) PyTorch: ()g?gzAIf set, acceptable_error_percentage should be between 0.0 and 1.0T)r+r,r( equal_nan)r+r,zSuppressed AssertionError: z. Error percentage z within acceptable range r\zONNX output is quantizedzPyTorch output is quantized)rcr/r`zipr'rEbroadcast_arraysr<testing assert_closer+r,r(AssertionErrorsumiscloseprodshapewarningswarndtypeuint8int8)rjpt_outsoptionsr/ort_outpt_outryerror_percentages r!#_compare_onnx_pytorch_outputs_in_nprs I#g,&^ 1#i.1AcRYl^[\]^&")"E"E"#c)-H3-N O  y'2 &&"$"5"5gv"F MM & &\\\\#// '   *#$rvvJJwW\\ U(GGGMM*(+$+ $'BBMM6qc:,,<+=>33N2OqR }}(GMMRWW,D 89||rxx'6<<277+B ;< ! s!.A(C H#B H2BHHc|jr#tjj|\}}n t |gg}t |d}t |g}t |||y)at Compare ONNX and PyTorch outputs. Args: onnx_outs: outputs from ONNX backend. pt_outs: outputs from PyTorch. options: options for verification. Raises: AssertionError: if outputs from ONNX model and PyTorch model are not equal up to specified precision. ValueError: if arguments provided are invalid. FrPN)r&r<jit_flattenrKrWr)rjrr_ pt_outs_nps r!_compare_onnx_pytorch_outputsrsW$YY''0 &y"5!'eDJ$Y3I' :wGr ct|tjtfr|f}t j |}|rt j |}||fSi}||fS)aPrepare input for PyTorch model execution. Any future changes/formatting to the input before dispatching to the PyTorch model should be made in this function. Args: args: positional arguments for PyTorch model forward method. kwargs: keyword arguments for PyTorch model forward method. Returns: args: positional arguments for PyTorch model forward method. kwargs: keyword arguments for PyTorch model forward method. )r3r<r=rGcopydeepcopy)argskwargss r!_prepare_input_for_pytorchr$s\$t,-w == D v& < <r ct||\}}|s)t|dkDrt|dtr|ifz}|S|r||fz}|S|}|S)aPrepare input for ONNX model export. Any future changes/formatting to the input before dispatching to the :func:`torch.onnx.export` api should be made in this function. Args: args: positional arguments for PyTorch model forward method. kwargs: keyword arguments for PyTorch model forward method. Returns: onnx_inputs: positional arguments for ONNX model export, as `args` in :func:`torch.onnx.export`. rrY)rrcr3rG)rr onnx_inputss r!_prepare_input_for_exportr>sg.dF;LD& c$i!m 48T(BbUl  fY&   r ct||}|r#tjj|\}}n|r |dik(r|dd}||Dcgc]}|| c}S|Scc}w)ahPrepare input for ONNX model execution in ONNX backend. Any future changes/formatting to the input before dispatching to the ONNX backend run should be made in this function. Args: args: positional arguments for PyTorch model forward method. kwargs: keyword arguments for PyTorch model forward method. remained_onnx_input_idx: indices of inputs to be used for ONNX model execution. flatten: whether to flatten the input before dispatching to the ONNX model execution. Returns: onnx_inputs: positional arguments for ONNX model execution in ONNX backend. rYN)rr<rr)rrr.r%rrrNs r!_prepare_input_for_onnxrVsl",D&9K++K8 Q RB.!#2& *(?@1 A@@As Acz tj|S#t$rtjd|cYSwxYw)zHUsed for preserving original model in case forward mutates model states.zHFailed to clone model. Model state might be mutated during verification.)rr Exceptionrr)rws r!_try_clone_modelrss;}}U##  V  s  ::c|t|jfd}||||r|D] }||i yy)apCompare outputs from ONNX model runs with outputs from PyTorch model runs. Args: pt_model: PyTorch model. onnx_model_f: ONNX model file path or file-like object. input_args: positional arguments for PyTorch model forward method. input_kwargs: keyword arguments for PyTorch model forward method. additional_test_inputs: additional positional arguments for PyTorch model forward method. options: options for verification. Raises: AssertionError: if outputs from ONNX model and PyTorch model are not equal up to specified precision. ct||\}}t }||i|}t|| j j}t |}t || y)Nrjrr)rrrr.r%rkr) input_args input_kwargspt_args pt_kwargs pt_model_copyrrrjrerpt_models r!%compare_onnx_pytorch_model_with_inputzJ_compare_onnx_pytorch_model..compare_onnx_pytorch_model_with_inputsi7 LQ(2 6I6-  g&E&Ew lK8 % r N)rr)) r onnx_model_frradditional_test_inputsrrtest_input_argsres ` ` @r!_compare_onnx_pytorch_modelr~sI.)wGL $**lC5 GO 1/2 F Gr c.eZdZdZddZdZddZd dZy) _GraphDiffz7A class to represent the difference between two graphs.c ||_||_y)zConstruct a _GraphDiff object. Args: graph_a (_C.Graph): First graph to compare. graph_b (_C.Graph): Second graph to compare. N)graph_agraph_b)selfrrs r!__init__z_GraphDiff.__init__s  r c"|jS)z!See function :func:`diff_report`.) diff_reportrs r!__str__z_GraphDiff.__str__s!!r cldj|jDcgc]}d|z c}Scc}w)N  )join splitlines)rlineslines r!_indentz_GraphDiff._indents,yy%2B2B2DE$$+EFFEs 1c |j}|j}t|}t|}||k(rytj|j d|j d}d|j dj|g}tj|j|jD]*\}}t|t|k7s|jdtjt|j dt|j d} d|j dj| g} |r|jnd} | r+| jd|j t| g|r|jnd} | r+| jd|j t| g|j| nd j|S) a)Return a string representation of the graph difference. The report shows the first pair of nodes that diverges. It also shows the source location of the pair of nodes. Returns: graph_diff_report (str): A string representation of the graph difference. Tz Graph diff:zFirst diverging operator:z node diff:NzFormer source location:zLatter source location:r)rrrudifflibndiffrrr itertools zip_longestnodesr7 sourceRanger5) rrr graph_a_str graph_b_str graph_diffgraph_diff_reportnode_anode_b node_diffsource_printoutstack_astack_bs r!rz_GraphDiff.diff_reports,,,,'l 'l + %]]  " "4 (+*@*@*F +DLL9L,MN'33GMMOW]]_U NFF6{c&k)!(()DE#MMK**40#f+2H2H2N $0bggi>P1Q"R28&,,.d#**2DLLW4NO39&,,.d#**2DLLW4NO"((9+ .yy*++r N)r_C.Graphrr)rrureturnrurru)rrrrrrrrrr r!rrsA"G.,r rct|dkr tdd}|D]5\}}|||||}||}t||j}|s3|cSy)aCheck if graph produced by `model_to_graph_func` is the same across `test_input_groups`. Args: model: See :func:`check_export_model_diff`. test_input_groups: See :func:`check_export_model_diff`. export_options: See :func:`check_export_model_diff`. model_to_graph_func: A function to convert a PyTorch model to a JIT IR graph. Returns: graph_diff_report (str): A string representation of the graph difference. z3Need at least two groups of test inputs to compare.Nr)rcr`rr) rwtest_input_groupsexport_optionsmodel_to_graph_func ref_jit_graphrr jit_graphrs r!_check_graph_diffrsx2 !NOOM)% f'tV^L  %M &}i@LLN $ $% r c |j}|j}tj|||5t ||}tj ||}tj ||\}}}}|cdddS#1swYyxYw)azAs part of the ONNX export steps, create a traced JIT graph from a PyTorch model. Args: model: See :func:`check_export_model_diff`. args: See :func:`check_export_model_diff`. kwargs: See :func:`check_export_model_diff`. export_options: See :func:`check_export_model_diff`. Returns: jit_graph (_C.Graph): A traced JIT graph. N)trainingverboserexporter_contextr_pre_trace_quant_model_create_jit_graph) rwrrrrr export_inputsrrs r!_traced_graph_from_modelrs}"&&H$$G   x 91$? ,,UMB"44UMJ 1a s ?A99Bc |j}|j}|j}|j}|j}|j } |j } |j} |tj}tj|||stjj}|t_|t_tj"|||5tj$|j&||} | i} tj(| || | t+||} tj,|| } tj.|| || | || ||  \}}}|cdddS#1swYyxYw)ayAs part of the ONNX export steps, export an ONNX JIT graph from a PyTorch model. Args: model: See :func:`check_export_model_diff`. args: See :func:`check_export_model_diff`. kwargs: See :func:`check_export_model_diff`. export_options: See :func:`check_export_model_diff`. Returns: onnx_graph (_C.Graph): An ONNX JIT graph. N)r dynamic_axes) opset_versionoperator_export_typeexport_modules_as_functionsrrrr[ output_namesr ONNX_DEFAULT_OPSETr_setup_trace_module_map_C_onnxOperatorExportTypesONNXrexport_onnx_opset_versionr_decide_constant_foldingdo_constant_folding_validate_dynamic_axesr_decide_input_format_model_to_graph)rwrrrrrrrrrr[rrr onnx_graphrs r!_onnx_graph_from_modelr 9sa$#00M)>>"0"L"L&&H$$G!..L ,,K!..L"55  !!%)DE &::??(5G%#7G   x 9#<<  . .0Dh   L $$\5+|T1$? 225-H  00      %   Aq/s BEE(c|i}|j}|jxsi}|j}|j}|j}|j xst j}|t_ |t_tj|||}|j}tj|||||}||tjj k(r t"j$j'||}|r?|t j(k\r,t%j*|||}t%j,|tj.rt%j0|||t%j2||}|dkrt%j4|t%j6|}t%j8|t%j,||j:r t=d|||fS)N) params_dictrr[ z ONNX graph: )rrr[rrrr rrrrrr_optimize_graphr TrainingModeEVALr<r _jit_pass_onnx_eval_peepholeONNX_CONSTANT_FOLDING_MIN_OPSET_jit_pass_onnx_constant_fold4_jit_pass_dce_allow_deleting_nodes_with_side_effectsonnx_shape_inference)_jit_pass_onnx_graph_shape_type_inference%_jit_pass_onnx_eliminate_unused_items,_jit_pass_onnx_cast_all_constant_to_floating%_jit_pass_filter_non_tensor_arguments#_jit_decay_packed_param_input_typesrprint) graphrrrrr[rrrs r!_onnx_graph_from_aten_graphr ys  )>>!..4"L ,,K&&H(<<"00QJ4Q4QM(5G%#7G 8818 JJLE  ! ! !  E8w';';'@'@@hh;;E;O   ZGG G55e[-X  ??F## 44UKW::5+NKq 77>::;GK**51;;EB ne$ + r c b|jxstj}|jxsi}|j}t j |j||}t jd|}|jxsi}|j|||d||j |||di \} } } } | | fS)NTFr) rr rrrr_decide_keep_init_as_inputkeep_initializers_as_inputs_decide_add_node_names custom_opsets _export_onnxr) r rrrrrval_keep_init_as_ipval_add_node_namesr%r export_maprs r!_onnx_proto_from_onnx_graphr*s #00QJ4Q4QM!..4"L)>>::22 55d$r}||| |fcYd}~Sd}~wwxYw#t@$r*}tCdtCd|tCd||d}~wwxYw)Nc3$K|]}|du ywr2r).0ws r! z$verify_aten_graph..ns.q}.srz%Unexpected error during verification.z jit graph: z onnx graph: )"r#rrBrLr4rc debugNameallrr<r _jit_interpret_graphr3r r*r7r8r ExportTypes PROTOBUF_FILEr _export_filerr7rr.r%rr)rkrrrr)rrrrverification_optionsoriginal_jit_graph graph_inputsarg jit_inputsrVweightsjit_input_and_parametersjit_outsonnx_params_dictrr)rA export_typenew_input_namesnew_input_argsrrerjrys r!verify_aten_graphr][s#24  JJLE 'LzES_EFJ3?J@Q3RSa{1;;=)SGS .g. .. .z*J)E'N:xx,,U4LMH hu .:: ~{E 4 ~/E:!# G"..<* -   8 8 ( (  -W6J6R6RS lK8  1 )# , UHi//iFS0B0 1eXy0 0 1  56 m/0 ne$ ssH3 H3(H8I-H=6I;A#II-I=I I IIIII J'%J  JceZdZUded<ded<ded<ded<ded<ded <dd Zdd Zdd Zdd ZddZddZ ddZ ddZ ddZ dZ y)GraphInfoPrettyPrinterGraphInfo | None graph_infozGraphInfoPrettyPrinter | None upper_printer lower_printerzMapping[int, str]graph_str_lambdasconnector_str_lambdaschildren_str_lambdasc||_|M|jA|j5t|j|_t|j|_yd|_d|_yr2)raupper_graph_infolower_graph_infor_rbrc)rras r!rzGraphInfoPrettyPrinter.__init__s]$  "++7++7!7 8S8S!TD !7 8S8S!TD !%D !%D r c|jy|jrD|jr8|jj|jjzdzSy)Nrr)rarbrc _total_rowsrs r!rkz"GraphInfoPrettyPrinter._total_rowssV ?? "   $"4"4""..043E3E3Q3Q3SSVWW r c|jy|jj}|jj}|dk(r.|r,d|jjj dnd}|d|rdn t dd|S) N...r(rr Xi')raessential_node_count has_mismatchessential_node_kindspopchr)r node_countrrerror_node_kinds r!_node_count_segment_strz.GraphInfoPrettyPrinter._node_count_segment_strs ?? "__99; 335 Q<446::<=Q ?  QlsF DAoEVWWr cN|jyd|jjS)Nrzid: )raidrs r!_graph_id_segment_strz,GraphInfoPrettyPrinter._graph_id_segment_strs' ?? "doo(()**r cpttt|j|j fSr2)maxmaprcrxr{rs r!_max_segment_columnsz+GraphInfoPrettyPrinter._max_segment_columnss0 d224d6P6P6RS T  r cF|dk(r4|j}|d|jt|z zz }|S|dk(r4|j}|d|jt|z zz }|Sd|cxkr|j krnyd|jzSy)zEGet the string representation of the graph segment at the given line.rrorr)rxrrcr{rk)rr result_strs r!_graph_segment_str_at_linez1GraphInfoPrettyPrinter._graph_segment_str_at_lines 19557J #!:!:! ? '+oo&R&R&T"MM 35 "ooJJL  '<<> z?a' * 0I+// 1=A( 2   *11"c: ; % & (//C8 9#sFN)rar`rrDr)rrDrru)rrrr0rrkrxr{rrrrrrrr r!r_r_sY  0000((,,++ & X+   * :r r_c:eZdZdZe d ddZddZy)OnnxTestCaseReprocb||_tj|\|_|_|_yr2) repro_dirrload_test_caserrLoutputs)rrs r!rzOnnxTestCaseRepro.__init__<s)"0@0O0O 1 - DKr Nc|-tjjjd}tj|t |t |||S)u3Create a repro under "{dir}/test_{name}" for an ONNX test case. The test case contains the model and the inputs/outputs data. The directory structure is as follows: dir ├── test_ │ ├── model.onnx │ └── test_data_set_0 │ ├── input_0.pb │ ├── input_1.pb │ ├── output_0.pb │ └── output_1.pb Args: proto: ONNX model proto. inputs: Inputs to the model. outputs: Outputs of the model. dir: Directory to save the repro. name: Name of the test case. If not specified, a name based on current time will be generated. Returns: Path to the repro. z%Y_%m_%d_%H_%M_%S_%f)datetimenowstrftimerexport_as_test_caserC)clsrrLrdirr_s r!create_test_case_reproz(OnnxTestCaseRepro.create_test_case_reproBsU8 <$$((*334JKD33  f  g      r cttj|j|j}|j d|j }t|dr(|jDcgc]}|j}}n0t|dr |j}ntdt||Dcgc]}|j|}}t|||ycc}wcc}w)a2Run the ONNX test case with options.backend, and compare with the expected outputs. Args: options: Options for validation. Raise: AssertionError: if outputs from options.backend and expected outputs are not equal up to specified precision. N get_outputsrzUnknown onnx session type: )rr7r8rr)rdrLr^rr_rr`rarr)rrre run_outputsorr_ expected_outss r!validatezOnnxTestCaseRepro.validatehs-RZZ -CW__U "&&tT[[9 < /,8,D,D,FGqAFFGLG \> 2'44L:4 ;M:NOP P8DEd+E E+KP H Fs .C8C#r2)rbytesrrur_ str | None)rr#)rrrr classmethodrrrr r!rr;s= IM# # 14# d/Z" d? d@d0Z#y )A GraphInfoz^GraphInfo contains validation information of a TorchScript graph and its converted ONNX graph. torch.Graphrtuple[Any, ...]rdict[str, Any]r)default_factory_experimental.ExportOptionsrNF)defaultinitzAssertionError | Nonemismatch_errorzSequence[_NumericType] | Nonerr`rhrir)rrurz)rrztorch.Graph | None _onnx_graph>prim::Constantprim::ListConstructaten::ScalarImplicitzfrozenset[str]_EXCLUDED_NODE_KINDScJd|_d|_d|_d|_d|_y)z2Clear states and results of previous verification.N)rrrrhrirs r!clearzGraphInfo.clears)"  $ $r c6t|jy)uPretty print `GraphInfo` tree. Each node represents a subgraph, showing the number of nodes in the subgraph and a check mark if the subgraph has output mismatch between torch and ONNX. The id of the subgraph is shown under the node. The `GraphInfo` object for any subgraph can be retrieved by calling `graph_info.find_partition(id)`. Example:: ==================================== Tree: ===================================== 5 X __2 X __1 ✓ id: | id: 0 | id: 00 | | | |__1 X (aten::relu) | id: 01 | |__3 X __1 ✓ id: 1 | id: 10 | |__2 X __1 X (aten::relu) id: 11 | id: 110 | |__1 ✓ id: 111 =========================== Mismatch leaf subgraphs: =========================== ['01', '110'] ============================= Mismatch node kinds: ============================= {'aten::relu': 2} N)r_rrs r!pretty_print_treezGraphInfo.pretty_print_trees@ t$113r ctd|jdjdd|rltdjddt|j|j0tdjddt|j|j r1tdjddt|j ytd jddy) zPretty print details of the mismatch between torch and ONNX. Args: graph: If True, print the ATen JIT graph and ONNX graph. z# Mismatch info for graph partition : rrz ATen JIT graph Nz ONNX graph z Mismatch error z No mismatch )rrzrrrrrr)rrs r!pretty_print_mismatchzGraphInfo.pretty_print_mismatchs 3DGG9B?FFr3OP  $++B4 5 $** +n++B45d&&'     $++B4 5 $%% & /((S1 2r c|jduS)zGReturn True if the subgraph has output mismatch between torch and ONNX.N)rrs r!rrzGraphInfo.has_mismatchs""$..r c\tfdjjDS)zUReturn the number of nodes in the subgraph excluding those in `_EXCLUDED_NODE_KINDS`.c3\K|]#}|jjvs d%yw)rN)kindr)rHnrs r!rJz1GraphInfo.essential_node_count..s) QVVXT=V=V-VA s!,,)rrrrs`r!rqzGraphInfo.essential_node_counts* zz'')   r c|jjDchc].}|j|jvr|j0c}Scc}w)zWReturn the set of node kinds in the subgraph excluding those in `_EXCLUDED_NODE_KINDS`.)rrrr)rrs r!rszGraphInfo.essential_node_kindssKZZ%%' vvxt888 FFH   s3Ac|jsgS|jduxs|jj xr+|jduxs|jj }|r|gSg}|j||jjz }|j||jjz }|S)zAReturn a list of all leaf `GraphInfo` objects that have mismatch.N)rrrhrir)rno_mismatch_childrenresultss r!rz&GraphInfo.all_mismatch_leaf_graph_infos  "I  ! !T ) U1F1F1S1S1U-U   ! !T ) U1F1F1S1S1U-U  6M  , t,,IIK KG  , t,,IIK KGr c(||jk(r|St|j}t||kDr^||dk(r'|j|jj|S||dk(r'|j|jj|Sy)z.Find the `GraphInfo` object with the given id.0N1)rzrcrhfind_partitionri)rrzcurrent_lengths r!rzGraphInfo.find_partitions =KTWW r7^ #.!S(T-B-B-N,,;;B??N#s*t/D/D/P,,;;B??r cX|tj}tjj|d}t |j |j |j\}}t||j |\}}tj||j|j||S)uExport the subgraph to ONNX along with the input/output data for repro. The repro directory will contain the following files:: dir ├── test_ │ ├── model.onnx │ └── test_data_set_0 │ ├── input_0.pb │ ├── input_1.pb │ ├── output_0.pb │ └── output_1.pb Args: repro_dir: The directory to export the repro files to. Defaults to current working directory if None. name: An optional name for the test case folder: "test_{name}". Returns: The path to the exported repro directory. onnx_debug) r<getcwdr=rr rrrr*rrrr)rrr_r rYrrs r! export_reprozGraphInfo.export_repros2   IGGLLL9 'B JJ++T-=-=( $ $/ ++-= q!77 4??DLL)T  r c t|jjDcgc]#\}}|j|jvr|%}}}t |dzdz }|dk\rt ||kDr||dzSycc}}w)aFind the pivot index to partition the graph. The pivot is the node that splits the graph into two parts. Each part should have the similar amount of nodes, excluding non essential ops, defined in `_EXCLUDED_NODE_KINDS`, such as `prim::Constant`. If the graph has an odd number of nodes, the upper part will have one more node. If the graph does not have any node that can be partitioned, return -1. Returns: The index of the pivot node. rrrrY)rbrrrrrc)rrNrincluded_node_indiceshalf_idxs r!_graph_partition_pivotz GraphInfo._graph_partition_pivot/s"$**"2"2"45! 1vvxt888 ! ! ,-2Q6 q=S!67(B(2Q6 6! s(A?c |j}|dk(rtjS|jj }t |j } dd}g}tj||}|j|||\}}} }t|D]}|jd|D]} |j| t|D]} | jtt tt |jD])\} } t!| | r| |vs |j#| +|S#t$$r}t'| ||d}~wwxYw)NrYc(|j||Sr2)r7) new_outputs bridge_values r!_process_bridge_value_for_upperzIGraphInfo._partition_upper_graph.._process_bridge_value_for_upperLs   | , r r)rzlist[torch.Value]r torch.Valuerr)rr<GraphrrrBr functoolspartial_partition_nodesrb eraseOutputregisterOutputreverseddestroyrL_has_uses_by_nodes eraseInput RuntimeErrorr)rpivotroriginal_outputsrrprocess_bridge_value_for_upperr dropped_nodescomplete_upper_nodes_setoutputnoderNrirys r!_partition_upper_graphz GraphInfo._partition_upper_graphEs++- B;;;=  ! 0 * :E  *, )2):): +[* &9=8M8M 589 5=2A+, !A   a  !! )F   ( )]+ D LLN !iU\\^0D&E!FG HAu&u.FG,$$Q'  $%'Gs=E E.E))E.cf|j}|dk(rtjS|jj }t |j }t |j}g} dd}tj||}|j|||\}} } } |D] } t| | s|j| "t|D]} |jd|D]} |j| |D]A} t!| | s|j#}| j%||j'| Ct)|D]}|| vs |j+|D]} |j1d|S#t,$r}t/|||d}~wwxYw)NrYcj|j}|j||j||Sr2)addInputreplaceAllUsesWith copyMetadata)rr new_inputs r!_process_bridge_value_for_lowerzIGraphInfo._partition_lower_graph.._process_bridge_value_for_lowerzs2(I  + +I 6  " "< 0 r r)rrrrrr)rr<rrrrBrrLrrr _produced_byr7rbrrrrrrrrrrr)rrrroriginal_inputsrrprocess_bridge_value_for_lower upper_nodes lower_nodesrcomplete_lower_nodes_setrrirrrys r!_partition_lower_graphz GraphInfo._partition_lower_graphps++- B;;;=  ! 0u||~.   .9   *3):): +U* &AE@U@U 58A = [!%=' +FFK0""6* ++, !A   a  !! )F   ( )% .E!%)AB!NN, ((3&&u-  . [) D33LLN ! A   Q   $$&Gs(F F0F++F0c ||vryt||rv|j|jvrZ|jt |g|j D]+}||vr|j |j|||||-y|jD]/}||vrt||s||vs|j||1yr2) _node_has_uses_byrrupdate _all_nodesrL_partition_noderrradd) rrrroriginal_graph_outputscovered_bridge_valuesprocess_bridge_valuerirs r!rzGraphInfo._partition_nodes + +  d$< = t888 $ + +Jv,> ? 11$$JJL,,*)(  ,,. L22&v/GH!77)--.B6.JK Lr c t|j}|d|}||d}t|}t|}t|j } t|j } |D]} |j | ||| | |||||fSr2)rBrrsetrrLr) rrrr rrrrrr r rs r!rzGraphInfo._partition_nodess U[[]#FUm EFm $.k#: #-k#: !$U]]_!5 !$ELLN 3 D  ((&%$      $ $   r cT|j}t|jj}|Jt |t |k(s+Jt |dt |d|jt ||Dcic]\}}|j |c}}Scc}}w)N vs z Graph: )rrBrrrcrrK)rr graph_outputsrVrs r!_bridge_kwargszGraphInfo._bridge_kwargss,,TZZ//12 """=!S &   J- !c'l^9TZZL I J .1-HITQ q IIIsB$c|jDcgc]}|j}}tfd|D}|tfd|Dz }|Dcic] }||vs|||} }t|t| zt|k(s-Jt|dt| dt|d||| fScc}wcc}w)Nc32K|]}|vs|ywr2r)rHrI bridge_kwargss r!rJzAGraphInfo._args_and_params_for_partition_graph..sQ!a=>P]1%Q  c32K|]}|vs|ywr2r)rHrI full_kwargss r!rJzAGraphInfo._args_and_params_for_partition_graph..sNQ+=Mk!nNr + rr)rLrKr4rc) rrrr full_paramsrir[rrIparamss `` r!$_args_and_params_for_partition_graphz.GraphInfo._args_and_params_for_partition_graphs7b N O V|FNsC C 'C crt|j|j|j|j|S)a Verify the export from TorchScript IR graph to ONNX. Export the TorchScript IR graph to ONNX, with the inputs, parameters and export options recorded in this object. Then verify the exported ONNX graph against the original TorchScript IR graph under the provided verification options. Args: options: The verification options. Returns: error: The AssertionError raised during the verification. Returns None if no error is raised. onnx_graph: The exported ONNX graph in TorchScript IR format. onnx_outs: The outputs from running exported ONNX model under the onnx backend in `options`. pt_outs: The outputs from running the TorchScript IR graph. )rrrrQ)r]rrrr)rrs r! verify_exportzGraphInfo.verify_export s4*! JJ((..!(   r c |j| t}|jjrt |j t t|j jdk(ryt |jt |jzt t|j jk(saJdt t|j jdt |jdt |jd|j|\|_ |_|_}|jy|j!dkryt#|j j|jDcic]\}}|j%|}}}|j}|j'}|j)|i||\}} t+||| |j|j,dz |_|j.j1||j.j3} |j5} |j)| | ||\} } t+| | | |j|j,d z |_|j6j1|ycc}}w) a Find all mismatches between the TorchScript IR graph and the exported onnx model. Binary searches the model graph to find the minimal subgraph that exhibits the mismatch. A `GraphInfo` object is created for each subgraph, recording the test inputs and export options, as well as the validation results. Args: options: The verification options. NrzNumber of graph inputs(z/) does not match the provided tensor arguments(rz).rr)rzr)rr#rrrrrcrBrrrrLrrrrrqrrKrrrrzrh find_mismatchrrri)rrrrIrVrr upper_graph upper_args upper_paramsr lower_graph lower_args lower_paramss r!r zGraphInfo.find_mismatch)s ?)+G    & & $**  tDJJ&&() *a / 4??#c$*:*:&;;s ""$ %@   &c$tzz/@/@/B*C&D%EF--0-A,B#c$JZJZF[E\\^ `  BFASAS B >T-t|Q    &   $ $ &! + *-TZZ->->-@$//)R !%AAKKM1   && 113 #'#L#L [+$  L!*      ww} !  ++G4--<<> 113 #'#L#L  [$  L!*      ww} !  ++G4A s(K )F)rr$)rr$r)rzset[str])rzlist[GraphInfo])rzrurr`NN)rrr_rrru)rr) r torch.Noderset[torch.Node]rr)r set[torch.Value]r r*r $Callable[[torch.Value], torch.Value])rrrrDr r+rzKtuple[list[torch.Node], list[torch.Node], set[torch.Node], set[torch.Node]])rrrz3Mapping[str, _NumericType | Sequence[_NumericType]]rMapping[str, torch.Tensor]rr,)rr#rEtuple[AssertionError | None, torch.Graph, _OutputsType, _OutputsType]r2)rVerificationOptions | None)$rrrrr0 dataclassesfieldr r,rrrrhrirzr frozensetrrrrrrrqrsrrrrrrrrrrrr rr r!rr~sh 2C+2C2C%333N/->K,=,=dQV,WN)W->[->->tRW-XG *X):):):4e)T&T):):):4e)T&Tk+B+&7k&7&7UD&QK#Q+4I,.% 4D3,/  , @D& #& 2<& & P,)V4l$L$L#2$L#2 $L !1 $L 0 $LC$LL% % % C % U % NJK0  0   * N >/3L5+L5r rc t|}|D]H}|jD]3}|jtt |j 5J|Sr2)rblocksrrrBr)r all_nodesrbs r!rrxsTE I : :A   ZQWWY8 9 :: r cHtfd|jDS)Nc3:K|]}|jvywr2)user)rHusers r!rJz%_has_uses_by_nodes..s9Ssxx5 9s)anyusesrUrs `r!rrs 9EJJL9 99r cJ|jD]}t||syy)NTF)rr)rrrs r!rrs),,. fe , r c&|j|vSr2)rr<s r!rrs ::<5  r c "| t}|tj} |tjj j k(r|jn7|tjj jk(r|jtj5t|i}tj||} tj|| }tj|| \} } } } tj | | }tj"| |t%| ||t'j(|||||}|j+||j-|j/|cdddS#1swYyxYw)a Find all mismatches between the original model and the exported model. Experimental. The API is subject to change. This tool helps debug the mismatch between the original PyTorch model and exported ONNX model. It binary searches the model graph to find the minimal subgraph that exhibits the mismatch. Args: model: The model to be exported. input_args: The input arguments to the model. do_constant_folding: Same as `do_constant_folding` in :func:`torch.onnx.export`. training: Same as `training` in :func:`torch.onnx.export`. opset_version: Same as `opset_version` in :func:`torch.onnx.export`. keep_initializers_as_inputs: Same as `keep_initializers_as_inputs` in :func:`torch.onnx.export`. verbose: Same as `verbose` in :func:`torch.onnx.export`. options: The options for the mismatch verification. Returns: A GraphInfo object that contains the mismatch information. Example:: >>> import torch >>> import torch.onnx.verification >>> torch.manual_seed(0) >>> opset_version = 15 >>> # Define a custom symbolic function for aten::relu. >>> # The custom symbolic function is incorrect, which will result in mismatches. >>> def incorrect_relu_symbolic_function(g, self): ... return self >>> torch.onnx.register_custom_op_symbolic( ... "aten::relu", ... incorrect_relu_symbolic_function, ... opset_version=opset_version, ... ) >>> class Model(torch.nn.Module): ... def __init__(self) -> None: ... super().__init__() ... self.layers = torch.nn.Sequential( ... torch.nn.Linear(3, 4), ... torch.nn.ReLU(), ... torch.nn.Linear(4, 5), ... torch.nn.ReLU(), ... torch.nn.Linear(5, 6), ... ) ... def forward(self, x): ... return self.layers(x) >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ONNX) >>> graph_info = torch.onnx.verification.find_mismatch( ... Model(), ... (torch.randn(2, 3),), ... opset_version=opset_version, ... ) ===================== Mismatch info for graph partition : ====================== ================================ Mismatch error ================================ Tensor-likes are not close! Mismatched elements: 12 / 12 (100.0%) Greatest absolute difference: 0.2328854203224182 at index (1, 2) (up to 1e-07 allowed) Greatest relative difference: 0.699536174352349 at index (1, 3) (up to 0.001 allowed) ==================================== Tree: ===================================== 5 X __2 X __1 \u2713 id: | id: 0 | id: 00 | | | |__1 X (aten::relu) | id: 01 | |__3 X __1 \u2713 id: 1 | id: 10 | |__2 X __1 X (aten::relu) id: 11 | id: 110 | |__1 \u2713 id: 111 =========================== Mismatch leaf subgraphs: =========================== ['01', '110'] ============================= Mismatch node kinds: ============================= {'aten::relu': 2} N)rrrr#r)r#r rr<rrr1r2rr3r4rrr rr_get_named_param_dict_apply_friendly_debug_namesrr r,r rr)rwrrrrr#rrrCrrr torch_outmodulerras r!r r sPv%'"55 c5::**333 UZZ,,11 1 5j"E))%1BC,,UD9+0+B+B5$+O(vy&11%@  ))%=     ' '$7!+,G       )((*$$&5s #CFF)rzlist | np.ndarray)rrB)T)r _OutputsType)rwstr | io.BytesIOrxz Sequence[str])rwrE)rwrEr)r)rjrDrrDrr#)rjrDrrrr#)r.r-r%r$) r _ModelTyperrEr_InputArgsTyper_InputKwargsType | NonerSequence[_InputArgsType] | Nonerr#) rw(torch.nn.Module | torch.jit.ScriptModuler3Sequence[tuple[tuple[Any, ...], Mapping[str, Any]]]rrrzfCallable[[torch.nn.Module, tuple[Any, ...], Mapping[str, Any], _experimental.ExportOptions], _C.Graph]rru) rwrJrrrzMapping[str, Any]rrrrr2)rrrrrdict[str, Any] | Nonerz"tuple[torch.Graph, dict[str, Any]])r rrrrrrz!tuple[bytes, Mapping[str, bytes]])rwrJrrKrz"_experimental.ExportOptions | Nonerru)rwrFrrGrrHrr$rzDMapping[str, Mapping[int, str] | Mapping[str, Sequence[int]]] | Noner[Sequence[str] | NonerrMr_C_onnx.TrainingModer int | Noner#r$rr$r0r$r?r$rrIrr.r') rrrrrrrrLrQr.rr-)rCollection[torch.Node]rr))rUrrrPrr$)rr(rrPrr$)rwrJrrrr$rrNrrOr#r$rr$rr.rr)\r __future__rr5rr/rrenumrr7rr<r:rtypingrrrrrr r rArEr<torch._C._onnxr _onnxr torch.onnxr r rrtorch.onnx._globalsrtorch.onnx._internalr torch.typesrrqr=ndarray _NumericTypennModuler ScriptModulerFrGru_InputKwargsTyperDEnumr dataclassr#r6rCrKrWrkr|rrrrrrrrrrrrr r r*r.rrrEr]r_rrrrrrr rr r!rbs& #    MMM II'1+VU\\2::56 588??EII$:$:: ; u||U38_4538$Xl+X56 0$))0 #5#5#5L$2>=K ,9.  .. .!.bHH H!H840+?JN:/G/G"/G/G* /G < /G ! /GdB,B,J& 3&J&0& & &R 3  0   6= 3= = =0 =  =F*.< </<'<( <~/ ' H:>( 3(J(7( (\-1 $ (,)-%,%9%9%>%> $(,"#>B*.!T T T *T  T  T &T 'T #T T "&T T T T <T (!T v*.7; K KK0K' K 5 K K K\O:O:d@Q@QF v5v5v5r:!!%%,%9%9%>%> $(,*. 3#    "& (r