
    ɯwg                   `	   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) ddl*Z*ddl+Z*ddl,Z*ddl-m.c m/Z0 dd	l1m2Z2 dd
l3m4Z4 ddl5m6Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z= ddl>m?Z? ddl@mAZAmBZB ddlCmDZD ddlEmFZG ddlHmIZImJZJ ddlKmLZL ddlMmNZNmOZO ddlPmQZQ ddlRmFZF ddlSmTZT ddlUmVZV  e	j                  eX      ZYe*j                  j                  Z[e*j                  j                  Z\eZ]e&e]e*j                  j                  f   Z_ G d de       Z` G d de       Za G d d e       Zb e%d!      Zce&e*j                  j                  j                  eff   Zg G d" d#      Zh eh       Zi G d$ d%      Zj G d& d'ek      Zle&ejelf   Zmdd(Zn G d) d*      Zo G d+ d,e      Zp G d- d.ep      Zq G d/ d0ep      Zr G d1 d2ep      Zs G d3 d4ep      Zt G d5 d6ep      Zue#ed7f   Zv G d8 d9eu      Zw G d: d;ew      Zx G d< d=ew      Zy G d> d?ew      Zz G d@ dAeu      Z{ G dB dCe{      Z| G dD dEe{      Z} G dF dGe{      Z~ G dH dIep      Z G dJ dKep      Z G dL dMep      Z G dN dO      Z G dP dQe       Zej                   G dR dS             Zej                   G dT dUe             Zej                   G dV dWe             Zej                   G dX dYe             ZddZZdd[Zedd\df	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd]Z e       Zd^ed_<   	 	 	 	 	 	 	 	 	 	 	 	 dd`Z ee      j"                  daz  dbz  Zg Zdcedd<   edd\def	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddfZ eGj*                  deg      	 	 d	 	 	 	 	 	 	 	 	 	 	 ddh       Zefdedi	 	 	 	 	 	 	 	 	 ddjZefdedi	 	 	 	 	 	 	 	 	 ddkZddlZ ej4                  dm      ZddnZddoZddpZddqZddrZ G ds dt      ZdduZ	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 ddvZ e*jH                         dwddx	 	 	 	 	 	 	 	 	 ddy       Z e*jL                         ddz       Zdd{Zdd|Zdd}Zdd~ZddZ e       Zd^ed<   	 d	 	 	 	 	 	 	 ddZddZddZy)a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)Path)AnyCallableDefaultDictDict	GeneratorIterableListMappingNoReturnOptionalProtocolSequenceSetTupleTypeTypeVarUnion)Self	TypeGuard)enable_python_dispatcher)counters)trace)is_integer_dtype)unset_fake_temporarily)make_fxguard_size_oblivious)immutable_dictimmutable_list)GraphTransformObserver   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_typec                       e Zd ZU ded<   ddZy)SearchFnstr__name__c                     y N selfargskwargss      d/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/torch/_inductor/pattern_matcher.py__call__zSearchFn.__call__n           Nr;   r   r<   r   returnr   )r5   
__module____qualname____annotations__r>   r8   r@   r=   r3   r3   k   s    Mr@   r3   c                      e Zd ZddZy)	ReplaceFnc                     y r7   r8   r9   s      r=   r>   zReplaceFn.__call__s   r?   r@   NrA   r5   rC   rD   r>   r8   r@   r=   rG   rG   r       r@   rG   c                  $    e Zd Z	 	 	 	 	 	 	 	 ddZy)TraceFnc                     y r7   r8   )r:   fnr;   r<   s       r=   r>   zTraceFn.__call__x        	r@   N)rN   zUnion[SearchFn, ReplaceFn]r;   r   r<   r   rB   torch.fx.GraphModulerI   r8   r@   r=   rL   rL   w   s'    ,58DG	r@   rL   Tc                      e Zd ZddZy)Multiplec                2    dt               vs| t        u sJ y y )NMULTIPLE)globalsrU   r:   s    r=   __init__zMultiple.__init__   s    *dh.>>>.>*r@   NrB   None)r5   rC   rD   rX   r8   r@   r=   rS   rS      s    ?r@   rS   c                       e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   	 	 d	 	 	 	 	 	 	 	 	 d fdZedd       ZddZddZ	ddZ
d dZd!dZd"dZ	 	 	 	 	 	 d#dZ	 	 d$	 	 	 	 	 	 	 	 	 d%dZ xZS )&Matcha  
    Represents a successfully matched pattern.

    The `Match` object is returned to represent a successfully matched
    pattern. Included in the Match are the pattern that was matched, the graph
    nodes matched, and any args that were used during the matching.

    The args and kwargs are specific to the type of pattern that was matched and
    provide hints about what was matched.
    PatternExprpattern	List[Any]r;   zDict[str, Any]r<   List[torch.fx.Node]nodesz'Dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxzOptional[torch.fx.Graph]replacement_graphc                    t         |           || _        t        |xs g       | _        |xs i | _        g | _        i | _        || _        d | _	        y r7   )
superrX   r^   listr;   r<   ra   rb   rd   re   )r:   rd   r^   r;   r<   	__class__s        r=   rX   zMatch.__init__   sO     	$	l
!%r@   c                .    | j                   j                  S r7   )rd   graphrW   s    r=   rk   zMatch.graph   s    xx~~r@   c                2   | j                   rwt        | j                   j                               t        |j                   j                               z  D ]-  }| j                   |   |j                   |   k7  s#t        d|       | j                  j                  |j                         | j                  j                  |j                         | j                   j                  |j                          | j                  j                  |j                         y )Nzkwarg mismatch: {})	r<   setkeysFailedMatchr;   extendra   updaterb   )r:   otherkeys      r=   rp   zMatch.extend   s    ;;4;;++-.U\\5F5F5H1II A;;s#u||C'88%&:C@@A 			$

%++&5<<(EMM*r@   c                f    | j                   rt        | j                         g| _         | S g | _         | S r7   )r;   tuplerW   s    r=   bundlezMatch.bundle   s.    *.))U499%&	 :<	r@   c                <    d| j                    d| j                   dS )NzMatch(..., , )r;   r<   rW   s    r=   __repr__zMatch.__repr__   s    TYYKr$++a88r@   c                    | j                   }t        | j                        D ]-  }|j                  r|j                  r|j                  |       / y r7   )rk   reversedra   _erasedusers
erase_node)r:   rk   ns      r=   erase_nodeszMatch.erase_nodes   s>    

$**% 	$A99QWW  #	$r@   c                    | j                   j                  D cg c]  }|| j                   j                  |   nd ! c}S c c}w r7   )rd   outputspattern_to_node)r:   ps     r=   output_nodeszMatch.output_nodes   sF     XX%%
 -.MTXX%%a(tC
 	
 
s   $A c                B    t        d | j                         D              S )Nc              3  &   K   | ]	  }|s|  y wr7   r8   ).0r   s     r=   	<genexpr>z$Match.output_node.<locals>.<genexpr>   s     8!aA8s   )nextr   rW   s    r=   output_nodezMatch.output_node   s    8t002888r@   c                \    t         j                  | | j                  j                  ||       y r7   )ReplacementPatternEntryreplace_with_graphrd   rk   )r:   re   r;   s      r=   r   zMatch.replace_with_graph   s$     	 22$((.."3T	
r@   c                   ddl m}m} t        |j                  |      r|j                  |j                  nt        j                         }|5  |t        j                  t        |      } ||t        j                  j                  |d             }t        j                  | | j                  j                   ||       ddd       y# 1 sw Y   yxY w)a	  Replace with a graph generated by tracing the replacement_fn.

        Args:
            run_functional_passes (bool). If we should run passes that
                assume functional IR (like DCE, remove_noop_ops), on the
                replacement graph.

        r   )NullHandlerVN)run_functional_passesc                     | j                   d   S Nvalmeta)args    r=   <lambda>z*Match.replace_by_example.<locals>.<lambda>   s    388E? r@   )torch._inductor.virtualizedr   r   
isinstance	fake_mode
contextlibnullcontext	functoolspartialfwd_onlytorchfxmap_argr   r   rd   rk   )	r:   replacement_fnr;   trace_fnr   r   r   contextreplacements	            r=   replace_by_examplezMatch.replace_by_example   s     	? q{{K8Q[[=P KK'') 	  	$,,4I # 0 07R SK $66		 	 	s   A2CC)NN)
rd   rc   r^   r]   r;   zOptional[Sequence[Any]]r<   zOptional[Dict[str, Any]]rB   rZ   )rB   torch.fx.Graph)rr   r\   rB   rZ   )rB   r\   rB   r4   rY   )rB   zList[Optional[torch.fx.Node]])rB   torch.fx.Node)re   r   r;   Sequence[Any]rB   rZ   NT)
r   rG   r;   r   r   zOptional[TraceFn]r   boolrB   rZ   )r5   rC   rD   __doc__rE   rX   propertyrk   rp   rv   r{   r   r   r   r   r   __classcell__ri   s   @r=   r\   r\      s    	 
O44	// )-+/&& & &	&
 )& 
&&  +
9$
9
!/
7D
	
 '+&*$!$ $ $	$
  $$ 
$r@   r\   c                  4    e Zd ZU dZded<   ddZd	dZd
dZy)ro   z
    Represents a unsuccessful match.

    The `FailedMatch` object is returned to represent a failure to match a
    pattern.
    r4   format_stringc                f    || _         t        |      dkD  rt        d|       || _        || _        y )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r   lenRuntimeErrorr;   r<   )r:   r   r;   r<   s       r=   rX   zFailedMatch.__init__  sB    * }#hivhwx  	r@   c                b     | j                   j                  | j                  i | j                  S r7   )r   formatr;   r<   rW   s    r=   __str__zFailedMatch.__str__  s(    (t!!(($))Ct{{CCr@   c                     yNFr8   rW   s    r=   __bool__zFailedMatch.__bool__       r@   N)r   r4   r;   r   r<   r   rB   rZ   r   rB   r   )r5   rC   rD   r   rE   rX   r   r   r8   r@   r=   ro   ro     s     	Dr@   ro   c                    t        |       S )z
    TypeGuards cannot act on `self`. Thus this function exists to let mypy
    recognize FailedMatch.__bool__ as a TypeGuard.
    )r   )ms    r=   is_matchr   '  s    
 7Nr@   c                  d    e Zd ZU dZded<   ded<   ded<   ded	<   	 d	 	 	 	 	 	 	 ddZddZddZy
)rc   zC
    Internal state needed while running PatternExpr._match().
    List[Optional[PatternExpr]]r   z*Dict[PatternExpr, Optional[torch.fx.Node]]r   r   rk   zList[NodeOrConstant]exclusive_node_setNc               V    || _         |i n
t        |      | _        || _        g | _        y r7   )r   dictr   rk   r   )r:   r   r   rk   s       r=   rX   zMatchContext.__init__9  s.     %4%<r$BW
"$r@   c                    || j                   v r)| j                   |   |k(  rt        | |      S t        d      S |j                  ||       }|| j                   vsJ |r|nd| j                   |<   |S )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   r\   ro   _match)r:   r^   noder   s       r=   matchzMatchContext.matchE  sy    d***##G,4T7++"#=>>NN4&d2222201tW%r@   c                    | j                   j                         D ci c]  \  }}|j                         r||| c}}S c c}}w r7   )r   itemshas_multiple_users)r:   r^   r   s      r=   filter_multi_user_patternsz'MatchContext.filter_multi_user_patternsQ  sL     "&!5!5!;!;!=
))+0@ TM
 	
 
s   Ar7   )r   r   r   z*Optional[Dict[PatternExpr, torch.fx.Node]]rk   r   rB   rZ   )r^   r]   r   NodeOrConstantrB   MatchResult)rB   z Dict[PatternExpr, torch.fx.Node])r5   rC   rD   r   rE   rX   r   r   r8   r@   r=   rc   rc   /  s^     )(??,,
 GK
%,
% D
%
 
% 

%

r@   rc   c                  V    e Zd ZdZed	d       Zd
dZddZddZ	 	 	 	 	 	 ddZ	ddZ
y)r]   z+
    Base class for types of patterns.
    c                     y r7   r8   r:   r   rd   s      r=   r   zPatternExpr._match^  s    r@   c                    	 t        | g|j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wNrk   )rc   rk   r   ro   r:   r   es      r=   r   zPatternExpr.matchb  s<    	djj9??dKK 	H	s   '* 	?:??c                     yr   r8   rW   s    r=   r   zPatternExpr.has_multiple_usersh  r   r@   c                4    | j                   j                  dz   S )Nz())ri   r5   rW   s    r=   r{   zPatternExpr.__repr__k  s    ~~&&--r@   c              #  L   K   | |j                   v r|j                   |     y y wr7   )r   r:   rd   searcheds      r=   find_anchor_nodeszPatternExpr.find_anchor_nodesn  s,      3&&&%%d++ 's   "$c                .    t        || j                        S )z
        Compare two `PatternExpr`s and return true if they are the
        same. Note this is NOT matching a pattern - it is comparing the pattern
        structures (for debugging).
        )r   ri   )r:   rr   s     r=   
pattern_eqzPatternExpr.pattern_eqt  s     %00r@   Nr   r   rd   rc   rB   r   r   r   rB   r   r   r   rd   rc   r   Set[torch.fx.Node]rB   z.Generator[Optional[torch.fx.Node], None, None]rr   r   rB   r   )r5   rC   rD   r   r   r   r   r   r{   r   r   r8   r@   r=   r]   r]   Y  sL      .,,+=,	7,1r@   r]   c                      e Zd ZdZddZy)Argzn
    Capture an arg which will become an input to the handler.  Args are
    passed in depth first order.
    c                     t        || |g      S )N)r;   r\   r   s      r=   r   z
Arg._match  s    S$dV,,r@   Nr   r   rd   rc   rB   r   r5   rC   rD   r   r   r8   r@   r=   r   r   }  s    
-r@   r   c                  (    e Zd ZdZddZddZddZy)	Ignoredz4
    Match an arg, but don't pass it to handler
    c                    t        ||       S r7   r   r   s      r=   r   zIgnored._match  s    S$r@   c                     y)N*r8   rW   s    r=   r{   zIgnored.__repr__  s    r@   c                     y)Nz	Ignored()r8   )r:   pps     r=   pretty_printzIgnored.pretty_print  s    r@   Nr   r   r   PatternPrettyPrinterrB   r4   )r5   rC   rD   r   r   r{   r   r8   r@   r=   r   r     s     r@   r   c                  @     e Zd ZdZd fdZddZddZd	 fdZ xZS )

KeywordArgD
    Capture a kwarg which will become an input to the handler.
    c                0    t         |           || _        y r7   rg   rX   namer:   r   ri   s     r=   rX   zKeywordArg.__init__      	r@   c                "    d| j                   dS )NzKeywordArg(ry   r   rW   s    r=   r{   zKeywordArg.__repr__  s    TYYM++r@   c                6    t        || | j                  |i      S )Nr<   )r\   r   r   s      r=   r   zKeywordArg._match  s    S$		4'899r@   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S r7   typingcastr   rg   r   r   r:   rr   ri   s     r=   r   zKeywordArg.pattern_eq  5    D%(w!%(DTYY%**-DDr@   r   r4   rB   rZ   r   r   r   	r5   rC   rD   r   rX   r{   r   r   r   r   s   @r=   r   r     s#    ,:E Er@   r   c                  L     e Zd ZU dZded<   d fdZd	dZd
dZd fdZ xZ	S )ExclusiveKeywordArgr   r4   r   c                0    t         |           || _        y r7   r   r  s     r=   rX   zExclusiveKeywordArg.__init__  r  r@   c                "    d| j                   dS )NzExclusiveKeywordArg(ry   r  rW   s    r=   r{   zExclusiveKeywordArg.__repr__  s    %dii]!44r@   c                    ||j                   v rt        d      S |j                   j                  |       t        || | j                  |i      S )Nzexclusive arg appears twicer  )r   ro   appendr\   r   r   s      r=   r   zExclusiveKeywordArg._match  sH    3)))<==%%d+S$		4'899r@   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S r7   r  r  s     r=   r   zExclusiveKeywordArg.pattern_eq  r  r@   r  r   r   r   )
r5   rC   rD   r   rE   rX   r{   r   r   r   r   s   @r=   r  r    s*     I5:E Er@   r  c                       e Zd ZU dZded<   ded<   	 d	 	 	 	 	 d fdZeedd              ZddZ	dd	Z
dd
Z	 	 	 	 	 	 ddZddZddZd fdZ xZS )_TargetExprz7
    Base class for filtering match by node.target
    zList[FnsType]fnszSet[FnsType]fns_setc                Z   t         |           t        |      st        |t              r|gn
t        |      }|D ]O  t        t        j                  j                        s(|j                  fdj                         D               Q || _        t        |      | _        || _        y )Nc              3  6   K   | ]  }t        |        y wr7   )getattr)r   overloadrN   s     r=   r   z'_TargetExpr.__init__.<locals>.<genexpr>  s     PX72x0Ps   )rg   rX   callabler   r4   rh   r   _opsOpOverloadPacketrp   	overloadsr  rm   r  r   )r:   r  r   rN   ri   s      @r=   rX   z_TargetExpr.__init__  s     	}
3(<se$s) 	QB"ejj99:

PPP	Q 3x
r@   c                     y r7   r8   rW   s    r=   opz_TargetExpr.op  rO   r@   c                t   | j                   d   }t        |t              s|j                  }t	        | j                         dkD  rd| dS | j                   d   t        t        |d       u rd| S t        | j                   d   t        j                  j                        rt        | j                   d         S |S )Nr   r/   [z, ...]torch.)	r  r   r4   r5   r   r  r   r  
OpOverload)r:   
first_reprs     r=   fns_reprz_TargetExpr.fns_repr  s    XXa[
*c*#,,Jtxx=1zl&))XXa[GE:t<<J<((UZZ%:%:;txx{##r@   c                    | j                   t        u rd}n"| j                   dk7  rd| j                    d}nd}| j                  j                   d| j	                          | dS )Nz
, MULTIPLEr/   rx   ry    ()r   rU   ri   r5   r)  )r:   comma_userss     r=   r{   z_TargetExpr.__repr__  s^    ::!&KZZ1_tzzl!,KK..))*!DMMO+<[MKKr@   c                X    t        | j                  t              xs | j                  dkD  S )Nr/   )r   r   rS   rW   s    r=   r   z_TargetExpr.has_multiple_users  s     $**h/A4::>Ar@   c                    t         r7   NotImplementedErrorr   s      r=   r   z_TargetExpr.find_anchor_nodes  s
     "!r@   c                    t        |t        j                  j                        xr2 |j                  | j                  k(  xr t        |      | j                  v S r7   )r   r   r   Noder#  extract_targetr  )r:   r   s     r=   
_match_fnsz_TargetExpr._match_fns  sD    tUXX]]+ 5477"5t$4	
r@   c                    | |j                   v xs6 | j                  t        u xs" t        |j                        | j                  k(  S r7   )r   r   rU   r   r   s      r=   _match_usersz_TargetExpr._match_users  s=    CKK -zzX%-4::$**,	
r@   c                    t        j                  t        |      }t        |   |      xrO | j
                  |j
                  k(  xr4 | j                  |j                  k(  xr | j                  |j                  k(  S r7   )r	  r
  r   rg   r   r#  r  r   r  s     r=   r   z_TargetExpr.pattern_eq  sf    D%(Gu% *588#*EII%* 

ekk)		
r@   )r/   )r  z!Union[FnsType, Sequence[FnsType]]r   zUnion[Multiple, int]rB   rZ   r   r   r   r   r   rB   r   )r   r   rd   rc   rB   r   r   )r5   rC   rD   r   rE   rX   r   r   r#  r)  r{   r   r   r5  r7  r   r   r   s   @r=   r  r    s     
 UV4=Q	   LB""+="	7"



 
r@   r  .c                       e Zd ZdZdd	 	 	 	 	 	 	 	 	 d fdZe	 	 	 	 	 	 dd       Ze	 	 	 	 	 	 dd       ZddZddZ	dd	Z
	 	 	 	 	 	 dd
Zd fdZ xZS )_TargetArgsExprzE
    Base class for filtering match by node.{target,args,kwargs}
    r/   )_usersc               j   t         |   ||       t        |      | _        t	        |      | _        t        d t        j                  ||j                               D              r| j                  | _        n| j                  | _        | j                  | j                  | j
                        | _        y )Nc              3  R   K   | ]  }t        |t        t        t        f       ! y wr7   )r   r   rh   ru   r   xs     r=   r   z+_TargetArgsExpr.__init__.<locals>.<genexpr>'  s&      
 q4u-.
s   %')rg   rX   ru   r;   r   r<   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)r:   r  r<  r;   r<   ri   s        r=   rX   z_TargetArgsExpr.__init__  s     	f%$K	6l 
__T6==?;
 
  ..DL..DL $TYY Dr@   c                n    g | |j                         }t        |       g|j                         }||fS r7   )rD  r   rn   )r;   r<   rD  specs       r=   rG  z_TargetArgsExpr.simple_flatten0  s:     +4*&--/*D	*FKKM*t|r@   c                \    dfdt        j                  | |g      \  }} |      }||fS )Nc           
     $   | j                   | S t        t        t        t        t        t
        i}t        j                  |j                  | j                   | j                         | j                  t        t        | j                                    S r7   )typer$   rh   ru   r#   r   pytreeTreeSpecgetr   mapchildren_specs)smapping	norm_specs     r=   rU  z1_TargetArgsExpr.pytree_flatten.<locals>.norm_spec<  sf    vv~%tUD.$OG??AFFAFF+		SA$4$456 r@   )rS  pytree.TreeSpecrB   rV  )rN  tree_flatten)r;   r<   flatrJ  rU  s       @r=   rE  z_TargetArgsExpr.pytree_flatten8  s5    	 (($8
dTzr@   c                   | j                         gt        t        | j                        | j                  j                         D cg c]  \  }}| d|  c}}}| j                  t        u r|j                  d       n-| j                  dk7  r|j                  d| j                          | j                  j                   ddj                  |       dS c c}}w )N=_users=MULTIPLEr/   _users=r,  rx   ry   )r)  rQ  reprr;   r<   r   r   rU   r  ri   r5   join)r:   kvr;   s       r=   r{   z_TargetArgsExpr.__repr__J  s    MMO
tyy!
 &*[[%6%6%89TQ1QCj9

 ::!KK)*ZZ1_KK'$**./..))*!DIIdO+<A>> :s   Cc           
        | j                         gfd| j                  D        | j                  j                         D cg c]  \  }}| dj	                  |        c}}}| j
                  t        u r|j                  d       n-| j
                  dk7  r|j                  d| j
                          d}| j                  j                   d|j                  |       dS c c}}w )	Nc              3  @   K   | ]  }j                  |        y wr7   )r   )r   r@  r   s     r=   r   z/_TargetArgsExpr.pretty_print.<locals>.<genexpr>Y  s     4Qbooa 4s   rZ  r[  r/   r\  rx   r,  ry   )r)  r;   r<   r   r   r   rU   r  ri   r5   r^  )r:   r   r_  r`  r;   
joiner_strs    `    r=   r   z_TargetArgsExpr.pretty_printV  s    MMO
4$))4
 7;kk6G6G6IJda1R__Q'()J

 ::!KK)*ZZ1_KK'$**./
..))*!JOOD,A+B!DD Ks    C'c                j   | j                  |      r+t        |j                        t        | j                        k7  rt        d||       S | j	                  ||      st        d|       S |j                  }|j
                  }t        |      t        | j
                        k  rddlm}  ||j                  |j                  |j
                        }|t        d||       S |\  }}t        |      t        | j                        k(  rDt        |      t        | j
                        k\  r#|D ci c]  }|| j
                  v s|||    }}n/t        d||       S |D ci c]  }|| j
                  v s|||    }}| j                  ||      \  }}	| j                  \  }
}|	|k7  rt        d|	|      S t        |      t        |
      k(  sJ t        ||       }t        t        j                         |
|      D ]  \  }}}t        |t               r3|j#                  ||      }t%        |      s|c S |j'                  |       Jt        |t(        j*                  j,                        s||k7  stt        d||      c S  |j.                  j1                  |       |j                  |j2                  | <   |S c c}w c c}w )Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionzargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})r5  r   r;   ro   r7  r<   torch.fx.operator_schemasre  targetrF  rH  r\   ziprB  countr   r]   r   r   rp   r   r   r3  ra   r  rb   )r:   r   rd   _args_kwargsre  normalized_args_and_kwargsi
node_items	node_spec
self_items	self_specr   r^   
child_nodechild_matchs                   r=   r   z_TargetArgsExpr._matchd  sa   t$DII#dii.(HGtTT  s+2D99		++w<#dkk**D);TYY*& *1"#KTSWXX!;wu:TYY/CLCDT4T6=RdkkAQq'!*}RGR&@$  /6Jdkk9Iq'!*}JGJ $UG <
I $ 5 5
I	!5y)LL:#j/111#t&))//*;Z&T 		"Aw
';/!ii<,&&%J6*:O"94 		 	
t++		$7 S Ks   )J+=J+J0-J0c              #    K   | |j                   v r|j                   |     y| j                  d   D ]  }t        |t              s|j	                  ||      D ]d  }t        |t
        j                  j                        s(|j                  D ].  }||vs| j                  |      s| |j                  |       0 f  yw)a  
        This is used when we are matching a pattern with multiple outputs.
        There is a partial match (stored in ctx) and we want to walk
        this pattern to find a connection to an already-matched node.

        Yields candidate nodes that `self._match` might like.
        Nr   )r   rH  r   r]   r   r   r   r3  r   r5  add)r:   rd   r   r^   
other_noder   s         r=   r   z!_TargetArgsExpr.find_anchor_nodes  s      3&&&%%d++,,Q/ 		3G';/")";";C"J 3J%j%((--@  * 0 0 3x/#t4&*
 (T 2	33		3s   ACACC&Cc                   t        j                  t        |      }t        |   |      xrW | j
                  d   |j
                  d   k(  xr6 t        d t        | j
                  d   |j
                  d         D              S )Nr/   c              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wr7   r   r]   r   r   abs      r=   r   z-_TargetArgsExpr.pattern_eq.<locals>.<genexpr>  6      Aq $.a#=Q16I   35r   )r	  r
  r   rg   r   rH  allrh  r  s     r=   r   z_TargetArgsExpr.pattern_eq  s    D%(Gu% %%a(E,B,B1,EE  5 5a 8%:P:PQR:ST 	
r@   )
r  z/Union[torch.fx.node.Target, str, Sequence[Any]]r;   r   r<  zUnion[int, Multiple]r<   r   rB   rZ   )r;   r   r<   zMapping[Any, Any]rB   z9Tuple[Sequence[Any], Union[_SimpleSpec, pytree.TreeSpec]]r   r   r   r   r   )r5   rC   rD   r   rX   staticmethodrG  rE  r{   r   r   r   r   r   r   s   @r=   r;  r;    s     ()	E<E E %	E
 E 
E& %6	B  %6	B "
?E0d33+=3	732	
 	
r@   r;  c                      e Zd ZdZdZy)CallFunctionzR
    Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
    call_functionNr5   rC   rD   r   r#  r8   r@   r=   r  r    s     
Br@   r  c                      e Zd ZdZdZy)
CallMethodzW
    Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
    call_methodNr  r8   r@   r=   r  r         
Br@   r  c                      e Zd ZdZdZy)
CallModulezP
    Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
    call_moduleNr  r8   r@   r=   r  r    r  r@   r  c                      e Zd ZdZddZy)_TargetExprVarArgsz[
    Matches a call_function node with any arguments which are passed into the pattern
    c                   | j                  |      st        d      S | j                  ||      st        d      S t        ||       }|j                  j                  |       |j                  |j                  | <   |j                  j                  |j                         |j                  j                  |j                         |S )Nfunction_mismatchmultiple_users)r5  ro   r7  r\   ra   r  rg  rb   r;   rp   r<   rq   )r:   r   rd   r   s       r=   r   z_TargetExprVarArgs._match  s    t$233  s+/00#t	t++		$	dii 	$r@   Nr   r   r8   r@   r=   r  r    s    r@   r  c                      e Zd ZdZy)CallFunctionVarArgsr  Nr5   rC   rD   r#  r8   r@   r=   r  r    s    	Br@   r  c                      e Zd ZdZy)CallMethodVarArgsr  Nr  r8   r@   r=   r  r        	Br@   r  c                      e Zd ZdZy)CallModuleVarArgsr  Nr  r8   r@   r=   r  r    r  r@   r  c                  B     e Zd ZdZdd fdZddZd	dZd
 fdZ xZS )ListOfz$
    Matches a repeated pattern
    c                b    t         |           t        |t              sJ || _        || _        y r7   )rg   rX   r   r]   r^   r   )r:   r^   r   ri   s      r=   rX   zListOf.__init__  s,    ';///r@   c                N    | j                   j                   d| j                   dS Nr,  ry   )ri   r5   r^   rW   s    r=   r{   zListOf.__repr__  $    ..))*!DLL>;;r@   c                (   t        |t        t        f      rt        |      dk(  rt	        d      S t        ||       }|j                         }d}t        |      D ]  \  }}t        |j                  ||j                        }|j                  | j                  |      }	|j                         }t        |	      s| j                  st	        d||	      c S {d}|j                  |	j!                                 |st	        d      S |j!                         S )Nr   non_listFr   zlist[{}]: {}Tzlist: no_match)r   rh   ru   r   ro   r\   r   	enumeraterc   r   rk   r   r^   r   r   rp   rv   )
r:   r   rd   r   r   matchedrm  rr  	child_ctxrs  s
             r=   r   zListOf._match  s    $u.#d)q.z**#t 88:&t_ 	+MAz$_J4D4DI $//$,,
CK'BBDOK(||&~q+FFGHH['')*	+ /00xxzr@   c                    t        j                  t        |      }t        |   |      xr@ | j
                  j	                  |j
                        xr | j                  |j                  k(  S r7   )r	  r
  r   rg   r   r^   r   r  s     r=   r   zListOf.pattern_eq  sU    D%(Gu% .''6.-	
r@   )F)r^   r]   r   r   rB   rZ   r   )r   r`   rd   rc   rB   r   r   r  r   s   @r=   r  r    s!    <0
 
r@   r  c                  ~     e Zd ZU ded<   d fdZedd       ZddZddZddZ		 	 	 	 	 	 ddZ
dd	Zd fd
Z xZS )MultiOutputPatternr   r   c                    t         |           t        |d   t              sJ t	        d |D              sJ |       t        |      | _        |d   j                  | _        y )Nr   c              3  H   K   | ]  }|d u xs t        |t                y wr7   )r   r]   r?  s     r=   r   z.MultiOutputPattern.__init__.<locals>.<genexpr>(  s#     Lq19:
1k ::Ls    ")rg   rX   r   r  r  rh   r   r#  )r:   r   ri   s     r=   rX   zMultiOutputPattern.__init__%  sU    '!*k222LGLLUgULG}!*--r@   c                h    t        j                  t        | j                  d         }|j                  S )Nr   )r	  r
  r  r   r  )r:   outputs     r=   r  zMultiOutputPattern.fns,  s&     [$,,q/:zzr@   c                N    | j                   j                   d| j                   dS r  )ri   r5   r   rW   s    r=   r{   zMultiOutputPattern.__repr__2  r  r@   c                    | j                   D cg c]  }|j                  |       }}dd }| j                  j                   d|j	                  |       }| d}|S c c}w )Nz,
z  z([z
]))r   r   ri   r5   r^  )r:   r   r@  r;   rc  str_outs         r=   r   zMultiOutputPattern.pretty_print5  sh    ,0LL9q"994&\
^^,,-R
0E/FGIT"	 :s   Ac                $   t        j                  t        | j                  d         }|j	                  ||      }t        |      s|S | j                  dd  D ]7  }|| j                  ||      }t        |      s|c S |j                  |       9 |S )Nr   r/   )r	  r
  r  r   r   r   _match_from_anchorsrp   )r:   r   rd   r  r   r^   rs  s          r=   r   zMultiOutputPattern._match<  s    [$,,q/:IIfd#{H||AB' 	"G227C@KK(""HH[!	" r@   c                    t        |j                        }t        d      }|j                  |t	                     D ]3  }|j                  ||      }t        |      r|c S t        |      |_        5 |S )Nzno anchor found)r   r   ro   r   rm   r   r   )r:   r^   rd   priorr   r   s         r=   r  z&MultiOutputPattern._match_from_anchorsL  sl     S(()$%67--c359 	.D		'4(A{"&u+C	. r@   c                    	 t        | j                  |j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wr   )rc   r   rk   r   ro   r   s      r=   r   zMultiOutputPattern.matchY  s>    	DJJ?EEdDQQ 	H	s   03 	AAAAc                   t        j                  t        |      }t        |   |      xr] t        | j                        t        |j                        k(  xr0 t        d t        | j                  |j                        D              S )Nc              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wr7   ry  rz  s      r=   r   z0MultiOutputPattern.pattern_eq.<locals>.<genexpr>d  r}  r~  )	r	  r
  r   rg   r   r   r   r  rh  r  s     r=   r   zMultiOutputPattern.pattern_eq_  sm    D%(Gu% DLL!S%77 emm< 	
r@   )r   zSequence[Optional[PatternExpr]]rB   rZ   )rB   z-Union[Callable[..., Any], str, Sequence[Any]]r   r   r   )r^   r]   rd   rc   rB   r   r   r   )r5   rC   rD   rE   rX   r   r  r{   r   r   r  r   r   r   r   s   @r=   r  r  "  s]    ((   
< ")5		
 	
r@   r  c                  J     e Zd ZdZd fdZedd       ZddZd	 fdZ xZ	S )
RepeatedExprzp
    Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
    c                R    t         |           || _        |j                  | _        y r7   )rg   rX   inner_patternr#  )r:   r  ri   s     r=   rX   zRepeatedExpr.__init__p  s#    *""r@   c                .    | j                   j                  S r7   )r  r  rW   s    r=   r  zRepeatedExpr.fnsu  s    !!%%%r@   c                   |j                  | j                  |      }t        |      s|S |j                  j	                  | j                         | j                  j                  |t                     D ]T  }t        | g|j                        j                  | j                  |      }t        |      s|c S |j                  |       V |S r   )
r   r  r   r   popr   rm   rc   rk   rp   )r:   r   rd   r   anchor_nodeanchor_ms         r=   r   zRepeatedExpr._matchy  s    IId(($/{H	
  --??SUK 	K#TF$**=CC""KH H%HHX	 r@   c                    t        j                  t        |      }t        |   |      xr% | j
                  j	                  |j
                        S r7   )r	  r
  r   rg   r   r  r  s     r=   r   zRepeatedExpr.pattern_eq  sF    D%(w!%( 
T-?-?-J-J.
 	
r@   )r  r  rB   rZ   )rB   zSequence[FnsType]r   r   )
r5   rC   rD   r   rX   r   r  r   r   r   r   s   @r=   r  r  k  s0    #
 & &"
 
r@   r  c                  f    e Zd ZdZddZe ej                  d      dd	d              Zd
dZ	ddZ
y)r   z
    Serializes Patterns to executable python.
    XXX: currently only used and tested for fuse attention patterns. May not cover
    all patterns.
    c                z    t         j                  j                  j                         | _        i | _        i | _        y r7   )r   r   rk   
_Namespace	namespacememoized_objs_namesmemoized_objs_pprW   s    r=   rX   zPatternPrettyPrinter.__init__  s*    224;= 8:r@   Nc                *   t               }t        | d      sJ | j                  |      }g }|j                  D ]2  }|j	                  |j                  |    d|j
                  |           4 |j	                  | d|        dj                  |      S )zU
        Serializes obj to python code with obj written out to `output_name`
        r   )r   z = 
)r   hasattrr   r  r  r  r^  )objoutput_namer   out_strr  rs   s         r=   runzPatternPrettyPrinter.run  s     "#sN+++""b"))) 	YCMMR33C89R=P=PQT=U<VWX	Y 	S	23yy  r@   c                    t        |t              r0| j                  j                  |      x}r|S | j	                  |      S t        |d      r|j                  |       S t        |      S )Nr   )r   r;  r  rP  memoizer  r   r]  )r:   r  memoized_names      r=   r   z!PatternPrettyPrinter.pretty_print  sb    c?+ $ 8 8 < <S AA}A$$||C((3'##D))Cyr@   c                    |j                  |       }|j                         }dD ]  }|j                  |d      } | j                  j	                  |d       }|| j
                  |<   || j                  |<   |S )N)zaten.r&  zprims.r+  )r   r)  replacer  create_namer  r  )r:   r  obj_strobj_nameprefixtmp_names         r=   r  zPatternPrettyPrinter.memoize  s{    ""4(<<>3 	4F''3H	4 >>--h=(0  %%,c"r@   rY   )r  )r  r]   r  r4   rB   r4   )r  r   rB   r4   )r  r;  rB   r4   )r5   rC   rD   r   rX   r  r   	lru_cacher  r   r  r8   r@   r=   r   r     s?    ;
 Y!  !"		r@   r   c                      e Zd ZddZy)_PassDictsTypec                     y r7   r8   )r:   r_  s     r=   __getitem__z_PassDictsType.__getitem__  r?   r@   N)r_   Tuple[str, torch.fx.node.Target]rB   List[PatternEntry])r5   rC   rD   r  r8   r@   r=   r  r    rJ   r@   r  c                  F    e Zd ZU ded<   ded<   ddZ	 	 d		 	 	 	 	 	 	 d
dZy)PatternEntryr]   r^   Callable[[Match], bool]extra_checkc                    t         r7   r0  r:   r   rk   r   s       r=   applyzPatternEntry.apply  s    !!r@   Nc                   |Ht        | j                  d      sJ | j                  j                  D ]  }| j                  |||        y t	        |t
        t        f      rqt        | j                  d      sJ |r,|| j                  j                  |f   j                  d|        y || j                  j                  |f   j                  |        y t        j                  t        t           |      }|D ]  }| j                  |||        y )Nr  prependr#  r   )r  r^   r  registerr   r   PatternMatcherPassr#  insertr  r	  r
  r   r  )r:   
pass_dictsrg  r  rN   r@  s         r=   r  zPatternEntry.register  s     >4<<///ll&& ?j"g>?
T+=$>?4<<...DLLOOV45<<QEDLLOOV45<<TBXn%=zJJ :a9:r@   r   r\   rk   r   r   r   rB   rZ   r   )r  /Union[_PassDictsType, Sequence[_PassDictsType]]rg  z!Union[torch.fx.node.Target, None]r  r   rB   rZ   )r5   rC   rD   rE   r  r  r8   r@   r=   r  r    sF    ((" 59	:C: 2: 	:
 
:r@   r  c                       e Zd ZU ded<   ddZy)LoweringPatternEntryCallable[..., Any]handlerc                    t        j                  | j                        t        j                  | j                  |            }|j	                  |      5  |j                  |t        |j                        |j                        }|j                  j                  |j                         |j                  |       d d d        |j                  d   |u sJ |j                          y # 1 sw Y   -xY w)N)r   wrapsr  r   inserting_beforer  ru   r;   r<   r   rq   replace_all_uses_withra   r   )r:   r   rk   r   r  r   s         r=   r  zLoweringPatternEntry.apply  s    /)//$,,/	0A0A$,,PU0VW##D) 	4--guUZZ7H%,,WK##DII.&&{3	4 {{2$&&&	4 	4s   A'C((C1Nr  )r5   rC   rD   rE   r  r8   r@   r=   r  r    s    r@   r  c                  $    e Zd ZU dZded<   ddZy)GraphPatternEntryz8
    A pattern that runs a function on the FX graph
    r  r  c                    |j                  |      5   | j                  |g|j                  i |j                   d d d        y # 1 sw Y   y xY wr7   )r  r  r;   r<   r  s       r=   r  zGraphPatternEntry.apply   sE    ##D) 	=DLL<<u||<	= 	= 	=s   +AANr  )r5   rC   rD   r   rE   r  r8   r@   r=   r  r    s      =r@   r  c                  F    e Zd ZU ded<   e	 	 	 	 	 	 	 	 	 	 dd       ZddZy)r   zCallable[..., List[Any]]normalize_argsc                    G fddt         j                  j                        }| j                         }t	        |      dk(  r|d   }n|d   sJ t        |d   j                  j                        }|D cg c]9  }t        |t         j                  j                        r|j                  |      |f; }	}t        |	t        j                  d            d   }	 	 	 	 	 	 	 	 	 	 d	dj                  |      5    ||      j                   }
t        |
t         j                  j                        r|
g}
d
d	 	 	 	 	 	 dfdt	        |      t	        |
      k(  rt!        ||
      D ]  \  }} ||        nt	        |      dk(  sJ  |d   |
       d d d        | j#                          y c c}w # 1 sw Y   xY w)Nc                  0     e Zd ZdZdZdZd fdZ xZS )<ReplacementPatternEntry.replace_with_graph.<locals>.ReplacerNc                   |j                   dv rt        | 	  |      S |j                   dk(  r|j                  }| j	                  |      \  }}j                  |||      }d|j                  v r}d|j                  vro|j                  d   |j                  d<   t        |j                  d   t        j                        r,d|j                  v sJ |j                  d   |j                  d<   |S t        d|       )N)placeholderr  r  r   tensor_metaz
unhandled )r#  rg   run_noderg  fetch_args_kwargs_from_envr  r   r   r   Tensorr1  )r:   r   rg  r;   r<   resultri   rk   s         r=   r  zEReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_node  s    7777 7+D1177o-![[F#'#B#B4#HLD&"00vFF		)e6;;.F-1YYu-=E*%dii&6E#0DII#==#=9==9QFKK6!M)Jtf*=>>r@   r   r   rB   r   )r5   rC   rD   r  r  get_attrr  r   )ri   rk   s   @r=   Replacerr    s    KKH? ?r@   r  r/   r   )rs   c                    | g}t               }|rc|j                         }||vrK||vrGt        |d      r;|j                  |       ||j                  |<   |j                  |j                         |rby y )Nr   )rm   r  r  ru  r   rp   all_input_nodes)r   tag_name	tag_valueinput_stopsqueuevisitedr   s          r=   percolate_tagszBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tags2  sk     FEeGiikw&;.V,KK$)2CHHX&LL!4!45 r@   c                    | j                   dk7  ry | j                  t        j                  k7  ry t	        | j
                        dk(  sJ | j
                  d   S )Nr  r&   r/   )r#  rg  operatorgetitemr   r;   r   s    r=   maybe_getitemzAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitemK  sJ    77o-;;("2"22499~***yy|#r@   c           	        | |J y t        | t        j                  j                        sJ |#| j	                  d        j                  |        y t        |t        j                  j                        rd|j                  vr%|j                  j                  | j                         dD ]2  }|| j                  v s 	||| j                  |   t                     4 | j	                  |       j                  |        y t        | j                  j                               }|D ]#  } |      }|t        d       
|||          % j                  |        y )Nr   )	recomputeac_graph_idzcan't handle)r   r   r   r3  r  r   r   rq   rm   rh   r   rn   AssertionError)oldnewr
  old_usesuseridxr;   rk   r  r  r  s         r=   r  z;ReplacementPatternEntry.replace_with_graph.<locals>.replaceS  s@    ;;&;!#uxx}}555;--d3$$S)c588==1CHH,1 %A Y#sxx/*3#((8:LcRViXY --c2$$S):  		 01$ ,D'-C{,^<<D#c(+	,
   %r@   )
r   r   r
  r4   r  r4   r  r   rB   rZ   r  )r  zUnion[torch.fx.Node, None]r  z3Union[torch.fx.Node, Sequence[torch.fx.Node], None]rB   rZ   )r   r   Interpreterr   r   rh   rk   ra   r   r3  indexminr  
itemgetterr  r  rh  r   )r   rk   re   r;   r  r   	last_nodera   r   indicesr   r  r  r  r  r  s    ` `         @@@r=   r   z*ReplacementPatternEntry.replace_with_graph	  s   	?uxx++ 	?( ))+|!$QI?"?a..445E &a/ Q#G 
 G)<)<Q)?@CI	6	6	6 	6 ,		6
 	6( ##I. U	69(#45994@K+uxx}}5*m$A&/A&HA& A& A&F < C$44 #L+ > &HCC%& <(A---Q5kU	6n 	e6U	6 U	6s   <>F9>B"F>>Gc           
         |j                   J | j                  |||j                    | j                  |j                  i |j                         y r7   )re   r   r  r;   r<   r  s       r=   r  zReplacementPatternEntry.apply  sN    &&222##D<u||<		
r@   N)
r   r\   rk   r   re   z+Union[torch.fx.Graph, torch.fx.GraphModule]r;   zSequence[torch.fx.Node]rB   rZ   r  )r5   rC   rD   rE   r  r   r  r8   r@   r=   r   r     sV    ,,SSS GS &	S
 
S Sj
r@   r   c                     yr   r8   )r   s    r=   _return_truer&    s    r@   c                F    t         j                  d| j                  |       y )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginfor5   )	search_fnr   s     r=   log_trace_failurer+    s    HHJ	r@   r8   c	           
         g t        j                         j                  j                         d f	d}	dfd}
t        u rt        j                         ryt        j                  d      5  |D cg c]*  }t        |t
        j                        xr |j                  , c}t         |      }n}t        j                  |      }|t        vsJ t        j!                  |       t#        ||	|
      }|j%                  |       |j&                  cddd       S c c}w # 1 sw Y   yxY w)	a  
    Create a replacement rule based on example functions that get traced
    to create patterns.  This supports both training and inference when
    run on a joint forward+backward graph.

    Args:
        search_fn: traced to give original pattern
        replace_fn: traced to give replacement graph
        example_inputs: example inputs for initial trace
        trace_fn: fwd_only or joint_fwd_bwd
        pass_dict: dict of passes to register to
        extra_check: additional check to run on match(using real shapes)
    c           
       	 t              }|D ]+  }|| j                  vst        d| d| j                          t        t        j                  j                  |D cg c]  }| j                  |    c}d             g }t        j                  j                  j                        5  t              D ]  \  }}t        |   t        j                        s%|r"t        |   j                        r
 ddd       yt        j                  |   j                         |   j!                         |   j                  |   j"                  |      |<   t%        j&                  |   j(                  |   j!                               D ]C  t        t        j*                        st-        fd|D              s3|j/                         E  }|s3|rdfd}	  ||z         }g }
t3        t5        t7        |      t7              z         |j8                  j:                        D ]  \  }}|t7        |      k  r|
j/                  |j<                         0|j8                  j?                  |      5  |j8                  jA                  ||t7        |      z
           }|jB                  |_        |jE                  |       |j8                  jG                  |       ddd        |
|z   }n
	        }tI        ||	      }| jK                         d
   }|J |jM                  |      }tO        |      r  |      r       | _(        	 ddd       y	 ddd       yc c}w # t        $ r}	t1        |	       Y d}	~	ddd       yd}	~	ww xY w# 1 sw Y   rxY w# t        $ r}	t1        |	       Y d}	~	ddd       yd}	~	ww xY w# 1 sw Y   yxY w)z
        Often shapes get burned into the pattern, so our initial match ran with
        `ignore_types=(int, ...)`.

        Recheck the match with the correct shapes.
        z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                     | j                   d   S r   r   )r   s    r=   r   z8register_replacement.<locals>.check_fn.<locals>.<lambda>  s    QVVE] r@   NF)dtypedevicerequires_gradc              3  :   K   | ]  }t        |k7          y wr7   r!   )r   r{  r`  s     r=   r   z9register_replacement.<locals>.check_fn.<locals>.<genexpr>  s       ?=>0a8?s   c                 >     | t        |       t              z
  d   S r7   )r   )args_newr;   r*  s    r=   search_fn_newz=register_replacement.<locals>.check_fn.<locals>.search_fn_new  s#    ((3x=3t93L3N*OPPr@   )argnamesexclusive_arg_namesscalar_workaroundr   T)r4  r   rB   r   ))rh   r<   r   r   r   r   _dynamoutilsdetect_fake_moder  r   r  r   r/  empty_stridedsizestrider0  rB  rC  shapeSymIntr  r  r+  rh  ranger   rk   ra   rg  inserting_afterr  r   r  r   fx_to_patternr   r   r   re   )r   r6  r   sym_argsrm  gradspecific_patternr5  specific_graphr   sym_arg_namesr  new_noder   specific_pattern_matchr;   r`  argnames_staticr7  r  
replace_fnr1  r8  r*  search_fn_patternr   s                  @@r=   check_fnz&register_replacement.<locals>.check_fn  s    ( 	D5<<'"99A
/RWR^R^Q_a 	 HH089d#9;R

 (*]]  11$7 S	$]3 /4d1gu||4 0a ?$	S	 S	 $11QQ("1gmm#Aw~~&*DG '__T!W]]DGNN<LM /%a63 ?BJ? < %OOA.	//*  1#Q%)1-D)Q %'M*-c(mc$i78&,,22+ I; s8},)001C1CD$+11AA+N I'5';';'G'G (S]): ;(H /7mmHO'==hG*00;;KHI II   -x7H%)1)T)B
 $1"%(;&7	$  %%'*D###%5%;%;D%A"./K@V4W*2:t*D'eS	 S	f gS	 S		 :T ( %))Q7$QS	 S	L%I I ( %))Q7$ES	 S	@%AS	 S	s   !M//0O OB*O/O%O*M46BOA(N)O:	N+AO%O4	N=N	ONON(#O+	O4O OOOOc                    g }D ]"  }|j                  | j                  |             $ t        dt        |       dz         D ].  }d| | vr n%|j                  | j                  d|              0 | r
J d|        |S )Nr/   	tangents_zleftover kwargs: )r  r  rA  r   )r<   r;   r   rm  rK  s       r=   r  z,register_replacement.<locals>.normalize_args9  s    # 	*DKK

4()	*q#f+/* 	5A1#f,KK

Yqc?34	5 9.vj99zr@   Ffunctionalize_rng_opsN)r^   r  r  r   r\   rB   r   )r<   r   rB   r_   )inspect	signature
parametersrn   joint_fwd_bwdr   is_inference_mode_enabledfunctorch_configpatchr   r  r1  gen_patternr   r  _seen_patternsru  r   r  r^   )r*  rL  example_inputsr   r  r  r8  r7  rM  rN  r  r@  r^   pattern_reprrK  r1  s   `` ` ````     @@r=   register_replacementr_    s1   0 H)))4??DDFGOh hT	 =  **, 
		e	< ES%
@AJq%,,';AOO;%
 $!!#G (G+//8>111<() )

 	$1 %
 s    D8/D34A5D83D88EzSet[str]_serialized_patternsc                V   dd}t         j                         st        dt                |j                  }ddlm}  |j                  d      5  t        ||||      }d d d        t        j                  |       }	|t        vrd}
t        j                  |       nd	}
 |       }t        t         | d
z  |
      5 }|
dk(  r|j                  |       n|j                  d       |j                  |	       |j                  d       d d d        |S # 1 sw Y   xY w# 1 sw Y   |S xY w)Nc                    t        j                  d      } t        j                  d      j                  |       }g }t        t        j
                  j                        D ]_  }t        t        j
                  j                  |      }t        |t              s8t        |t        t        f      sO|j                  |       a dj                  |      }d| d}| | S )Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            z            # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr   dirr   	_inductorpattern_matcherr  r   rM  
issubclassr]   r  r  r^  )auto_generated_msgfile_templatepattern_matcher_importsr   attrformatted_importss         r=   get_file_templatez-_serialize_pattern.<locals>.get_file_templatep  s    %__
 !
 &'&
( 	 #%778 	5D5??::DAD$%*TK;U*V'..t4	5
 %MM*ABPQbPccij!2 344r@   z0Could not find serialized patterns directory at r   r'   FrQ  )r  wr{  z.pyz

r  r   )SERIALIZED_PATTERN_PATHis_dirr   r5   torch._functorchr(   rZ  r[  r   r  r`  ru  openwrite)unique_namer*  r]  r   r8  ro  pattern_namerY  r^   serialized_pattern
write_moderk  fs                r=   _serialize_patternr{  i  s)    5D #))+>?V>WX
 	
 %%L;				e	< ViCTUV .11'{1S//
  .
%'M	%<.(<<j	I QGGM"GGFO	"#	 N)V V Ns   D=ADDD(	fx_passesserialized_patternszvList[Tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsFc
                   t        |      }dt        j                  v rt        | ||||      }
nT|j                  }t        j                  d|       }|rt        ||       st        j                  d|        t        ||       }
t        j                  |      D ]'  }t        |t              s|j                  !d |_        ) t         j#                  |
      t$        v r|	ry t&        j)                  |||||
f       t+        |||||||||
	       y )NPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)rM  )ru   osenvironr{  r5   	importlibimport_moduler  r(  warningr  rN  	tree_iterr   r,   constantr   r  r\  r~  r  r_  )rv  r*  rL  r]  r   r  r  r8  r7  skip_duplicatespatrw  r   r   s                 r=   gen_register_replacementr    s    >*N+ NH>O
 !))##<\NK
 ;/KKV a%/  c:&3<<+C
  CL  $6?&&	NH.?E 
r@   rQ  c                n   g t        j                  |       j                  j                         }|i }g }d}|D ]4  }||v r|j	                  ||          |j	                  ||          |dz  }6  || |      }	t        |	t        t        t        t        j                  t        j                  f|||      S )Nr   r/   )ignore_typesr6  r8  r7  )rT  rU  rV  rn   r  rC  intfloatrh   r   r0  r/  )
r*  r]  r   r8  r7  r6  flat_inputs	input_idxargname	search_gms
             r=   r[  r[    s     A""9-88==?@H KI ''09:~i89NI K0I5$ekkB+/ r@   r  c                     d fd}|S )z
    Register an aten to inductor IR replacement pattern.  The decorated
    function is saved and then called a lowering time allowing direct
    pattern to inductor IR conversion.
    c                n    t        |       sJ t        |       j                         d| _        | S )Nr^   r  r  r  T)r  r  r  _inductor_lowering_functionr  r  	pass_dictr^   r  s    r=   	decoratorz,register_lowering_pattern.<locals>.decorator(  s;       g	

(9g(
..2+r@   r  r  rB   r  r8   r^   r  r  r  r  s   ```` r=   register_lowering_patternr    s      r@   c                     d fd}|S )zk
    Register a pattern that runs a function on the FX graph, allowing
    custom transformation code.
    c                `    t        |       sJ t        |       j                         | S )Nr  r  )r  r  r  r  s    r=   r  z)register_graph_pattern.<locals>.decorator?  s3       g	

(9g(
.r@   r  r8   r  s   ```` r=   register_graph_patternr  3  s      r@   c                B    |t        t        | j                              u S r7   )r   iterra   )rk   r   s     r=   is_start_of_fx_graphr  I  s    4U[[)***r@   z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)c                   | j                   dk(  r*t        j                  | j                  j                        r0y| j                   dk(  r t        j                  | j                        ry| j
                  j                  d      d uS )Nr  Tr  out)r#  _mutation_op_researchrg  r5   r<   rP  r  s    r=   is_mutation_opr  S  sb    ww/!!!$++"6"67	M	!!!$++.;;??5!--r@   c                    d| j                   v sJ d|j                   v sJ | j                   d   |j                   d   k(  S Nmutation_region_idr   )r{  r|  s     r=   same_mutation_regionsr  ]  sE    166)))166)))66&'1662F+GGGr@   c                ,   |}d|j                   vr3t        | |      s'|j                  }d|j                   vrt        | |      s'|j                   j                  dd      }||ur0|j                  }t        |      r|dz  }||j                   d<   ||ur0|S )Nr  r   r/   )r   r  prevrP  r   r  )rk   r   r   r  s       r=   get_mutation_region_idr  c  s    A
aff
,5I%QR5SFF aff
,5I%QR5S$8!<
4-FF!!#'9#$	 4-
 r@   c                V    dt        t        | j                              j                  vS r  )r   r  ra   r   r   s    r=   "should_compute_mutation_region_idsr  p  s!    tD,='>'C'CCCr@   c                h    d}| j                   D ]!  }t        |      r|dz  }||j                  d<   # y )Nr   r/   r  )ra   r  r   )rk   r  nds      r=   compute_mutation_region_idsr  t  s=    kk ;"!#(:$%;r@   c                  B     e Zd Z	 d	 	 	 d fdZddZddZd	dZ xZS )
r  c                X    t         |           t        t              | _        || _        y r7   )rg   rX   r   rh   patterns	pass_name)r:   r  ri   s     r=   rX   zPatternMatcherPass.__init__}  s*     	  	 #r@   c                     | j                   |   S r7   )r  )r:   items     r=   r  zPatternMatcherPass.__getitem__  s    }}T""r@   c                `   | j                   syt        |t        j                  j                        r|j
                  }nJt        |t        j                  j                        r|}|j                  }nt        dt        |             t        |      rt        |       t        j                  t        |      }d}g }d}| j                   D ]0  \  }}|dk(  rd}|j                  |j!                  ||d             2 |r"|j                  |j!                  dd             | j"                  | j"                  nd}	t%        ||	t&        j(                        5  t+        t,        j.                  j1                  |      d	      D ]  }
t3        |
      }|
j4                  dk(  r|
j4                  |f| j                   vr9t7        |
d
      rG| j                   |
j4                  |f   D ]-  }|
j8                  r t|j:                  j=                  |
      }t?        |      r,tA        tC        tE        ||jF                                    dk7  rdtH        jJ                  jM                  d      |
jN                  k(  r-tP        jS                  d|
|
jT                  ||j:                         t?        |      s|jW                  |      s|dz  }|jY                  |||
       tZ        d   dxx   dz  cc<   tZ        d   dxx   tA        |jF                        z  cc<   0  	 d d d        |S # 1 sw Y   |S xY w)Nr   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Fr  T)r#  rg  sort)r#  r  rh  )reverse)allow_cpu_inputsr/   !TORCHINDUCTOR_PATTERN_MATCH_DEBUGz
%s%s %s %sinductorpattern_matcher_countpattern_matcher_nodes).r  r   r   r   GraphModulerk   Graphowning_moduler   rM  r  r  r   r   r  r  
find_nodesr  r%   trace_configlog_url_for_graph_xformsortedrB  rC  from_iterabler4  r#  r1   r~   r^   r   r   r   rm   rQ  ra   r  r  rP  r   r(  r  r;   r  r  r   )r:   gmrk   get_mutation_region_id_partialri  ra   has_call_moduler#  rg  r  r   entryr   s                r=   r  zPatternMatcherPass.apply  s   }}b%((../HHEEHHNN+E$$B\]abd]e\fg  .e4'.)2):):"E*
& -- 	QJB]""&U--F-OP		Q
 LL))])GH&*nn&@DNNFW	#	<??
 	V y<<UCTR V'-77m+(= 
 9PUV!]]DGGV+<= VE||++D1A !C(F$P QRVWW zz~~&IJdiiWL$		1emmT{u'8'8';
Aud3 ,-DEJE ,-DEQWWUE!VV	V@ A	V@ s   EL#-L#?AL##L-c                8    | j                   j                          y r7   )r  clearrW   s    r=   r  zPatternMatcherPass.clear  s    r@   r7   )r  Optional[str]rB   rZ   )r  r  rB   r  )r  rP   rB   r  rY   )r5   rC   rD   rX   r  r  r  r   r   s   @r=   r  r  |  s/     $(# # 
##<|r@   r  c                     t         r7   r0  rz   s     r=   _not_implementedr    s    
r@   c                  	
 |xs i }|j                         D ci c]  \  }}||
 c}}
t        
      t        |      k(  sJ d
fdt        j                         	 G 	fddt        j
                  j                        } ||       j                         }t        |t              st        t        j                  |            S |S c c}}w )z
    Convert an FX graph into a PatternExpr.  This is useful for simple
    patterns that can only match single functions and fixed-length lists.
    c                    t        | t        t        f      r| v rt        |          S t	        |       v r
t               S t        | t              rt        d | D              r| r
t               S | S )Nc              3  <   K   | ]  }t        |t                y wr7   )r   r   )r   ys     r=   r   z5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>  s     &I!z!W'=&Is   )r   r  r  r   rM  r   rh   r  )r@  r  inv_scalar_workarounds    r=   process_argz"fx_to_pattern.<locals>.process_arg  sd    a%&10E+E3A6777l"9a3&Iq&I#Ia9r@   c                  l     e Zd ZeZeZeZ	 	 	 	 	 	 	 	 dfdZ	 	 	 	 	 	 	 	 dfdZd fdZ	 xZ
S ) fx_to_pattern.<locals>.Converterc                    t              }|t              k  r|   }n1r|j                  d      sJ |}nt        j                  dd|      }|}|v rt        |      S t        |      S )Ntangentz_\d+$r+  )r   r   
startswithresubr  r   )	r:   rg  r;   r<   r   r   r6  argnumr7  s	         r=   r  z,fx_to_pattern.<locals>.Converter.placeholder  su     VA3x= {((333"f5***400!$''r@   c                    t        j                  ||f      \  }}t        v r>|D cg c]
  } |       }}|j                         D ci c]  \  }}| |       }}}t	        |g|i |S c c}w c c}}w r7   )rN  tree_maprh   r   r  )r:   rg  r;   r<   r{  r_  r  r  s         r=   r  z.fx_to_pattern.<locals>.Converter.call_function  s     "??;vGLD&|#0451A558>G1![^+GG8888 6Gs   A2A7c                ^   t         |   |      }|j                  dk(  rst        |t              rct        |      t        |j                  d         k(  sJ t        ||j                  d         D ]  \  }}t        |j                        |_        ! |S t        |j                        |_        |S )Nr  r   )	rg   r  r#  r   ru   r   r;   rh  r   )r:   r   rvrr   ri   s        r=   r  z)fx_to_pattern.<locals>.Converter.run_node  s    !!$BttxJr5$92w#affQi.000!"affQi0 -FAs!#))nAG- I qww<Ir@   )rg  r4   r;   r   r<   Mapping[str, Any]rB   z&Union[ExclusiveKeywordArg, KeywordArg])rg  r4   r;   r   r<   r  rB   r]   )r   r   rB   r   )r5   rC   rD   r  r  r  r  r  r  r  r   )ri   r6  r  r7  r  r  s   @r=   	Converterr    sf    &&#	(	(%2	(<M	(3	("	9	9%2	9<M	9	9	 	r@   r  )r@  rQ   rB   zUnion[T, KeywordArg, Ignored])r   r   rB  ri  r   r   r  r  r   r]   r  rN  tree_leaves)r  r  r6  r8  r7  r_  r`  r  r^   r  r  r  s    `` `    @@@r=   rC  rC    s     */R.?.E.E.GHdaQTH$%->)???? __F( (EHH(( (T m!Gg{+!&"4"4W"=>>Nw Is   CT)r   get_decomp_fnc                  t               5  | |       n	t               } t        | |d      | }ddd       ddlm} |r, |j
                         |j
                  j                          j                          |S # 1 sw Y   OxY w)z>Build a normalized inference graph, for use with fx_to_patternNreal)tracing_moder/   remove_noop_ops)r   r0   r    fx_passes.post_gradr  rk   eliminate_dead_code	recompile)rN   r;   r   r  decompositionsr  r  s          r=   r   r     s     
"	# E,8MO>Q>S 	 >WRf=tD	E 5!
$$&LLNIE Es   %A>>Bc           
        d	 	 	 	 	 	 	 	 dfd}t         j                  j                  d      5   t        | d |t	               dd      |  ddd       sJ ddlm}  |j                         dd	lm	} t               }t        t         j                  j                  j                  j                  t!        d
      t!        d            }t#        ||t$              j'                  |j(                         |j+                  j                         t         j,                  j                  j/                         j                  _        j                  j3                          j5                          S # 1 sw Y   *xY w)z=Build a normalized training graph, for use with fx_to_patternNc                <    rJ t        |       t        | |fi |S r7   )clone_graphr+   )joint_graphinputsr<   r  s      r=   record_joint_graphz)joint_fwd_bwd.<locals>.record_joint_graph;  s(     v% f???r@   c                    t        |       S r7   )r*   )grm  s     r=   r   zjoint_fwd_bwd.<locals>.<lambda>F  s    + r@   TF)partition_fnr  keep_inference_input_mutations
enable_logr/   r  )pointless_viewr   r=  )r^   r  r  )r  rP   r  r   r<   r   rB   z1Tuple[torch.fx.GraphModule, torch.fx.GraphModule])r   _guardstracingr)   r0   r  r  rk   fx_passes.joint_graphr  r  r  opsatenviewdefaultr   r  r&  r  r  r  r   CodeGen_codegenr  r  )rN   r;   r  r  r  matcher_passr^   r  s          @r=   rW  rW  6  sJ    *.B@)@3@@LO@	:@ 
		t	$ 	
++.0+/	
 	 I24BHH5%'L		##Z%6
68JG \h|$$%rxx  ..0BHHHH  "LLNI? s   E77Fc                    g }t         j                  j                  | j                  | j                  f|j
                         |S r7   )r   r   r   r;   r<   r  )r   r;   s     r=   rj  rj  e  s1    )+D	HHaffahh'5Kr@   c                   t        t        | j                              }t               }t	        t               }d }|r|j                         }t        |      D cg c]	  }||vs| }}|r||d      j                  |       n^|j                  |       |r|j                  |ur|j                  |       |}|j                  t        |j                  |d                   |r|s!t        |      t        | j                        k(  sJ y c c}w )Nr  r8   )rh   r}   ra   rm   r   r  rj  r  ru  r   rp   r   )rk   pendingreadywaitingcursorr   r@  waiting_fors           r=   stable_topological_sortr  k  s     8EKK()G EE $G F
{{}"'+@Q%q@@ KO$++D1IIdO&++T1d#F NN8GKKb$9:;   3u:U[[)99999 As   	D
#D
c                t     t        j                  d      t        j                         d fd              }|S )z0Wrapper around lazy init functions in fx_passes/Nc                 :   t         d   j                         } t        j                  j	                  d       5  t               5  t               5          }d d d        d d d        d d d        | t         d<   S # 1 sw Y   $xY w# 1 sw Y   (xY w# 1 sw Y   ,xY w)Nr  )r   copyr   r  r   r   r-   )counters_refr  rN   s     r=   	lazy_initz%init_once_fakemode.<locals>.lazy_init  s      
+002]]""4( 	*@*B 	NDT 	TF	 	 	  ,	 	 	 	 	 	s;   BBA9BB9B>BB	
BB)rB   r   )r   r  r  )rN   r  s   ` r=   init_once_fakemoder    s;     __R	  	 r@   c                     d fd}|S )z2Function for extra_check to put pass behind a flagc                $    t        t              S r7   )r  r(   )r   r   s    r=   
flag_checkzconfig_flag.<locals>.flag_check  s    vt$$r@   )r   r\   rB   r   r8   )r   r  s   ` r=   config_flagr    s    % r@   c                L     G d dt               } ||       j                         S )Nc                        e Zd Zd fdZ xZS )clone_graph.<locals>.CopyGraphc                J   t         |   |      }t        |t        j                  j
                        rn|j                  j                  j                  |j                         | j                  j                  j                  |j                  d       |j                  _        |S r7   )rg   r  r   r   r   Proxyr   r   rq   	new_graph_graph_namespacer  r   )r:   old_noderI  ri   s      r=   r  z'clone_graph.<locals>.CopyGraph.run_node  sq    w'1H(EHHNN3""))(--8%)^^%D%D%P%PMM4&" Or@   )r"  r   rB   r   )r5   rC   rD   r  r   r   s   @r=   	CopyGraphr    s    	 	r@   r#  )r.   	transform)input_graphr#  s     r=   r  r    s$    K  [!++--r@   r\  c                    t        | j                        |kD  r| j                  |   S | j                  j                  |      S r7   )r   r;   r<   rP  )r   
arg_number
kwarg_names      r=   get_arg_valuer)    s>    
 tyy>J& 			* [[__Z(r@   c           	        |g}t        |t        j                  j                        r7|j	                  |j                         D cg c]  }t        ||       c}       | D cg c]  }|j                  |v s| c}S c c}w c c}w r7   )r   r   r  r   rp   r!  r  rg  )ra   rN   r  r  r   s        r=   filter_nodesr+    sf    $C"ejj112

",,.IhGB)IJ"9TdkkS&8D99 J9s   A>#B7Bc                    | j                   dk(  r4t        | j                  j                  | j                        j
                  S | j                  S )zFor call_function and call_method, we directly use the target function;
    For call_module, the target is string, and we treat the module class
     as a function.
    r  )r#  r  rk   r  rg  ri   r  s    r=   r4  r4    s;    
 ww-tzz//=GGG;;r@   )r   r   rB   zTypeGuard[Match]rS  )r*  r  r   r   rB   rZ   )r*  r3   rL  rG   r]  Iterable[Any]r   rL   r  r  r  r  r8  )Union[Dict[str, Union[float, int]], None]r7  Sequence[str]rM  zUnion[PatternExpr, None]rB   r   )rv  r4   r*  r3   r]  r-  r   rL   r8  r.  rB   r]   )rv  r4   r*  r3   rL  rG   r]  r-  r   rL   r  r  r  r  r8  r.  r7  r/  r  r   rB   rZ   )Nr8   )r*  r3   r]  r   r   rL   r8  r.  r7  r/  rB   r]   )
r^   r]   r  r  r  r  r  r   rB   z2Callable[[Callable[..., Any]], Callable[..., Any]])rk   r   r   r   rB   r   r9  )r{  r   r|  r   rB   r   )rk   r   r   r   rB   r  )rk   rP   rB   r   )rk   rP   rB   rZ   )r;   r   r<   r   rB   r   )r8   r8   Nr8   )r  z+Union[torch.fx.GraphModule, torch.fx.Graph]r  zSequence[Type[Any]]r6  r/  r8  r.  r7  r/  rB   r]   )
rN   r  r;   r   r   r   r  zOptional[Callable[..., Any]]rB   rP   )rN   r  r;   r   rB   rP   )r   r   rB   zList[torch.fx.node.Argument])rk   r   rB   rZ   )rN   r  rB   zCallable[[], Any])r   r4   rB   zCallable[[Match], Any])r%  rP   rB   rP   r7   )r   r   r'  r  r(  r  rB   r   )ra   zIterable[torch.fx.Node]rN   r   rB   r`   )r   r   rB   ztorch.fx.node.Target)r   
__future__r   r   dataclassesr   r  rT  rB  loggingr  r  r  rd  r	  abcr   r   collectionsr   pathlibr   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   typing_extensionsr   r   r   torch._guardstorch.fxtorch.utils._pytreer:  _pytreerN  torch._dispatch.pythonr   torch._dynamo.utilsr   torch._inductor.configr   r  torch._prims_commonr   torch._subclasses.fake_tensorr   "torch.fx.experimental.proxy_tensorr    %torch.fx.experimental.symbolic_shapesr"   torch.fx.immutable_collectionsr#   r$   (torch.fx.passes.graph_transform_observerr%   
_functorchr(   rY  _functorch.aot_autogradr)   r*   _functorch.partitionersr+   _subclassesr,   r-   r   r.   r+  decompositionr0   loweringr1   	getLoggerr5   r(  r  r  primsConstantr3  r   r3   rG   rL   rQ   r   Targetr4   FnsTyperS   rU   r\   r   ro   r   r   rc   r]   r   r   r   r  r  _SimpleSpecr;  r  r  r  r  r  r  r  r  r  r  r   r  	dataclassr  r  r  r   r&  r+  r_  rm   r`  rE   r{  __file__parentrq  r~  r  rZ  r[  r  r  r  compiler  r  r  r  r  r  r  r  rC  no_gradr   enable_gradrW  rj  r  r  r  r  r\  r)  r+  r4  r8   r@   r=   <module>rV     s  !F #         	 	   # #     & .    $ $ ; ( 8 0 @ 6 F I K 3 C 7 4   . ; g!yy~~		x./x  
h  CL $$c)
*? ? :w wt, 8 E;&''
 '
T!1# !1H-+ -k E E(E+ E4N
+ N
b CHo`
k `
F?    (, * * ,
[ ,
^F
 F
R#
; #
L3 3lX 
 : : :: 
< 
 
 	= 	= 	= `
l `
 `
F ,8CG)+26nnn "n 	n
 @n )n An 'n 0n 
nb "% h &FFF "F 	F
 AF FR x.//+=@UU     $ ,8CG)+!666 6 "	6
 6 @6 )6 A6 '6 6 
6r e4
 DH)+!  A	
 '  5B ,8
 ( 	
  84 ,8
 ( 	
  8,+ "**VW.H
D;M M` )+ CG)+I3I%I I A	I
 'I IX 
 #'26
  	
 0  2 + +\!:H&. 5   GK
%(6C:r@   