
    ɯwg!3                    Jr   U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
mZmZmZmZmZmZmZ d dlmZ d dlZd dlZd dlZd dlZd dlmc mZ d dlmZ d dlmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ d dl,m-Z-m.Z. d d	l/m0Z0m1Z1m2Z2m3Z3m4Z4 d
dl5m6Z6 ddl7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z> ddl:m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJ ddlmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZS ddlTmUZUmVZV  ej                  eX      ZYi ZZeej                  j                  ede
f   f   e]d<   i Z^eej                  j                  eede
f      f   e]d<    e_       Z`eej                  j                     e]d<   ej                  j                  Zaej                  j                  Zbej                  j                  Zc e_       Zdeej                  j                     e]d<    e_       Zeeej                  j                     e]d<    e_       Zfeej                  j                     e]d<   i Zgeej                  j                  ej                  j                  f   e]d<   ej                  j                  Zhdede
f   deede
f      fdZid Zjd Zkd Zld  Zm eleaj                  eaj                  eaj                  eaj                  eaj                  eaj                  eaj                  eaj                  eaj                  eaj                  eaj                  eaj                  eaj                  g       ej                  ej                  ej                  ej                  ej                  ej                   ej                  ej                  ej                  ej                  ej                  ej
                  ej                  d!Zd"efd#Zd$ Zd% Zd&e&fd'Zd( Zd) Zd* Zd+ Zd, Zd-e&j$                  d-fd.Zd/ Zdrd0Z	 	 	 	 	 	 dsd1Zdtd2Zdtd3eHd"ej0                  fd4Z eecj4                  d5      d3eHd"ej0                  fd6       Zd-d7d3eHd"ej0                  fd8Z eeaj:                  j0                  d5      d3eHd"ej0                  fd9       Zd-d7d3eHd:ej>                  fd;Z eecjB                  d5      d3eHd:ej>                  fd<       Zdd=e&j$                  d-ddd-d-df	d>Zd? Z e        	 dtd@Z eeajL                  d-dA      dB        Z eeajN                  d-dA      dC        Z eeajP                  eajR                  eajT                  eajV                  ecjX                  g      dD        Z eeadE      r  eeaj^                        e        eeaj`                  d5      dudF       Z eeajb                  d5      dudG       Z eeajd                  g      dudH       Z eeajf                        dI        Z eeajh                        dJ        Z eeajj                        dK        Z eeajl                        dL        Z eeajn                  jp                        dM        Z eeajr                        dN        Z eeajt                  d5      dO        Z eecjv                  d5      dP        Z eeajx                  d5      dQ        Z eeajz                        dR        Z eeaj|                  d5       eeaj:                  d5       eeaj~                  d5      dS                      Z eeaj                  d5      dT        Z eeaj                  d5      dvdU       Z eeaj                  d5      dudV       Zn eeaj                  d5      dudW       Z eeaj                  d5      dudX       ZodwdYZ eehj                  d5      dZeHd[eHd\eHd]ed^ed_ed"ej0                  deHfd`       Z eehj                  d5      dZeHd[eHd\eHd]ed^ed_ed"ej0                  deHfda       Z eehj                  jp                  d5      dZeHdbedced^ed_ed"ej0                  deHfdd       Z eehj                  jp                  d5      dZeHdbedced^ed_ed"ej0                  deHfde       Z eehj                  j                  d5      dZeHdbeHdceHd^ed_ed"ej0                  deHfdf       Z eehj                  j                  d5      dZeHdbeHdceHd^ed_ed"ej0                  deHfdg       Z eeaj                        dwdh       Z eeaj                  d5      dxdiedjedkefdl       Z eeaj                  d5      dxdiedjedkefdm       Z eeaj                  d5      dxdiedjedkefdn       Z eeaj                  d5      do        Z eeaj                  d5      dydp       Z eeaj                  d5      dwdq       Z eeaj                  d5      dwdr       Z eeaj                  d5      ds        Z eeaj                  d5      dt        Z eeaj                  d5      du        ZېdwdvZ eeaj                        dzdx       Zݐd{dyZ e j                  d      dz        Zdud{ej                  fd|Zdud{ej                  fd}Zd{d~ej                  j                  fdZd|dZd Z eej                  j                  j                  d5      d        Z eeaj                  d5      d        Z eeaj                  d5      d        Z eeaj                  j                  d5      d        Z eeaj                        d        Z e j                  d      d        Zd Z eeaj                  jp                        Z eeaj                  j                        Z eeaj                  jp                        Z eeaj                  j                        Z eeaj                          eeaj                        d        Z eeaj                        d        Z ee9j                  d5      d        Z ee9j                  d5      d:ej>                  fd       Z ee9j                  d5      d        Z  ee9j                  d5      d        Z ee9j                  d5      d ddee   deHdediefd       Z ee9j                  d5      d ddededee   deHdief
d       Z eeaj                  d5      d-d-ddZeHdeHdedefd       Zd Zd Z	d Z
d ZdhZd Z eeaj                          eeaj                          eeaj                           eeaj"                          eeaj$                  d-        eeaj&                  jp                  d-        eeaj(                          eeaj*                  d-        eeaj,                          eeaj.                          eeaj0                          eeaj2                          eeaj4                          eeaj6                          eeaj8                  jp                          eeaj:                  jp                          eeaj<                          eeaj>                  j@                          eeajB                  jp                          eeajD                  jp                          eeajF                          eeajH                  d-        eeaj                  e        eeajJ                  e        eeajL                  e	        eeajN                  e	        eeajP                  e	        eeajR                          eeajT                          eeajT                          eeajV                          eeajX                          eeajZ                          eeaj\                  e        eeaj^                          eeaj`                          eeajb                          eeajd                          eeajf                          eeajh                          eeajj                          eeajl                          eeajn                  e	        eeajp                          eeajr                  e        eeajt                          eeajv                          eeajv                  jx                          eeajz                          eeaj|                          eeaj~                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                  j                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                  jp                          eeaj                  jp                  d-        eeaj                  e        eej                  j                  j                          eej                  j                  j                          eeaj                          eeaj                          eeaj                  e	        eeaj                          eeaj                          eeaj                          eeaj                          eeaj                          eeaj                  j                          eeaj                  jp                  ed-        eeaj                  jp                  ed-        eeaj                  jp                  ed-        eeaj                  jp                  ed-        eeaj                  jp                  ed-        eeaj                  jp                  ed-        eeaj                  jp                  ed-        eeaj                  jp                  ed-        eeaj                  jp                  e        eeaj                  jp                  e        eeaj                  jp                  e        eeaj                  jp                  e        eeaj                          eeaj                  d5      dtd       Z} eeaj                        ddd       Z~d Z eead      r  eeaj                         e~        eecj                        d        Z eeaj                  d5      dedefd       Z eeaj                  d5      d}d       Zd Z eej                  eaj
                  g      dddd-dd       Z eej                        drd       Z eej                        d        Z eeaj                        d        Z eeaj                        d        Zd Z eeaj                  d5      d        Zd Z eej                  eaj                  g      dddddddd       Zd Zd Z  eeaj"                         ee            Z e ed            Z e ed             Zd Z eeaj*                        dddddd       Z eeaj,                        dddddd       Z eeaj.                        dddddd       Z eecj0                  jp                        d        Z eej2                  eaj2                  g      d        Z eeaj4                  d5      dtd       Z eeaj6                  d5      d~dÄ       ZdĄ Zdń ZdƄ ZdǄ Z eeaj@                  d5      dȄ        Z eeajB                  d5      dɄ        Z eeajD                        dtdʄ       Z eeajF                        dtd˄       Zd̄ Zd̈́ Z eeajL                  d5      dtd΄       Z ee9jN                  d5      dtdτ       ZdЄ Z eeajR                  jp                  d-ѫ      Z eeajV                  jp                  d-ѫ      Z eeajR                  d5      d҄        Z eeajV                  d5      dӄ        ZedԄ        Z eeaj\                  d5      dudՄ       Z eeaj^                  d5      defdք       Zdd=dלdej                  j                  dedee   defdۄZ eeajb                  d5      ddܜdedee   fd݄       Z eeajd                  d5      defdބ       Z eeajf                  d5      defd߄       Z eeajh                  d5      defd       Z eeajj                  d5      d=ddedefd       Z	 	 ddeee   df   dedefdZ eeajn                  jp                        dud[ee   fd       Z eeajp                  jp                        dud[ee   fd       Z eeaj                  jp                        	 drdee   dee   fd       Zx eeaj                  jp                        	 drdee   dee   fd       Zy eeajr                  jp                        	 	 	 ddee   dee   dee   fd       Z eeajt                  jp                        	 	 	 ddee   dee   dee   fd       Zd Z eecjx                  jp                        d        Z eeajz                  d5      dwd       Zdej|                  deej|                  ef   fdZdej|                  dej|                  fdZdej|                  dej|                  dej|                  fdZ	 ddZd Zd ZdddZ eecj                  d5      	 dtd       Z eecj                  d5      d        Z eeaj                          eeaj                  jp                  d-ѫ      Z eeaj                  d5      d        ZvddZɐd  Zʐd Zːd Z eeaj                  jp                  d-ѫ      Z eeaj                        d        Z eeaj                  jp                  d-ѫ      Z eeaj                        d        Z eeaj                  jp                  d-ѫ      ZҐd Z eeaj                        d        Z eeaj                  jp                        	 dd       Z eeaj                  jp                  d-ѫ      Z eeaj                  jp                  d-ѫ      Z eeaj                  d5      	 	 	 	 	 dd       Zp eeaj                  d5      	 	 	 	 	 dd	       Z֐d
 Z eeaj                  jp                  d-ѫ      Z eeaj                  d5      	 dud       Zq eeaj                  jp                  d-ѫ      Z eeaj                  d5      	 dud       Zڐd Zܐd ZݐdudefdZސd Z eeaj                        dddd       Zd Zd Zd Zd Z eeaj                  ecj                  g      dudd-dd       Z eeaj                        dudd-dd       Zd Zed        Z eeaj                  j                  d-ѫ      Z eeaj                  j                  d-ѫ      Z eeaj                  j                  d-ѫ      Z eeaj                  d=      d        ZdtdZ eeaj                        d         Z eeaj                  d5      dtd!       Zed"        Zed#        Z eeaj                  d=      dud$       Z eeaj                  gd=      d%        Zd3e:j                  dee:j                     fd&Z eecj                  gd=      d'        Z eeaj                  eaj                  j                  gd=e&j                  A      d(        Z eeaj                  ecj                  gd=      d)        Z eeaj                         d*        Z  eeaj                  ecj                  g      dddd+       Z eeaj                  jp                        Z eeaj
                  jp                        Z eeaj                  jp                        Z eeaj                  jp                        Z
 eeaj                  jp                        Z eeaj                        drd,       Z eeaj
                        drd-       Z eeaj                        d.        Z eeaj                  d5      dud/       Z	 eeaj                  d5      dud0       Z eeaj                        dddd1       Z eeaj                        dd2       Z eeaj                   d5      dd3       Z eeaj$                  d5      dd4       Z  eecj(                         eސd5               eeaj*                         eސd6            Z  eeaj.                         eސd7            Z  eeaj2                         eސd8ej                  9            Z  eeaj6                         eސd:ej                  9            Z eeaj:                  d=d;<      Z eeajv                  jx                  d-ѫ      Z eeajv                  jx                  d5      ddwd-d=d>       Z eeajv                  jp                  d5      dd?       Z;drd@Z dA Z! e!eajD                        Z" e eajF                        Z# e eajH                        Z$ eeajJ                        Z% e!eajL                        Z& e!eajN                        Z' eeajP                        Z( eeajR                  d=B      Z) e!eajT                          e!eajV                          eeajX                        Z, eeajZ                        Z- eeaj\                        Z. eeaj^                  dCD      Z/ eeaj`                        Z0 eeajb                        Z1 eeajd                        Z2 e eajf                          e eajh                        Z4  eeajj                  e&j                  5      e4        e eajl                          e eajn                          e eajp                          e!eaj                          eeajr                  dd=ej
                  E      Z9 eeajt                  dd=ej
                  E      Z: eeajv                  dd=ej
                  E      Z; eeajx                  dd=ej
                  E      Z< eeajz                        Z= eeaj|                        Z>  eeaj~                        e=         eeaj                        e>        eeaj                        ZA eeajX                        Z, e eaj                        ZB eeaj                          eeaj                  dFD      ZD eeajj                          eeaj                  ej
                  9         eeaj                        eA        eeaj                  ej
                  9        eeaj                  ej
                  9        eeaj                  ej
                  9        eeaj                  ej
                  9      ZJ eeaj                  ej
                  9        eeaj                  ej
                  9        e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                          e eaj                         ddGl]m^Z^m_Z_ dH Z`e_D ]S  Za e`eaea      D ]  \  ZbZcZd eebeaecedI         e`ecea      D ]  \  ZbZcZd eebeaecedI        U  eeaj                  j                  ed=B      Zf eeaj                  j                  ed=B      Zg eeaj                  j                  ed=B        eeaj                  j                  e      Zi eeaj                  j                  e      Zj eeaj                  j                  e)        eeaj                  j                  e)        eeaj                  jp                  eA        eeaj                  jp                  e,        eeaj                  j                  e        eeaj                  j                  e        eeaj                  j                  e      Zq eeaj                  j                  e      Zr eeaj                  e'        eeaj                  j                  e=        eeaj                  j                  e=        eeaj                  j                  e>        eeaj                  j                  e>        eeaj                  j                  e=        eeaj                  j                  e=        eeaj                  j                  e>        eeaj                  j                  e>        eeaj                  eB        eeaj                  eD        eeaj                  e}       dJ Z{ e{eaj                  j                  eaj                  j                  ef        e{eaj                  j                  eaj                  j                  eg        e{eaj                  j                  eaj                  j                  ei        e{eaj                  j                  eaj                  j                  ej        e{eaj                  j                  eaj                  j                  eq        e{eaj                  j                  eaj                  j                  er       dK Z eeaj                   e        eeaj                  e-        eeaj                  e.        eeaj                  e/        eeaj                  e0        eeaj
                  e1        eeaj                  e2        eeaj                  e        eeaj                  j                  e        eeaj                  j                  e        eeaj                  e9        eeaj                  e:        eeaj                  e;        eeaj                  e<        eeaj                  e)        eeaj                  e%        eeaj                   e&         eeaj"                        e-         eeaj$                        e.         eeaj&                        e0         eeaj(                        e1         eeaj*                        e2        eeaj,                  eaj"                          eeaj.                  eaj$                          eeaj0                  eaj&                          eeaj2                  eaj(                          eeaj4                  eaj*                          eeaj6                        drdL       Z eeaj8                  j                        dM        Z eeaj:                  j                        dN        Z eeaj<                        dO        Z e-j>                         D ]  \  ZZ  e e.e            e         eeaj                        dP        Z eej                  jF                  jH                        dQ        Z eej                  jL                  jN                        dR        Z eej                  j                  jP                  jR                        dS        Z eej                  jV                  dT      r; eej                  jV                  jP                  jp                        dU        Z eej                  j                  jZ                        dddV       Zd dWlmZ  ee        ee      dX        Z eej                  jb                  jd                        dY        Z eej                  jb                  jf                        dZ        Z eed5      d[e:jh                  defd\       Z eej                  j                  jl                  jp                        d]        Z eej                  jb                  jn                        d^        Z	 d dlZej                  jr                  Z eejt                        d_        Z eejx                        d`        Z eej|                        da        Z eej                        db        Z eej                        dc        Z eej                        dd        Z eej                        de        Z eej                        df        Z eej                        dg        Z eej                        dh        Z eej                        di        Z eej                        dj        Z eej                        dk        Z eej                  j                  j                        dl        Zddnl7mڐZ  e6eګ       ddol7mېZ  eېj                           eېj                          ddpl7mސZ  eސj                          ddql7mZ  ej                          y# eאef$ r eYj                  dm       Y w xY w(      N)defaultdict)AnyCallableDictListOptionalSetTupleUnion)patch)associative_scan_op)triton_kernel_wrapper_mutation)canonicalize_dimcanonicalize_dimscheckdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDget_computation_dtypeis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator)CeilDivFloorDivIdentity
IntTrueDivModularIndexing   )import_submodule   )configinductor_primsirtest_operators)decompositionsget_decompositions)	DtypeView
ExpandViewIndexingConstant	is_tritonops_wrapperPermuteView	Pointwise	ReductionSqueezeView	TensorBoxvalidate_irView)	ceildivdecode_device
is_dynamicis_gpuis_pointwise_use,needs_fallback_due_to_atomic_add_limitationspad_listlikesympy_productuse_scatter_fallback)opsV.	lowerings_maybe_layout_constraints	fallbacksneeds_realized_inputsforeach_opsinplace_foreach_opsinplaceable_foreach_opsfnreturnc                      t         t        j                  j                        sy t        v r	t            S  t
        v r
dt         <   y fd}t               } ||      S )zHGet layout constraints. Returns None if there are no layout constraints.Nc                     | t         j                  j                  j                  u rt        t
        <   t
           S | t         j                  j                  j                  u r
d t
        <   y t        d|        )NzUnknown layout constraint tag: )torch_CTagneeds_fixed_stride_orderconstrain_to_fx_stridesrB   flexible_layoutAssertionError)tagrH   s    ]/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/torch/_inductor/lowering.pyhandle_layout_constraint_tagz>maybe_layout_constraints.<locals>.handle_layout_constraint_tagi   sf    %((,,777,C%b),R00EHHLL000,0%b) #B3%!HII    )
isinstancerL   _ops
OpOverloadrB   rA   get_layout_constraint_tag)rH   rU   rS   s   `  rT   maybe_layout_constraintsr[   \   s`    b%**//0	&&(,,	Y(,!"%J $B
'C',,rV   c                 (   t         j                  j                  j                  t         j                  j                  j                  g}|D ]  }|| j
                  v s|c S  t        t         j                  j                  t        j                        S N)	rL   rM   rN   rO   rQ   tagsgetattrr$   #custom_op_default_layout_constraint)rH   tags_by_priorityrS   s      rT   rZ   rZ   w   sj    --$$   "''>J 588<<!K!KLLrV   c                 $    | st        d|       y )Nzinductor does not support NotImplementedError)condmsgs     rT   
assert_nyirg      s    !$>se"DEE rV   c                 D    t         t        t        t        f      r D cg c]  }t	        |       c}S t
        j                          t         t        j                  j                        r-t
        j                   fd j                         D               y y c c}w )Nc              3   6   K   | ]  }t        |        y wr]   )r_   ).0overloadrH   s     rT   	<genexpr>z,add_needs_realized_inputs.<locals>.<genexpr>   s      %
&.GB!%
   )rW   listtuplesetadd_needs_realized_inputsrD   addrL   rX   OpOverloadPacketupdate	overloads)rH   xs   ` rT   rq   rq      sx    "tUC()689)!,99b!"ejj112$$ %
24,,.%
 	
 3 :s   Bc                     t        | t        j                  j                        r)| j	                         D ]  }|t
        t        | |      <    y |t
        | <   y r]   )rW   rL   rX   rs   ru   rB   r_   )rH   
constraintrk   s      rT   add_layout_constraintry      sL    "ejj112 	JH?I%gb(&;<	J )3!"%rV   )r   r#   r!                     	   
         dtypec                 b    t        | t              s| S | t        v sJ d|  d       t        |    } | S )Nzid z missing from DTYPE_ID_LOOKUP)rW   intDTYPE_ID_LOOKUPr   s    rT   decode_dtyper      s=    eS!O#Os5'1N%OO#E"ELrV   c                     t        | t              r4t        | j                               xs t	        | j                               S t        | t
        j                        r| j                  du S t        | t              S NT)	rW   r3   r   	get_dtyper   sympyExpr
is_integerr   rv   s    rT   is_integer_typer      sX    !Y.Q2B1;;=2QQ	Auzz	"||t##!S!!rV   c                 t    t        | t              rt        | j                               S t        | t              S r]   )rW   r3   r   r   boolr   s    rT   is_boolean_typer      s*    !Y..!T""rV   type_promotion_kindc                 \    d }|D cg c]
  } ||       }}t        |d| i\  }}|S c c}w )Nc                     t        | t        t        j                  f      r| S t	        | d      sJ t        | j                               }t        j                  dg|z  | j                               S )Nr   r#   r   )
rW   r   r   Basichasattrlenget_sizerL   zerosr   )inpdims     rT   construct_inputz+get_promoted_dtype.<locals>.construct_input   sW    cFEKK01J3,,,clln%C;;sSy@@rV   r   )r   )r   argsr   arginps_r   s          rT   get_promoted_dtyper      s@    A -11SOC 1D1!4Q=PQHAuL 2s   )c                 0   t        | t        t        f      s| g} nt        |       } t        |       D ]b  }t        |t        j                  j
                        s(|j                         D ](  }t        ||      }|t        vs| j                  |       * d | S r]   )
rW   rn   ro   rL   rX   rs   ru   r_   rA   append)aten_fnrH   rk   other_fns       rT   get_overloadsr      s    ge}-)w-7m -b%**556LLN -"2x09,NN8,-- NrV   c                     t        | t        j                  j                        r|| j                  v S t        | t        j                  j
                        r|| j                         v S yNF)rW   rL   rX   rs   _qualified_op_namerY   name)op	namespaces     rT   in_namespacer      sO    "ejj112B1111	B

--	.BGGI%%rV   c           
         	
 t               D cg c]  \  }}t        |t              s| c}}
|s|ry
rw|rt        j                  	nG D cg c]1  }t        |t
        t        j                  f      st        |dd       |3 }}t        |d|i	 	
fd} D cg c]
  } ||       c} |r
rt        
t        
D cg c]  } |   	 c}       D ]
  \  }}| |<    t        t                     D ]Y  }t         |   t        j                        s!t!        j"                   |   t%         
d      j'                                      |<   [  S c c}}w c c}w c c}w c c}w )Nr   r   c                     t        | t              rt        |       S t        | t        j                        r5t        j                  | j
                  d      j                               S | S Nr   )rW   r3   to_dtyper&   Constantvalue
get_device)r   r   r   indicess    rT   promoteztransform_args.<locals>.promote  sV    #y)U++C-{{399eT'!*5E5P5P5RSS
rV   r   )	enumeraterW   r3   rL   r   r   r   r   r_   r   zipbroadcast_tensorsranger   r&   r   r+   creatern   r   )r   	broadcastr   convert_input_to_boolirv   apromoting_argsr   r   r   s   `        @@rT   transform_argsr     s`   &t_ITQ
1i0HqIG4' JJE
 a&%++!671gt,8 N  '5HE
	 %))q
)W!2g4NT!W4N!OP 	DAqDG	s4y! 	XA$q'2;;/$++DGT$wqz:J:S:S:U5VWQ	X KC J& *4Ns   EE6EE!E&c                     t        j                        fd       }t        |       }t        j	                  |       t
        j	                  t        j                  ||             |S )a  
    Add a foreach lowering to lowerings dict.

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                  N    t        |       dk  sJ  | i |}t        |       |S )Nr!   )r   r4   )r   kwargsout	decomp_fns      rT   wrappedz+_register_foreach_lowering.<locals>.wrapped4  s/    4yA~~((C
rV   )	functoolswrapsr   rE   rt   rA   dictfromkeys)r   r   r   aten_fnss    `  rT   _register_foreach_loweringr   (  sX     __Y   W%Hx T]]8W56NrV   c                      t        j                         fd       }t                t        j	                  t
        j                   |             |S )a  
    Add a lowering to lowerings dict

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                     t        |       } d}t        |       dk(  r t        | d   t         t        f      rd}| d   } t	        d D              sIt        d |j                         D              rJ t        d |j                         D              rJ d       t        |       } |r| g}  | i |}t        |       |S )	NFr#   r   Tc              3   H   K   | ]  }|t         v xs t        |d         yw)_c10d_functionalN)rC   r   )rj   rH   s     rT   rl   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>Y  s)      
JLR9_DR1C DD
s    "c              3   <   K   | ]  }t        |t                y wr]   )rW   r3   rj   rv   s     rT   rl   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>\  s     M:a3M   c              3   &   K   | ]	  }|d k(    yw)r   N r   s     rT   rl   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>^  s       U
   zout= ops aren't yet supported)
rn   r   rW   ro   allanyvalueskeysr   r4   )	r   r   unpackedr   r   r   r   r   r   s	       rT   r   z#_register_lowering.<locals>.wrappedO  s    BFt*t9>ja4-@H7D  
PW
 
 MV]]_MMMM $*KKM  /./  )02G
 6D((C
rV   )r   r   r   rA   rt   r   r   )r   r   r   r   r   r   s   ````` rT   _register_loweringr   A  sJ     __Y  < G$GT]]7G45NrV   Fc                 >    t        j                  t        | |||      S )z+
    Shim to support decorator syntax.
    r   r   r   )r   partialr   )r   r   r   r   s       rT   register_loweringr   t  s&     /3 rV   c                 P   g }t        j                  t        |       t        |      t        j                  d            D ]  \  }}|dk(  r|j                  |       |dk(  r|j                  |       4t        j                  j                  j                  ||       t        t        j                  |      j                        t        t        j                  |      j                        k  r|j                  |       |j                  |        t        t        |            S )z
    Broadcasting logic based on symbolic shapes.

    We give the shapes 0 and 1 concrete values, while all other shapes
    are symbolic sympy formulas.
    r#   )	fillvalue)	itertoolszip_longestreversedr   Integerr   r@   graphsizevarsguard_equalsr   expandfree_symbolsro   )r   boutputrv   ys        rT   broadcast_symbolic_shapesr     s     F%%Xa[EMM!,< !1 6MM!!VMM!GG))!Q/5<<?//03u||A7S7S3TTa a ! &!""rV   c                 R   |	|J d       ||t         j                  }t        d | D              s| S t        d | D              r*|xs t	        | d|ifd}| D cg c]
  } ||       c}S t        d | D              }g }| D ]  }t        |t        t        f      rn|j                  t        j                  t        j                  ||j                         |j                               t!        |j#                                            t        |t$        j&                        re|j                  t        j                  t)        ||j                         |j                               t!        |j#                                            |j                  |        |S c c}w )NzEonly one of override_return_dtype or type_promotion_kind may be givenc              3   f   K   | ])  }t        |t        j                  t        t        f       + y wr]   )rW   r   r   r   floatr   s     rT   rl   z$promote_constants.<locals>.<genexpr>  s"     HAz!ekk367H   /1c              3   f   K   | ])  }t        |t        t        t        j                  f       + y wr]   )rW   r   r   r   r   r   s     rT   rl   z$promote_constants.<locals>.<genexpr>  s"     
D:a#uekk23
Dr   r   c                     t        | t        j                        r t        j                  | t        d             S t        j                  | t        d             S r]   )rW   r   r   r&   r,   r7   r   rv   r   s    rT   
const_funcz%promote_constants.<locals>.const_func  sC    !U[[)**1e]45HII{{1e]4-@AArV   c              3   l   K   | ],  }t        |t        t        t        j                  f      s)| . y wr]   )rW   r3   r+   r&   r   r   s     rT   rl   z$promote_constants.<locals>.<genexpr>  s$     WA:a)Z1U#VaWs   *44)r   DEFAULTr   r   r   nextrW   r   r   r   r+   r   r&   r   r   r   rn   r   r   r   r,   )inputsoverride_return_dtyper   r   rv   exr   r   s          @rT   promote_constantsr    st   %)<)DONOD $)<)D=EEHHH

DV
DD% 
);*
)<*
	B (..!
1..	WW	WB
C a#u&JJ!!KK2<<>2==?CT"++-EX
 5;;'JJ!!$QH' JJqM" J) /s   F$c           	      L     d ddt         t           f fd}|S )Nalphar   c           	        	
 "t        t        t        |            r	rJ  | S t        |      }r%| '| dk7  r"t	        |      }t        |d   |       |d<   n| J |D cg c]  }|j                          c}
|d   j                         xs |d   j                         t        t        |d   j                               j                        	|dd  D ]`  }t        |t        j                        rt!              t!        |j                               k(  rDJ d d d|j                                  t"        j$                  d uxr t'        t"        j$                  dd       d uxr t"        j$                  j(                  j*                  d uxrX t"        j$                  j(                  j*                  j-                  dd      xr" t.        j0                  t.        j2                  fv 	
fd	}sSd }|D ]7  }t        |j                         j                        s'|j                         } n |s|d   j                         }xs }t5        j6                  ||
      S c c}w )Nr#   r   zndim mismatch  current_nodelow_precision_pointwise_barrierFc                 .   t        |       t              k(  sJ d|  d        t        j                  k(  r 	D cg c]
  } ||        c} S 
r.r,t        j                  k(  r 
	D cg c]
  } ||        c} S g }	D ]K  } ||       }r.t	        j
                  |d      }t	        j
                  |      }|j                  |       M  | }r.t	        j
                  |d      }t	        j
                  |      S |S c c}w c c}w )Nzwrong ndim r  F)use_compute_types)r   rL   r   float64r?   r   r   )indexloadinputs_loadedr   downcastr   emulate_precision_castsrH   is_gpu_deviceloadersoverride_fn_when_gpu_float64override_fn_when_input_boolrangess        rT   inner_fnz/make_pointwise.<locals>.inner.<locals>.inner_fn  s   u:V,LE7!F8.LL,

"'B'N2W4UTT%[4UVV,!U]]*3g5Vdd5k5VWW "# .Du+C.#&<<Ue#T!ll8U;!((-. -(*  #||C%PH<<%88
- 5V 6Ws   D1Ddevicer   r  r  )r   mapr-   r  rn   mulmake_loaderr   r   r9   r7   r   typerW   r&   BaseConstantr   r@   r   r_   r	  metagetrL   bfloat16float16r0   r   )r  r   rv   otherr  r  r   r   r  r  r  r  allow_alpharH   override_devicer  r  r   triton_fallbacks          @@@@@rT   innerzmake_pointwise.<locals>.inner  sY   &3s9f/E+F""?"F++"6+@A UaZf U3r
= =,23q1==?3##%%>)<)<)>}VAY-A-A-CDIIJABZ 	AEeR__5V I : At1VHAenn.>-?@A 	A GG4 96dB9$$))59 $$))--.OQVW9 %..%--88 	 	 	6 F !,,.--.\\^F --/ *F	
 	
w 4s   &I4r   r3   )rH   r   r&  r  r  r%  r'  r(  s   ``````` rT   make_pointwiser*    s(     /3 L
tI L
 L
\ LrV   c                 F     dddt         t         t              f fd}|S )Nr#   r  r   c                 ^   d }t        t        j                  j                  j                        dk(  xs* t        j                  j                  j
                  t        v }t        j                  j                  j                  D ]7  }|j                  D ]&  }|j                  dk(  r|j
                  t        v r%d}( 9 d }|D ]  }t        |t        t        f      s|} n |J d       g }|D ]H  }t        |t        t        f      s|j                  |gt        |      z         8|j                  |       J  |t        |       }	d gt        |      z  }
|	j                         D ]  \  \  }}}g }|D ]~  \  }}r	 |d| i}n | }||
|<   t        j                  j                  |t         j"                        sJ|sM|sP|j%                          |j                  |j'                                 |st        j                  j)                  |        t+        d |
D              sJ |
S )Nc                 ,   t        t              }t        |       D ]v  \  }}t        |  xs t        j
                  }d }|D ].  }t        |t              s|j                  j                         } n |J d       |||f   j                  ||f       x |S )Nz.foreach op should have at least one tensor arg)r   rn   r   r8   r$   #combo_kernel_foreach_dynamic_shapesrW   r3   datar   r   )	arg_pairsr   r   r   use_foreachr  ts          rT   
group_argsz9make_foreach_pointwise.<locals>.inner.<locals>.group_args(  s    d#C$Y/ =4"D))WV-W-W   A!!Y/!"!2!2!4
 &DCD&V[)*111d)<= JrV   r   call_functionTz1at least one input must be a list to a foreach opr  c              3   $   K   | ]  }|d u 
 y wr]   r   r   s     rT   rl   z8make_foreach_pointwise.<locals>.inner.<locals>.<genexpr>n  s     2Q1D=2s   )r   r@   r   r	  userstargetrF   r   rE   rW   rn   ro   r   r   itemshas_featureBackendFeatureFOREACHrealizeget_operation_nameregister_operation_listr   )r  r   r3  realize_outputsnodeusera_list_inputinputbroadcast_inputsgroupsoutputsr  r1  groupoperation_list
output_indr   r   r%  pw_fns                     rT   r(  z%make_foreach_pointwise.<locals>.inner%  s+   	$ $$**+q0 Bww##**.AA 	 GG((.. 	+D

 +?2{8R&*O+	+
  	E%$/$	
 $	?>	?$  	/EedE]3 ''#l2C(CD ''.		/ C!123&3|,,,2LLN 	@(!V[5(*N G "D66F"D\F&,
# GG''0F0FG#'NN$"))&*C*C*EF#G& //?-	@0 2'2222rV   r)  )rJ  r%  r(  s   `` rT   make_foreach_pointwiserK  $  s%    45 JtDO, JX LrV   rv   c                     | j                         k(  r|rt        |       S | S fd} t        |      |       S )Nc                 4    t        j                  |       S )N)	src_dtype)r?   r   )rv   r   rN  s    rT   	_to_dtypezto_dtype.<locals>._to_dtypey  s    ||Au	::rV   r   )r   cloner*  )rv   r   copyrO  rN  s    `  @rT   r   r   t  sC    IEuQx&Q&; B>)5A!DDrV   r   c                 @   |j                   s| j                         j                   rk| j                         r/t        | |      }t        j
                  j                  ||        |S  t        t        j                  j                  d      | |      S t        | |d      S )Nr   Fadd_to_fallback_setTrR  )
is_complexr   r   
empty_liker&   InplaceCopyFallbackr   fallback_handlerprimsconvert_element_typedefaultr   )rv   r   dsts      rT   _convert_element_typer`    s    1;;=33::< Qe,C""))#q1J#**22  Au4((rV   rW  c                   | j                         }||k(  r|rt        |       S | S d } ||      } ||      }||k7  r* t        t        j                  j
                        | |      S t        t        j                  | |            S )Nc                     | j                   rt        j                  |       j                  S t        j                  |       j                  S r]   )is_floating_pointrL   finfobitsiinfor   s    rT   _get_primitive_bitwidthz1to_dtype_bitcast.<locals>._get_primitive_bitwidth  s5    "";;u%***;;u%***rV   )	r   rQ  r[  atenviewr   r3   r*   r   )rv   r   rR  x_dtyperg  src_bitsdst_bitss          rT   to_dtype_bitcastrm    s~    kkmG%uQx&Q&+ 'w/H&u-H80		0E::))!U344rV   c                 &   |j                   s| j                         j                   r`t        j                  t        j
                  j                  t        j                  j                  j                  j                  | |            S t        | |      S r]   )rX  r   r3   r   r&   ComplexViewrL   r?   rh  ri  r   rm  r   s     rT   _view_dtyperp    se    1;;=33NN!!%)).."5"5";";QF
 	
 Au%%rV   r  c                    t        |      }| j                         |k(  r|rt        |       S | S t        j                  t
        j                  j	                  | |            S r]   )r7   r   rQ  r3   r   r&   
DeviceCopy)rv   r  rR  s      rT   	to_devicers    sM    6"F||~uQx&Q&BMM00F;<<rV   c                     t        | |d      S )NTrW  )rs  )rv   r  s     rT   _device_putru    s    QT**rV   Tc
                 .   |xs | j                   }t        |      }
|rt        d|z         }|t        |      }t        |
|||rnd||	      }
 t        | |||      |
      }
t	        t
        |      r" t        t        t
        |      d|      |
       |
S )z3A pointwise function that maps ops.{name} to inputs
libdevice_N)r   r  r  r%  r'  r   )r   r   )__name__r.   r*  r   r   r\  r_   )r   r   r   r   r   r   r  r%  use_libdevice_for_f64r'  rH   fn_libdevices               rT   register_pointwiser{    s     #7##D	T	B"<$#67".&12M&N#	
3$?5J\PT'
B
	/3	

 	

B ud	
E4  $"7	
 		
 IrV   c                  .   d} t        d      fd}fd}t        |      t        |t        j                        gfd} t	        t
        j                        |      }t        t        |       r! t	        t        t        |       d      |       |S )z2A pointwise function that maps ops.frexp to inputsfrexpc                       | i |d   S r   r   r   r   r}  s     rT   frexp0zregister_frexp.<locals>.frexp0      d%f%a((rV   c                       | i |d   S Nr#   r   r  s     rT   frexp1zregister_frexp.<locals>.frexp1  r  rV   rP  c                  0     d   | i | d   | i |fS Nr   r#   r   )r   r   pw_fnss     rT   rH   zregister_frexp.<locals>.fn  s.    vay$)&)96!9d+Ef+EEErV   NrS  )
r.   r*  rL   int32r   rh  r}  r   r\  r_   )r   r  r  rH   r}  r  s       @@rT   register_frexpr    s    D E)) 	vvU[[AF
F
	




B ud	
E4  $	
 	 IrV   c                 8    t        ||      }t        | |      }|S )Nr%  )rK  r   )r   pointwise_lowering_fnr%  rH   s       rT   register_foreach_pointwiser    s"    
 
  5;	OB	#GR	0BIrV   )r   r   c           
         d }t        |t        t        f      r t        |      |      }t        |t        t        f      r t        |      |      }| ||g}t	        |d   |d   t
        j                        }t        |      D cg c]  \  }}t        |t              s| }}}t        |t        |D cg c]  }||   	 c}       D ]
  \  }}|||<    t        t        |            D ]Y  }t        ||   t        j                        s!t        j                   ||   t#        ||d      j%                                     ||<   [  t'        ||      |d   t)        |d   |      t)        |d   |            S c c}}w c c}w )Nc                  &    t        j                  |  S r]   )r?   wherer   s    rT   rH   zwhere.<locals>.fn  s    yy$rV   r#   r!   rS  r   rP  )rW   r   r   constant_liker   r   r   r   r3   r   r   r   r   r&   r   r+   r   rn   r   r*  r   )	re   r   r   rH   r   r   r   rv   r   s	            rT   r  r    so     !eS\"M!Q!eS\"M!Q!Q<DQa.M.U.UE 't_ITQ
1i0HqIGIG.'0JQa0JKL 1Q3t9 Td1gr{{+ ''Qd71:6F6O6O6Q1RSDGT ;>"E:Q$q'5)8DGU+C  J0Js   E9 E96E?c                     t        |       dk(  r$t        | d   t        t        f      rt	        | d    S t        j                  t        | D cg c]  }|j                          c}g       }g }| D ]b  }|j                         }t        |      t        |      k7  st        d t        ||      D              rt        ||      }|j                  |       d |S c c}w )Nr#   r   c              3   V   K   | ]!  \  }}|d k(  xr |d k7  xs |d k7  xr |d k(   # ywr#   Nr   rj   r   r   s      rT   rl   z$broadcast_tensors.<locals>.<genexpr>3  s;      ,
=AQa1fa7Q!V%6Q7,
s   '))r   rW   rn   ro   r   r   reducer   r   r   r   r   r   )r   rv   r7  rF  sizess        rT   r   r   )  s    
6{aJvay4-@ &),,(//!&#AQAJJL#A2 F G 

u:V$ ,
EHPVEW,
 )
 q&!Aq N $Bs   C
c                     | S r]   r   r   s    rT   nopr  ;  s    HrV   
lift_freshc                    t        | t              sJ |(t        t        j                  | j                              S t        |t
        t        j                  f      r)t        j                  j                  j                  |      nt        d |D              }t        t        | j                               |      }t!        t        |t              s|fn|      }g }t#        | j                               D ]X  \  }}||v r>t        j                  j                  j%                  t        j&                  |d            rH|j)                  |       Z || j                         k7  rt+        | |      S | S )Nc              3   n   K   | ]-  }t         j                  j                  j                  |       / y wr]   )r@   r   r   evaluate_static_shaperj   ds     rT   rl   zsqueeze.<locals>.<genexpr>M  s%     J177##99!<Js   35r#   )rW   r3   r2   r   r/  r   r   r   r@   r   r   r  ro   r   r   r   rp   r   evaluate_exprEqr   ri  )rv   r   dims	new_shaper  ss         rT   squeezer  D  s   a###
{++AFF344 cC,- 	
..s3JcJJ 
 C

-s
3CZU3v=DI!**,'  1T	agg..<<UXXa^LQ 
 "+ajjl!:49AArV   c                 ,    t        t        | |            S r]   )rQ  r  )rv   r   s     rT   squeeze_copyr  [  s    C!!rV   c                     t        | |      }t        | t              sJ t        |t              sJ |j                  | _        | S r]   )r  rW   r3   r/  rv   r   vals      rT   squeeze_r  `  s=    
!S/Ca###c9%%%XXAFHrV   c                     t        |       rt        | dt        j                        S t	        d      } t        |t        j                        |       S )NFr   isinfrP  r   	full_likerL   r   r.   r*  rv   rH   s     rT   r  r  i  ?    qE44	W	B?>"EJJ?BBrV   c                     t        |       rt        | dt        j                        S t	        d      } t        |t        j                        |       S )NFr   isnanrP  r  r  s     rT   r  r  q  r  rV   c                 f    t        |       rt        |       S t        d      } t        |      |       S )Nceilr   rQ  r.   r*  r  s     rT   r  r  y  s/    qQx	V	B>"a  rV   c                 f    t        |       rt        |       S t        d      } t        |      |       S )Nfloorr  r  s     rT   r  r    /    qQx	W	B>"a  rV   c                 f    t        |       rt        |       S t        d      } t        |      |       S )Nroundr  r  s     rT   r  r    s/    qQx!!~b!!$$rV   c                 f    t        |       rt        |       S t        d      } t        |      |       S )Ntruncr  r  s     rT   r  r    r  rV   c                    ddl m} t        | g      \  } t        | t        j
                        rt        j                  | t        |            S t        | t              sJ t        |t        t        f      sJ t        | j                               t        |      k(  r| S  || j                               st        j                  j                  j                  t!        | j                                     }|dkD  rL ||      sD| j#                  t        j                  j                  j                  t!        |            |z         t        t        j                  | j$                  t        |                  S )Nr   )free_unbacked_symbols)%torch.fx.experimental.symbolic_shapesr  r  rW   r&   r  r+   r   ro   r3   rn   r   r@   r   r   	size_hintr=   
mark_reuser/  )rv   r  r  x_size_products       rT   r   r     s   KaS!DQ!R__%  E%L11a###edE]+++QZZ\eEl* .))33M!**,4OP
 A&;E&BLL  **=+?@NR Z&&qvvuU|<==rV   c                     t        |      }|D ]  }d||<   	 | }t        |      D ]  \  }}|dk7  st        ||      } t        ||      S Nr  )rn   r   	unsqueezer   )r   shapebroadcast_dimensionsr  broadcast_dimensionvidxrv   s           rT   broadcast_in_dimr    sf    UA3 $!#
$ 	
AA, "Q7!S!A" !UrV   c                 6    t        | |j                               S r]   )r   r   )rv   r   s     rT   	expand_asr    s    !QZZ\""rV   c                 4   t        | j                               t              t              kD  rFt        j                  d      gt              t              z
  z  z   t        | t                    } t              t        | j                               k(  sJ t        | j                               }d}t        t                    D ]  }|   dk(  rd}||   |   z  ||<    |r*t        || j                         | j                               S t        d t              D              rt        t        | |            S fd}t        j                  j                   j#                  t%                    }|dkD  rD| j'                  t        j                  j                   j#                  t%        |            |z         | j)                         t+        j,                  | j                         | j                         |t        |            S )	Nr#   Fr   Tr   r  c              3   :   K   | ]  \  }}|d k(  xs |d k(    ywr  r   r  s      rT   rl   zrepeat.<locals>.<genexpr>  s$     
A$!QAFa1f
As   c                    t        |       t              k(  sJ t        |       } t        t                    D ]B  }|   dk7  s|   dk(  rt        j                  d      | |<   -t        | |   d|         | |<   D  |       S )Nr#   r   )r   rn   r   r   r   r    )r  r   old_sizerepeatsx_loaders     rT   r  zrepeat.<locals>.inner_fn  s    5zS\)))Us7|$ 	IAqzQA;!#$}}Q/E!H.uQxHQKHE!H	I rV   r  )rn   r   r   r   r   ri  r   emptyr   r   r   r   rQ  r   r@   r   r   r  r=   r  r  r0   r   )	rv   r  new_sizezero_tensorr   r  old_size_productr  r  s	    `     @@rT   repeatr    s   AJJL!H
7|c(m#MM!$%WH)EFQDN#w<3qzz|,,,,AJJL!HK3w<  /1:?KqkGAJ./
 XQ[[]1<<>JJ

A#gx*@
AAVAx())	 ww''11-2IJ!	GG&&}X'>?CSS	
 }}H||~kkmH~	 rV   c                     t        | t              sJ t        |t        t        f      sJ t        t	        j
                  | j                  |            S r]   )rW   r3   rn   ro   r5   r   r/  )rv   r  s     rT   ri  ri    sB     a###edE]+++T[[/00rV   c                     t        | t              sJ t        |t        t        f      sJ t        t	        j
                  | j                  t        |                  S r]   )rW   r3   rn   ro   r/   r   r/  )rv   r  s     rT   permuter    sF    a###dT5M***[''d<==rV   c           
          t        | t              sJ t        | |d      }t        t        j                  j                  | j                  |||||            S )Nr   clamp)rW   r3   _validate_dimr&   	SliceViewr   r/  )rv   r   startendstepr  s         rT   slice_r  	  sK    a###
3
"CR\\((eS$e(TUUrV   c                    t        | t              r>t        | j                  t        j                        r| j                  j                         } | j                          t        j                  |       st        d|  d      t        j                  |       \  }}t        j                  |j                  |j                  |D cg c]  }t        j                  |       c}|D cg c]  }t        j                  |       c}t        j                  |xs d            }t        t        j                  ||            S c c}w c c}w )Nzunrealized as_strided(z, ...)r   )rW   r3   r/  r&   BaseViewunwrap_viewr<  is_storage_and_layoutrd   as_storage_and_layoutFixedLayoutr  r   r   r   ReinterpretView)rv   sizestridestorage_offsetstorage
old_layoutr  
new_layouts           rT   
as_stridedr    s    !YJqvvr{{$CFF IIK##A&!$:1#V"DEE2215GZ"&'Qa'"()Qa)^(q)J R''<==	 	()s   E &Ec                 d    t        | t              sJ t        | |||      j                  | _        | S r]   )rW   r3   r  r/  )rv   r  r  r  s       rT   as_strided_r  #  s/    a###48==AFHrV   c                 4    t        | |||      }t        |      S r]   )r  rQ  )rv   r  r  r  results        rT   as_strided_copyr  *  s    48F=rV   c                     g d} D ]1  }j                  |||j                            z   f       d   d   }3  D cg c]  }|j                          c} fd}t         d   j                               }d   d   |<   t	        j
                   d   j                          d   j                         ||      S c c}w )Nr   r  c           	        
 t        j                  |    t        j                        }g }g }t	        t                    D ]?  

dk(  r$t        j                  dt        j                        n)t        j                  
   d   t        j                        }t        j                  
   d   t        j                        }t        j                  ||      }t        j                  ||      }
dk(  r|}n*
t              dz
  k(  r|}nt        j                  ||      }|j                  |       t        |       t           
   d   z
        <   |j                  t        j                  |
fdd             B |d   }	t	        t              dz
  dd      D ]  
t        j                  |
   |
   |	      }	! |	S )Nr   r#   c                                 S r]   r   )r   idx_loadinputs_loaderss   rT   <lambda>z1pointwise_cat.<locals>.inner_fn.<locals>.<lambda>\  s    -N1-h7 rV           r  r!   )r?   
index_exprrL   int64r   r   constantgeltand_r   rn   r   maskedr  )r  idx_dimmasksmasked_loadsr  r  
start_condend_condmasknext_valr   r  r   r   r  inputs_rangess             @@rT   r  zpointwise_cat.<locals>.inner_fn:  s   ..S5;;7s6{#  	A 6 Q,^^M!$4Q$7E 
 ..q!1!!4ekkBC/Jvvgs+HAvc&kAo%!xx
H5LLCyH %Xc]]15Ea5H%HIHSM

75 	D  #Fq("b1 	AyyaQH	 rV   r  )r   r   r  rn   r0   r   r   r   )r   r   prev_endr   r  r  r  r  s   ``    @@rT   pointwise_catr  0  s    9;MH )h3<<>#3F(FGH $R() 4::Ccoo':N.` F1I&&()H!"%b)HSMay##%Qi!!#	 k ;s   CrC  scaleszero_pointsaxis	quant_min	quant_maxc           	        	
 t        j                               dk(  sJ d       t        j                               dk(  sJ d       | j                         t        j                  k(  rt        | t        j                        } | j                         t        j                  k(  sJ d| j                                 t        | j                               k  s!J dt        | j                                       | j                         j                         	j                         
	
f	d}t        j                  | j                         || j                               S )Nr#   expect scales 1 dimexpect zero_points 1 dim<Expecting input to have dtype torch.float32, but got dtype: Expecting axis to be < c                 b  	 | 
   f} |       } |      } |      }t        t        j                        \  }}j                  t        j                  k7  r$t	        j
                  |t        j                        }j                  t        j                  k7  r$t	        j
                  |t        j                        }t	        j                  |      }t	        j                  ||z        |z   }t	        j                  |t	        j                  ||            }	t	        j
                  |	      S Nr   )_create_constantsrL   float32r   r?   r   r  
reciprocalr  maximumminimum)r  channel_idxrC  scale
zero_pointqminqmax	inv_scaler  clampedr  r   input_loaderr  r  r  scales_loaderr  zero_points_loaders             rT   r  z;quantized_decomposed_quantize_per_channel.<locals>.inner_fn  s    4ylS!k*'4
&y)5==Q
d<<5==(LL6E+j%++>JNN5)	ii	)*Z7++dCKKc$:;||GU++rV   r  )r   r   r   rL   r"  r   r   r  r0   r   r   rC  r  r  r  r  r  r   r  r+  r,  r-  s    `````` @@@rT   )quantized_decomposed_quantize_per_channelr/  u  sF    v !Q&=(==&{##%&!+G-GG+ENN*.U]]*Z	EeooFWEXYZ*#  9	 U^^%5!6 789  $$&L&&(M$002, ," !~~	 rV   c                 j  	
 t        j                               dk(  sJ d       t        j                               dk(  sJ d       | j                         |k(  sJ d| d| j                                 t        | j                               k  s!J dt        | j                                       | j                         j                         	j                         
	
fd}t	        j
                  | j                         t        j                  || j                               S )	Nr#   r  r  Expecting input to have dtype , but got dtype: r  c                    |    f} |       } 	|      } |      }j                   t        j                  k7  r$t        j                  |t        j                        }
j                   t        j                  k7  r$t        j                  |t        j                        }t        j
                  t        j                  |t        j                        |      |z  }|S r]   )r   rL   r   r?   r   sub)r  r$  rC  r%  r&  r  r  r+  r  r,  r  r-  s         rT   r  z=quantized_decomposed_dequantize_per_channel.<locals>.inner_fn  s    4ylS!k*'4
<<5==(LL6E-j%--@Jggcll5%--8*EM
rV   r  	r   r   r   r  r0   r   r   rL   r   r.  s    ```    @@@rT   +quantized_decomposed_dequantize_per_channelr6    s)    v !Q&=(==&{##%&!+G-GG+U"T	'w.?@Q?RST"#  9	 U^^%5!6 789  $$&L&&(M$002  !mm~~	 rV   r%  r&  c                    | j                         t        j                  k(  rt        | t        j                        } | j                         t        j                  k(  sJ d| j                                 | j                         fd}t        j                  | j                         t        j                  |t        |      t        |            | j                               S )Nr  c                 L    
|       }t        d|z  |t        j                        \  }}t        j                  ||z        |z   }t        t        j                        \  }}t        j
                  t        j                  ||      |      }t        j                  |	      S )N      ?r   )r  rL   r   r?   r  r#  r"  r   )r  r%  r&  rC  r)  r  r'  r(  r*  r   r+  r  r  s            rT   r  zBquantized_decomposed_quantize_per_tensor_default.<locals>.inner_fn  s    S! 1%K5==!
	: ii	)*Z7&y)5==Q
d++ckk#t4d;||GU++rV   r%  r&  r  )r   rL   r"  r   r   r  r0   r   r   r   r   r   r   r   rC  r%  r&  r  r  r   r  r+  s      ``` @rT   0quantized_decomposed_quantize_per_tensor_defaultr<    s     ENN*.U]]*Z	EeooFWEXYZ* $$&L, !""E%LS_
 ~~ rV   c                 `   | j                         |k(  sJ d| d| j                                 | j                         fd}t        j                  | j	                         t
        j                  t        j                  |t        |      t        |            | j                               S )Nr1  r2  c                      |       }t        ||t        j                        \  }}t        j                  t        j
                  |t        j                        |      |z  }|S r  )r  rL   r   r?   r4  r   )r  r%  r&  rC  r  r+  s        rT   r  zDquantized_decomposed_dequantize_per_tensor_default.<locals>.inner_fn  sM    S!-eZu}}Uzggcll5%--8*EM
rV   r:  r  )r   r  r0   r   r   rL   r   r   r   r   r   r   r;  s          @rT   2quantized_decomposed_dequantize_per_tensor_defaultr?    s     	U"T	'w.?@Q?RST" $$&L !mm""E%LS_
 ~~ rV   c                 0  	 | j                         t        j                  k(  rt        | t        j                        } | j                         t        j                  k(  sJ d| j                                 t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       | j                         j                         j                         		fd}t        j                  | j                         || j                               S )Nr  r   r#   expect scale as scalar tensor"expect zero_point as scalar tensorc                     	|       } t        j                               dk(  rdnd      } t        j                               dk(  rdnd      }j                  t        j                  k7  r$t        j                  |t        j                        }j                  t        j                  k7  r$t        j                  |t        j                        }t        j                  |t        j                  |      z        |z   }t        
t        j                        \  }}t        j                  t        j                  ||      |      }t        j                  |      S )Nr#   r   r   r   )r   r   r   rL   r   r?   r   r  r!  r  r#  r"  )r  rC  _scale_zero_pointr  r'  r(  r*  r   r+  r  r  r%  scale_loaderr&  zero_point_loaders           rT   r  zAquantized_decomposed_quantize_per_tensor_tensor.<locals>.inner_fn:  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKiiv 667+E&y)5==Q
d++ckk#t4d;||GU++rV   r  )r   rL   r"  r   r   r   r   r  r0   r   r   
rC  r%  r&  r  r  r   r  r+  rG  rH  s
    ````` @@@rT   /quantized_decomposed_quantize_per_tensor_tensorrJ    sf    ENN*.U]]*Z	EeooFWEXYZ*u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  $$&L$$&L"..0, , !~~	 rV   c                   	 t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       | j                         |k(  sJ d| d| j                                 | j                         j                         j                         		fd}t	        j
                  | j                         t        j                  || j                               S )	Nr   r#   rA  rB  r1  r2  c                 *    |       } t        j                               dk(  rdnd      } 	t        j                               dk(  rdnd      }j                  t        j                  k7  r$t        j                  |t        j                        }j                  t        j                  k7  r$t        j                  |t        j                        }t        j                  t        j                  |t        j                        |      |z  }|S )Nr#   rD  r   )r   r   r   rL   r   r?   r   r4  )
r  rC  rE  rF  r  r+  r%  rG  r&  rH  s
        rT   r  zCquantized_decomposed_dequantize_per_tensor_tensor.<locals>.inner_fnh  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKggcll5%--8+FO
rV   r  r5  rI  s
    ``    @@@rT   1quantized_decomposed_dequantize_per_tensor_tensorrM  O  sL    u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  	U"T	'w.?@Q?RST" $$&L$$&L"..0	 	 !mm~~	 rV   c                 N   | d   j                         j                  dk(  }|rt        d | D              rp| D ]  }|j                           t        d | D              rt	        t
        j                  g|  \  } } t        t
        j                  j                        | |      S t        |       dk(  rt        | d         S t        | d   |d      }t        | dt        j                  i}| D cg c]  }t        ||       } }dt         t"        t$        j&                  f   dt$        j(                  fd	d
 fdt+        fd| D              }dt,        ffd|r)t#        t$        j.                  j1                  | |            S fdd}ddt2        j4                  j6                  fdt        |       |k  s1t        |       t8        j:                  k  rt        fd| D              rt        fdt<        j>                  j@                  D              }	t+        fd| D              xr |	}
t        fd| D              xr t+        fd| D               }|
s|r|stC        | |      S t#        t$        j.                  j1                  | |            S c c}w )Nr   cpuc              3   |   K   | ]4  }|j                         t        j                  t        j                  fv  6 y wr]   )r   rL   int8uint8rj   rC  s     rT   rl   zcat.<locals>.<genexpr>~  s.      ;@ejj%++66s   :<c              3   T   K   | ]   }t        |j                               d k(   " yw)r{   N)r   r   rS  s     rT   rl   zcat.<locals>.<genexpr>  s!     >es5>>#$)>s   &(r#   r   rv   rI   c                    t        | t              rJt        | j                  t        j                        r| j                  j                         S | j                  S t        | t        j                        r| j                  S | S r]   )rW   r3   r/  r&   r  r  
StorageBoxr   s    rT   unwrap_tensorzcat.<locals>.unwrap_tensor  sV    a#!&&"++.vv))++vva'66MrV   c                     t        | t        j                        xr$ t        | j                  t        j                        S r]   )rW   r&   ComputedBufferr/  r1   )r2  s    rT   is_reductionzcat.<locals>.is_reduction  s)    !R../TJqvvr||4TTrV   c                     t        | t        t        j                  f      r  |             S  |       xs> t        | t        j                        xr" t        fd| j                         D              S )Nc              3   h   K   | ])  } t         j                  j                  |             + y wr]   )r@   r   
get_buffer)rj   readcan_fuse_reductions     rT   rl   z2cat.<locals>.can_fuse_reduction.<locals>.<genexpr>  s-       #177#5#5d#;<s   /2)rW   r3   r&   rV  r0   r   get_read_names)r2  r_  rZ  rW  s    rT   r_  zcat.<locals>.can_fuse_reduction  sh    a)R]]34%mA&677O !R\\*  ,,. 	
rV   c              3   .   K   | ]  } |        y wr]   r   rj   r2  r_  s     rT   rl   zcat.<locals>.<genexpr>  s     Ba.q1B   c                 6   t        j                  |       r:t        j                  | d      \  }}t         j                  j	                  |       S t        | t        t         j                  f      r  |             S t        | t         j                        ryy)NF)freezeT)	r&   r  r  ConcatKernelcan_realize_into_without_copyrW   r3   rV  r0   )rv   r  r   should_lower_cat_inputrW  s      rT   rh  z#cat.<locals>.should_lower_cat_input  sx     ##A&11!EBJGQDDWMMMa)R]]34)-*:;;a&rV   c                 H   t        | t        t        j                  f      r  |             S t        | t        j                        sy| j                         j                  }| j                         D ]*  }| t        j                  j                  |            z  }, |S r   )rW   r3   r&   rV  r0   inner_fn_opcountnum_opsr`  r@   r   r]  )rv   countr^  op_countrW  s      rT   rm  zcat.<locals>.op_count  s    a)R]]34M!,-- !R\\*""$,,$$& 	8DXagg00677E	8 rV   r   r!   r   c                 n    | t         j                  j                  t         j                  j                  fv S r]   )rh  catr^  constant_pad_ndr   s    rT   additional_pointwise_opsz%cat.<locals>.additional_pointwise_ops  s(    dhh&&(<(<(D(DEEErV   c              3   4   K   | ]  } |      k    y wr]   r   )rj   r2  MAX_SIMPLE_OP_COUNTrm  s     rT   rl   zcat.<locals>.<genexpr>  s     Cq22Cs   c              3   6   K   | ]  }t        |        y wr]   )r:   )rj   userr  s     rT   rl   zcat.<locals>.<genexpr>  s!      
 S":;
rm   c              3   .   K   | ]  } |        y wr]   r   rj   r   rh  s     rT   rl   zcat.<locals>.<genexpr>  s     >&s+>rc  c              3   .   K   | ]  } |        y wr]   r   rx  s     rT   rl   zcat.<locals>.<genexpr>  s      "
,/"3'"
rc  c              3   .   K   | ]  } |        y wr]   r   rb  s     rT   rl   zcat.<locals>.<genexpr>  s     <(+<rc  )"r   r  r   r<  require_channels_lastrh  ro  r[  r^  r   rQ  r  r   r   r   r   r   r3   r&   rV  IRNoder   r   rf  r   rL   rX   rY   r$   max_pointwise_cat_inputsr@   r	  r6  r  )r   r   
cpu_devicerC  r   r   r   fusable_reductionMAX_COMPLEX_POINTWISE_CATpointwise_usesfuse_pointwise_usehorizontal_fuse_catrt  rr  r_  rZ  rm  rh  rW  s               @@@@@@@rT   ro  ro  {  sj   %%',,5Jc DJ 
  	EMMO	>v>>-dhh@@IFA1 0 01&#>>
6{aVAY
q	3
*C	%D%L%LE /55shsE"5F5
y"--78 
RYY 
U

 B6BBT " //<==$ !"FUZZ%:%: F 6{//	V77	7CFCC 
~~++
 
 >v>>Q> 	 " "
39"
 
 =<V<<< 	 "5>O --R__++FC899E 6s   5J"offsetdim1dim2c                   
 | j                         t              t              t              t        k7  fd       t        j
                  j                  j                  t        j                  |d            }|r\t        j
                  j                  j                  t        j
                  j                  j                     |z            d      }n[t        j
                  j                  j                  t        j
                  j                  j                        |z
        d      }d
|r| df
nd|f
t              D cg c]  \  }}|fvs| }}}|j                  |       
fd}	t        t        j                   j#                  | ||	            S c c}}w )N)r  rankc                      d  d S )Nz(diagonal dimensions cannot be identical z, r   )r  r  s   rT   r   zdiagonal.<locals>.<lambda>  s     HbQUPVW rV   r   )r   r   c                     | d   }dgt        	      z  }d}t              D ]1  }|k(  r|d   z   ||<   |k(  r|d   z   ||<   %| |   ||<   |dz  }3 |t        	      dz
  k(  sJ |S )Nr  r   r#   r!   )r   r   )
r  diag_idxoriginal_idxcur_dimr  base_idxr  r  num_dimsoriginal_shapes
        rT   	reindexerzdiagonal.<locals>.reindexer  s    r7sS00x 	ADy"*Xa["8Qd"*Xa["8Q"%g,Q1	 #n-1111rV   )r   r   r   r   r@   r   r   r  r   Ltevaluate_maxevaluate_minr   r   r3   r&   GenericViewr   )rC  r  r  r  offset_negative	diag_sizer   r  r  r  r  r  r  s     ``      @@@rT   diagonalr    s   ^^%N>"H84D84D	W gg&&44UXXfa5HIOGG$$11GG))t$v-~d/C 	
	 GG$$11GG))t$nT&:V&C 	
	 HGQ<v;$^4N41a$8MQNEN	LL   R^^**5%CDD' Os    F:/F:c                 0    t        t        | |||            S r]   )rQ  r  )rC  r  r  r  s       rT   diagonal_copyr  ,  s    %t455rV   c                 P    t        |       }t        ||||      }t        ||       |S r]   )rQ  r  	mutate_to)rC  srcr  r  r  r   r7  s          rT   diagonal_scatterr  1  s*    5\FffdD1FfcMrV   c           	          t        j                  || j                         |         }t        t	        | |||dz         |      S r  )r5   handle_negative_indexr   r  r  )rv   r   r  s      rT   selectr  9  s;    

$
$S!**,s*;
<C6!S#sQw/55rV   c           
         t        | |d      }t        |t        j                        r)t        j
                  j                  j                  |      }t        |t        t        j                  f      rIt        j
                  j                  j                  | j                         |         }|g||z   dz
  |z  z  }g }d}|D ](  }||z   }|j                  t        | ||||             |}* |S )Nr   r#   r  )r  rW   r   r   r@   r   r   r  r   r   r   r   r  )	rv   r  r   r  x_sizer  r  r  r  s	            rT   splitr  ?  s    
3
"C%$   66u=%#u}}-.!!77

S8IJFUNQ.589FE dlfQUCu=> MrV   c                      t        | ||d      S )NFr  )r  )rv   r  r   s      rT   split_with_sizesr  R  s    E3e,,rV   c                     t        | |d      }t        j                  j                  j	                  | j                         |         }g }t        |      D ]  }|j                  t        | ||               |S r   )	r  r@   r   r   r  r   r   r   r  )rv   r   r  r  r   s        rT   unbindr  W  sg    
3
"CWW33AJJL4EFFF6] )fQQ'()MrV   c                 >   | j                         }t        |      }t        ||      |dk(  rt        t	        | d      |      S |   }t
        j                  j                  }|j                  ||       |j                  d       t        ||z
        dz   }|j                  |      dkD  r-| j                  |j                  t        ||z  |                   g |d  ||dz   d  |}	fd}
t        t        j                   j#                  | |	|
            S )Nr   )r  r#   c                 D    | d   |    z  z   }g | d  || dz   d S )Nr  r#   r   )r  dim_idxr   r  s     rT   r  zunfold.<locals>.reindexeru  s?    b'CHtO+8Tc8G8c#'B&788rV   )r   r   r   r  r  r@   r   r   	guard_leqguard_ltr   r  r  r   r3   r&   r  r   )rv   	dimensionr  r  r  ndimdim_sizer   new_dim_sizeout_sizer  r   s      `       @rT   unfoldr  a  s   JJLEu:D
4
+Cqyi1o400SzHwwHtX&aHtOT2Q6L(#a'	X''t0CX(NOPDtDlDU379-=DtDH9 R^^**1h	BCCrV   c                     t        | |d      }t        | j                               }|j                  |t	        j
                  d             t        | |      S r  )r  rn   r   insertr   r   ri  )rv   r   r  s      rT   r  r  |  sF    
3
"CQZZ\"IS%--*+9rV   c                     t        | |      }t        | t              sJ t        |t              sJ |j                  | _        | S r]   )r  rW   r3   r/  r  s      rT   
unsqueeze_r    s>    
As
Ca###c9%%%XXAFHrV   c                    t         j                  j                  j                  j	                  t        j                  |            }t        | j                               }|dk  r|||z   z  }d|cxk  r	||z   k  sJ  J |S r   )	r@   r   r   	shape_envr  r   sympifyr   r   )rv   r   r  r  s       rT   r  r    sr    
''


$
$
2
25==3E
FCqzz|D
Qwtf}#dVm#####JrV   r  c                     t        | |d      }t        j                  j                  j	                  | j                         |         dz  }t        | |d|      }t        | |||dz        }t        |t        |            S )Nr   r!   )	r  r@   r   r   r  r   r  r  sigmoid)rv   r   new_lenr   r   s        rT   glur    so    
3
"Cgg44QZZ\#5FG1LGq#q'"Aq#w!,Aq'!*rV   c                 @     |rt         j                           fd}|S )Nc                  x    d }t        j                  |t        j                  j                  g| i |      S )Nc                 d    t        | t        j                        rt        j                  |       S | S r]   )rW   r&   r|  r3   r   r   s    rT   wrap_tensorsz7fallback_handler.<locals>.handler.<locals>.wrap_tensors  s%    *4Q		*B9##A&IIrV   )pytreetree_mapr&   FallbackKernelr   )r   r   r  kernels      rT   handlerz!fallback_handler.<locals>.handler  s<    	J "++226KDKFK
 	
rV   )rC   rr   )r  rV  r  s   `  rT   r[  r[    s    f
 NrV   c                  .    t        j                  d       y )NzjTorchinductor does not support code generation for complex operators. Performance may be worse than eager.)warningswarnr   rV   rT   _warn_complex_not_supportedr    s    MMtrV   r2  c                    | j                         rv|ri|j                  t        j                  j                  j
                  j                  t        j                  j                  j                  j                  fv ryt                yy)z0Do not support reading or writing to this tensorFT)rX  r7  rL   r?   rh  ri  r   r\  r]  r^  r  r2  parents     rT   unsupported_input_tensorr    s]    ||~fmmIINN%%IIOO0088(
 
 #%rV   c                 X    t        | |      ry| j                  xr t        j                  S )z2Do not support writing tensor but can read from itT)r  is_cpur$   disable_cpp_codegenr  s     rT   unsupported_output_tensorr    s$    6*882222rV   r@  c                 0   | j                   t        j                  j                  u ry| j                   t        j                  j                  u ryd }t        j                  | j                  i | j                  D ]  } ||| d      s y  || | d      S )NFc                 B   t        | t        j                  j                        syd| j                  vryt        j                  | j                  d         D ]F  }t        |t        j                  j                        s(|rt        ||      s7 yt        ||      sF y y)NFr  T)rW   rL   fxNoder   r  tree_leaves_subclasses
FakeTensorr  r  )r@  r  	is_outputr   s       rT   check_skip_conditionzCfallback_node_due_to_unsupported_type.<locals>.check_skip_condition  s    $.		!&&tyy'78 		 DdE$5$5$@$@A,T6:+D&9		  rV   )r  T)	r7  rh  view_as_complexr^  lift_fresh_copyr  arg_tree_leavesr   r   )r@  allow_cpu_inputsr  r   s       rT   %fallback_node_due_to_unsupported_typer    s    {{d**222 {{d**222* %%tyy@DKK@ TU;  dd;;rV   c                 "   | t         vs
J d|         |rt        t        j                  d            rt	        | g      rt
        j                  r&| t        j                  j                  j                  v sgt        j                  j
                  j                  r4dt        j                  j
                  _        t        j                  d       t        d|  d      fd}t!        | t        j"                  j$                        r*| j'                         D ]  }t)        | |      } ||        y t!        | t        j"                  j*                  t        j"                  j,                  f      r	 ||        y t/        d|  d	t1        |              )
Nz*both a fallback and a decomp for same op: CIFznA make_fallback error occurred in suppress_errors config, and suppress_errors is being disabled to surface it.zmake_fallback(a.  ): a decomposition exists, we should switch to it. To fix this error, either add a decomposition to core_aten_decompositions (preferred) or inductor_decompositions, and delete the corresponding `make_fallback` line. Get help from the inductor team if unsure, don't pick arbitrarily to unblock yourself.c                 n    t        |        t        |         t        | d       t        |             S NrS  )rq   ry   r   r[  )op_overloadlayout_constraints    rT   register_fallbackz(make_fallback.<locals>.register_fallback  s;    !+.(!+/@AG $G[)
 	
rV   zUnsupported fallback z with type )r(   r   osgetenvr)   r$   fallback_randomrL   _decompdecompositions_for_rngextra_random_decomps_dynamosuppress_errorslogwarningrR   rW   rX   rs   ru   r_   rY   HigherOrderOperatorRuntimeErrorr  )r   r  r  r  olr  s    `    rT   make_fallbackr    sH   ^#V'QRTQU%VV#4!t$ ""emm::OOO ==//38EMM  0KKH RD !f f
 	

 "ejj112,,. 	+B!"b/Kk*	+ 
B..

0N0NO	P"22$k$r(LMMrV   c                 T    d}| D ]  }||z  }	 t        |t        j                        S )z
    TorchInductor offset calculation differs from PyTorch eager offset
    calculation for random ops (tl.rand vs torch.rand). In future, we should
    strive for same impl for tl.rand and torch.rand.
    r#   r   tensorrL   r  )r  numelr  s      rT   philox_rand_offsetr  !  s2     E 	%u{{++rV   c                 B  	
 t        j                  || t         j                  j                  |             j	                         
|j                         |j                         		
fd}t        j                  ||t        |             }t        |       }||fS )Nc                    t        j                   g       t        j                        }t        j                   g       t        j                        }t        j                  t        j
                   |       t        j                        |      }t        j                  ||      }t        j                  |      S r]   )r?   r   rL   r  rr   r  rand)	r  seed_index_exproffset_index_exprrand_index_exprr  r   offset_loader
random_posseed_loaders	        rT   r  zphilox_rand.<locals>.inner_fn:  s     ,,{2DLLr):EKKH''NN:e,ekk:<M
 
 ||FE**rV   r  )
r&   r  FlexibleLayoutcontiguous_stridesmake_indexerr  r0   r   rn   r  )r  seedr  r  r  r   r  random_values_nodeoffset_noder  r  r  s        `   @@@rT   philox_randr  -  s     
,,T2	
 ln  ""$K&&(M+ #))Dz	 %T*K{**rV   c           	          t         j                  r\t        j                  t        j
                  t        j                  j                  t        j                  j                  | ||            S t        d      )Nz&should be handled in replace_random.py)r$   r  r  r  r3   r   r&   r  rh  native_dropoutr^  rR   )rv   ptrains      rT   r  r  T  sY    $$T%8%8%@%@!QN
 	

 EFFrV   c                 x   t         j                  s-| j                         t        j                  d      k(  sJ d       | j                          t        |      dk(  st        |d   t              rt        j                  j                  nt        j                  j                  }t        j                  || g|  | S )NrO  Tthis should be handled in decomps unless config.fallback_random or the device is CPUr   )r$   r  r   rL   r  r<  r   rW   r   rh  
bernoulli_Tensorr&   InplaceBernoulliFallback)rv   r   r  s      rT   r  r  _  s    !!Q\\^u||8 & ^]^  IIK t9>ZQ7 	__## 
 Q66HrV   c                     t         j                  s-| j                         t        j                  d      k(  sJ d       t        t        |       g| S )NrO  r  )r$   r  r   rL   r  r  rQ  )rv   r   s     rT   bernoulli_pr  n  sP    !!Q\\^u||8 & ^]^  eAh&&&rV   c                     t         r]   rR   r   s    rT   _foobarr  w  s    
rV   c                 .    t         j                  d       y )Nz1using triton random, expect difference from eager)r  info)salts    rT   _warn_triton_randomr  |  s    HH@ArV   c                  J    t        t        j                  j                         y r]   )r  r@   r   creation_timer   rV   rT   warn_triton_randomr!    s    --.rV   c                      |j                  dd       t        | i |S t        j                  r|j	                  dd        t        | i |S t        d      N	generatorz-should have been handled in replace_random.py)r!  fallback_rand_generatorr$   r  popfallback_rand_defaultrR   r   r   s     rT   r  r    sV    zz+t$0&777			

;%$d5f55
H
IIrV   c                      |j                  dd       t        | i |S t        j                  r|j	                  dd        t        | i |S t        d      r#  )r!  fallback_randn_generatorr$   r  r&  fallback_randn_defaultrR   r(  s     rT   randnr,    sV    zz+t$0'888			

;%%t6v66
H
IIrV   c                 l    t        j                  |      }t         j                  j                  | |      S r]   )r&   get_stride_orderExternKernelrequire_stride_order)input_tensorr  stride_orders      rT   inductor_force_stride_orderr3    s)    &&v.L??//lKKrV   c                     t        d      )Nz.should be handled in fuse_seed_creation_pass()r  )r  s    rT   inductor_seedr5    s    
I
JJrV   c                 z    t                t        j                  t        j                  | t        |                  S r]   )r!  r3   r   r&   RandomSeedsr7   )rl  r  s     rT   inductor_seedsr8    s)    BNN5-2GHIIrV   c                 |      fd}t        j                   j                          j                         |g       S )Nc                 L    t        j                  j                               S r]   )r?   	load_seedget_name)r   r  seedss    rT   r  z&inductor_lookup_seed.<locals>.inner_fn  s    }}U^^-u55rV   r  )r0   r   r   r   )r=  r  r  s   `` rT   inductor_lookup_seedr>    s9    6 !oo	 rV   r  r  r  modec                  	 t         j                  rJ dv sJ g | } t        j                  }|j	                         }t        j                  ||| t
        j                  j                  |       |      j                         |j                         		fd}t        j                  |||g |       }|j                          |S )N)r  r,  r?  c                      t        t               g       t        j                   |       t        j                              S r]   )r_   r?   r  rL   r  )r  r@  r  r  s    rT   r  z!inductor_random.<locals>.inner_fn  s7    !wsD!ONN:e,ekk:
 	
rV   r  )r$   r  rL   r   r   r&   r  r  r  r  r  r0   r   r<  )
r  r  r@  r  r   r  r  r  r  r  s
     `     @@rT   inductor_randomrC    s    %%%%$$$$T7DMME__FtR..AA$GPVln  ""$K
 w	F NNMrV   lowhighc                f   	 t         j                  rJ g |}t        j                  }|j	                         }t        j                  |||t
        j                  j                  |      |      j                         |j                         	 	fd}t        j                  |||g |      S )Nr?  c           	         t        j                   g       t        j                   |       t        j                        t        j                  t        j
                        t        j                  t        j
                              S r]   )r?   	randint64r  rL   r  r  )r  rE  rD  r  r  s    rT   r  z"inductor_randint.<locals>.inner_fn  sV    }}ONN:e,ekk:NN3,NN4-	
 	
rV   r  )r$   r  rL   r  r   r&   r  r  r  r  r  r0   r   )
rD  rE  r  r  r  r   r  r  r  r  s
   ``      @@rT   inductor_randintrI    s     %%%%T7DKKE__FtR..AA$GPVln  ""$K
 w	 rV   	out_int32right
boundariesrK  rL  c                   t        j                               dk(  sJ t        j                  j	                  | t
        j                        r.t        j                  j	                  t
        j                        s/ t        t        j                  j                  d      | |      S j                          j                         d   | j                         }| j                         |rt        j                  nt        j                   fd}t#        j$                  ||| j                               S )Nr#   FrU  rJ  r   c                 f     |       }t        j                  |j                               }|S r]   )r?   	bucketizer<  )r  r  r   rM  boundaries_sizeindex_dtyper+  rL  s      rT   r  zbucketize.<locals>.inner_fn  s:    5!--!
 rV   r  )r   r   r@   r   r9  r:  	BUCKETIZEr[  rh  rP  r  r<  r   r  rL   r  r  r0   r   )	rC  rM  rK  rL  r  r  rQ  rR  r+  s	    ` `  @@@rT   rP  rP    s    z""$%*** 	
E>#;#;<GG
N,D,DEQ 5 55Q:%
 	
  ))+A.OF$$&L!*%++K
 
 ~~	 rV   c                     t        j                  t        j                  t        j                  j
                  ||f      \  }}||fS r]   )r  tree_map_onlyr&   r|  r/  require_stride1r   r   r   s      rT   require_denserX  )  s;    ''
		2??22T6NLD& <rV   c                     t        j                  t        j                  t        j                  j
                  ||f      \  }}||fS r]   )r  rU  r&   r|  r/  require_contiguousrW  s      rT   rZ  rZ  0  s;    ''
		2??55f~LD& <rV   c                     t        j                  t        j                  t        j                  j
                  ||f      \  }}||fS r]   )r  rU  r&   r|  r/  r{  rW  s      rT   r{  r{  7  s;    ''
		2??884.LD& <rV   c           
          d t        fdt        || j                        D              }|j                         D ci c]  \  }}| || j                  |          }}}||fS c c}}w )Nc                     t        | t        j                        rPt        j                  |j                  d   j                               }t        j                  j                  | |      S | S Nr  )rW   r&   r|  r.  r   r  r/  r0  )r   fx_argr2  s      rT   apply_constraintz1constrain_to_fx_strides.<locals>.apply_constraint?  sN    c299%..v{{5/A/H/H/JKL??77\JJ
rV   c              3   6   K   | ]  \  }} ||        y wr]   r   rj   r   r_  r`  s      rT   rl   z*constrain_to_fx_strides.<locals>.<genexpr>E  "      *5#vf%rm   ro   r   r   r8  r   fx_noder   r   kr  r`  s        @rT   rP   rP   >  ss      9<T7<<9P D EKLLNSDAqa!!W^^A%677SFS< T    !A(ztorchvision::roi_alignc           
          d t        fdt        || j                        D              }|j                         D ci c]  \  }}| || j                  |          }}}||fS c c}}w )Nc           
         t        | t        j                        s| S |j                  d   }|j	                         }t        j
                  |      }|r<|d   dk7  r4t        t        t        t        | j                                                 }|j                  s t        j                  j                  | |      S dt        | t              sJ t        | j                               dvr| S fd}	 | j                           ||       r=t         j"                  j%                  t        j                  j'                  |       |      S 	 fd}t        | j*                  t        j,                        r[ ||       sS || j/                               r=t         j"                  j%                  t        j                  j'                  |       |      S t        j                  j                  | |      S # t(        $ r Y w xY w)Nr  r  r   r   rz   r{   c           	           t         fdt        t         j                               dz
        D              }t        j
                  j                  j                   j                         d         dk(  xr |S )Nc              3      K   | ]D  }t         j                  j                  j                  j	                         |         z  d k(   F ywr   N)r@   r   r   r  
get_stride)rj   r   	ALIGNMENTrv   s     rT   rl   z`sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned_realized_tensor.<locals>.<genexpr>n  sC      " !!++ALLN1,=>JqP"s   A
Ar#   r  )r   r   r   ro  r@   r   r   r  )rv   aligned_stridesrp  s   ` rT   is_aligned_realized_tensorzMsdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned_realized_tensorm  si    ! "s1<<>2Q67" O
   **1<<>"+=> '&'rV   c                     t         j                  j                  j                  | j	                         d         z  dk(  S )Nr  r   )r@   r   r   r  r   )rv   rp  s    rT   
is_alignedz=sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned  s4    GG$$..qzz|B/?@9LQRRRrV   )rW   r&   r|  r   r  r.  rn   r   r   r   r   is_cudar/  r0  r3   ro  r@   r   try_match_insignificant_stridesrealize_inputAttributeErrorr/  r  r  )r   r_  meta_valmeta_strider2  rr  rt  rp  s          @rT   r`  z)sdpa_constraint.<locals>.apply_constraintV  s   #ryy)J;;u%oo'**;7L,1s3<<>/B)C DEL??77\JJ 	#y)))s||~f,J	'	NN)#.ww>>OO11#6  /	S chh,c?coo/077BB55c:K  33CFF  		s   3AG- -	G98G9c              3   6   K   | ]  \  }} ||        y wr]   r   rb  s      rT   rl   z"sdpa_constraint.<locals>.<genexpr>  rc  rm   rd  re  s        @rT   sdpa_constraintr|  S  su    3Gj  9<T7<<9P D EKLLNSDAqa!!W^^A%677SFS< Trh  )r  c                    |}| j                         |j                         k7  rt        || j                               }| j                         |j                         k7  rt        || j                               }| j	                         |j	                         k7  r%t        || j	                               }t        |      S t        |      S r]   )r   rs  r   r   r   r   rQ  )selfr  non_blockingrv   r   s        rT   rR  rR  W	  s    ACNN,,a*+~~3==?*Q()}}#,,.(Q(Sz8OrV   )memory_formatc          	          t        j                  | j                         | j                         | j	                         t        | j                                     S Nr  )r0   r   r   r   r  rn   r   )rv   r  s     rT   rQ  rQ  e	  s@     ||~kkmAJJL!	 rV   c                    g }t        | t              rt        | j                  t        j                        rw| j                  } t        | t        j                        rF|j                  | j                                | j                  } t        | t        j                        rFt        |       } t        |       } |r:| j                  } |d d d   D ]  }t        j                  | |      }  t        |       } | S r  )rW   r3   r/  r&   r  r   
get_layoutrQ  )rv   reinterpret_view_layoutslayouts      rT   clone_preserve_reinterpret_viewr  p	  s    !!YJqvvr7I7I$JFFB../$++ALLN;A B../ aLaAFF.tt4 	.F""1f-A	.aLHrV   r  c                \    fd}t        j                  t        |      || g      S )Nc                 D    t        j                  | d   z  z         S )Nr   r   r?   r  )r  r   r  r  s    rT   rH   ziota.<locals>.fn	  s!    ~~dU1Xo5UCCrV   r  )r0   r   r7   )lengthr  r  r   r  requires_gradrH   s    ```   rT   iotar  	  s1    D V$x	 rV   r   r  c           	      
   | j                         |j                         k(  sJ | j                         t        | d      t        j                  j
                  j                  t        j                  d            r| j                            z   t        j                  j
                  j                  d       t        j                  j
                  j                  | j                                   t        t        |      | j                               }|j                         fd}t        j                  | j!                         | j                         |t#        | j                                     S )Nr   c           	          t        j                  t        j                  t        j                  |    t        j
                        t        j                  t        j
                               |        |             S r]   )r?   r  eqr  rL   r  )r  r   r  
src_loaderr  s    rT   r  z select_scatter.<locals>.inner_fn	  sX    yyFFs3x5uekk2 sOSM
 	
rV   r  )r   r  r  r@   r   r   r  r   r  r   r  r  r   r  r0   r   r   rn   )rv   r  r   r  r  r  r  s     `` @@rT   select_scatterr  	  s   ;;=CMMO+++}}H
3
"Cww%%ehhua&89

S))GGq%(GGeQZZ\#%67
3$ajjl
3C"J
 ||~kkmAJJL!	 rV   c           	      L   	
  j                         |j                         k(  sJ  j                         
t         d       j                            t        j
                  j                         \  t         j                               }t        z
  dz
  z         |<   t        ||      }|j                         		 
fd}t        j                   j                          j                         |t         j                                     S )Nr   r#   c                    	dk(  rk(  r
dk(  r |       S t        j                  |    t        j                        }t	        |       t        |    	z
  
      <   g }	dk7  rZ|j                  t        j                  |t        j                  t        j                  	      t        j                                     k7  rZ|j                  t        j                  |t        j                  t        j                        t        j                                     
dk7  rz|j                  t        j                  t        j                  t        |    	z
  d
      t        j                        t        j                  dt        j                                     |sJ t        j                  t         j                   |      }t        j"                  |fdt%              rdnd      }t        j&                  || |             S )Nr   r#   c                              S r]   r   )src_idxr  s   rT   r   z1slice_scatter.<locals>.inner_fn.<locals>.<lambda>	  s    Jw' rV   r  )r?   r  rL   r  rn   r   r   r  r   r   r  r  r    r  r   r  r  r  r   r  )r  r	  r  src_valr  r   r  r  r  r  r  rv   r  s       @rT   r  zslice_scatter.<locals>.inner_fn	  s   A:#/daic?"..S5;;7s)C5 0$7A:KKNN5<<#6D (?KKNN5<<#4ekkB 19KKNN'C5(8!TBEKK LLEKK0	 t$/**' #A

 yySM
 	
rV   r  )r   r  r  r   r&   r  normalize_start_endrn   r   r   r0   r   r   )rv   r  r   r  r  r  src_sizer  r  r  r  s   ` ````  @@@rT   slice_scatterr  	  s    ;;=CMMO+++}}H
3
"Czz|C H11!S%EJE3AJJL!HS5[D1H5t<HSM
h
C"J,
 ,
\ ||~kkmAJJL!	 rV   c                 j    t        | t        t        f      rt        |       dkD  rt	        | d         S | S r   )rW   rn   ro   r   _unwrapr   s    rT   r  r  	  s-    !dE]#A
qt}HrV   r   r  r  
pin_memoryc                    t        |d t        j                  fv d|        t        | d       t        t	               t
              rxs t        j                  nxs t        j                         g }t         t        j                        r fd}nt         t        t
        f      r fd}nt               dk(  s't         d   t        t
        f      rBt               dk  r4|j                  t        j                  t                             fd}n5t        j                  j!                  t        j"                   |            S t%        j&                  t)        |      ||	      S )
Nlayout=r  c                 0    t        j                        S r]   r  r  r/  r   s    rT   r  ztensor.<locals>.inner_fn
  s    >>$..rV   c                 0    t        j                        S r]   r?   r  r  s    rT   r  ztensor.<locals>.inner_fn
  s    <<e,,rV   r   r   c                       fdt              dk(  rt        j                  d      S  dt                    S )Nc           	      j   | |k  sJ || z
  dk(  rt        j                  |          S || z
  dz  | z   }t        j                  t        j                  t        j                  d   t
        j                        t        j                  |t
        j                               | |       ||            S )Nr#   r!   r   )r?   r  r  r  r  rL   r  )r  r  midbinary_searchr/  r   r  s      rT   r  z/tensor.<locals>.inner_fn.<locals>.binary_search
  s    s{"{;!#<<UU;;U{q(50yyFFuQx=S%++6 "%-!#s+ rV   r   )r   r?   r  )r  r  r/  r   s   `@rT   r  ztensor.<locals>.inner_fn
  s7     4yA~||Au-- CI..rV   r  r  )rg   rL   stridedrW   r  r   r  get_default_dtyper   r   r   r   r   r   r@   r   add_tensor_constantr  r0   r   r7   )r/  r   r  r  r  r  r  s   ``     rT   r  r  
  s   v$..'&0BC:~|,'$-%$2002!F$$	/ 
D5#,	'	- 
Ta:d1gs|<TaemmCI./	/( ww**LLU6:
 	
 V$	 rV   c                 z    t        | t              r|t        | |      } |t        | |      } | S t	        | ||      S )Nr  )rW   r3   r   rs  r  )r/  r   r  s      rT   	as_tensorr  >
  sD    $	"D%(DT6*D$eF33rV   c                 8    t        | t        j                        S r  r  )r/  s    rT   long_tensorr  I
  s    $ekk**rV   c                    ddl m}  |t        j                  j                  j
                  t        j                  j                  j                  d         }t        |      dk(  sJ |       t        t        |j                                     \  }}t        j                  |||       }t        j                  j                  |      |_        t        j                  j!                  |       t        j                  j                  j                  d   }t#        |t$        j&                  t$        j(                  t$        j*                  f      r|j,                  j.                  S t1        j2                  |      S )Nr   )resolve_unbacked_bindingsunbacked_bindingsr#   r  )r  r  r@   r   r   r  r	  r   r   r   iterr8  r&   DynamicScalarregister_bufferr   register_operationrW   rL   SymIntSymFloatSymBoolr@  exprr   r  )r/  r  r  binding_symkeypathbufferr  s          rT   _local_scalar_denser  N
  s   O 2	""AGG$8$8$=$=>Q$R  !Q&9(99&   %6%<%<%> ?@Kk7D9F''))&1FKGGv& ''


#
#E
*C#ennemmDExx}}}}S!!rV   c                      y r]   r   )r/  rf   s     rT   _assert_scalarr  |
  s     rV   c                    | t        | t        t        f      st        d      rj                  t        t        t        f      rfd}nTt        t
        j                        rfd}n3t        j                               dk(  sJ j                         fd}t        j                  ||t        |            S )Nr   c                 0    t        j                        S r]   r  r  r   r   s    rT   r  z_full.<locals>.inner_fn
  s    <<u--rV   c                 0    t        j                        S r]   r  r  s    rT   r  z_full.<locals>.inner_fn
  s    >>%//rV   r   c                      g       S r]   r   )r  value_loaders    rT   r  z_full.<locals>.inner_fn
  s    ##rV   r  )rW   r   r   r   r   r   r   r   r   r  r0   r   rn   )
fill_valuer  r   r  r  r   r  s     `  @@rT   _fullr  
  s    Ej3,/GE74K%#u&	. 
E5;;	'	0 5>>#$)))((*	$ Dz	 rV   c                 8     t        t        |            | fi |S r]   create_tensor_liketensor_constructor)rv   r  r   s      rT   r  r  
  s     =0<=aJ6JJrV   c                 "     d d d d dd d fd
}|S )NF)namesr   r  r  r  r  c                    t        | d u d       t        |d t        j                  fv d|        t        | d       t        |      }|xs t        j                         }t        |      dk(  r6t        |d   t        t        t        j                  f      rt        |d         }|D ]  }t        |t        j                        sJ  |D cg c]  }t        j                  |       }}t        |||      S c c}w )Nnamed tensorsr  r  r#   r   )rg   rL   r  r7   r  r   rW   rn   ro   Sizer  r   r   r  )	r  r   r  r  r  r  r  r  r  s	           rT   r(  z!tensor_constructor.<locals>.inner
  s     	5D=/26dEMM22gfX4FGz><0v&2002t9>ja4

2KLa>D  	3A!!U\\222	3)-.AQ..Z55 /s   C8r   )r  r(  s   ` rT   r  r  
  s#     6. LrV   )r  r   r  r  r  r  c                     t        | d u d       t        |      }t        |      dk(  r6t        |d   t        t
        t        j                  f      rt        |d         }t        |d ||||      S )Nr  r#   r   r   r  r  r  )	rg   r7   r   rW   rn   ro   rL   r  empty_strided)r  r   r  r  r  r  r  s          rT   r  r  
  sf     u}o.6"F
4yA~*T!WtUEJJ.GHT!W~d%v* rV   c                       dddddd fd
}|S )zZ
    Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
    NF)r   r  r  r  r  c                   t        | d       t        |d t        j                  fv d|        || j                         }nt	        |      }|xs | j                         }t        | j                               } |||||      S )Nr  r  r  )rg   rL   r  r   r   r   rn   r   )rv   r   r  r  r  r  r  creation_fns          rT   _constant_likez*create_tensor_like.<locals>._constant_like
  s     	z><06dEMM22gfX4FG=KKME 'E)1<<>AJJL!fV

 	
rV   r   )r  r  s   ` rT   r  r  
  s     duTX
 rV   c                 *    t        t        |             S r]   r  )r  s    rT   r  r  
  s    0<==rV   c                      d d d d d fd
}|S )Nr  c                \   t        |t        t        f      sJ t        | d       t        |d t        j
                  fv d|        t        |      xs | j                         }|xs | j                         }|D cg c]  }t        j                  |       }}t        |||      S c c}w )Nr  r  )rW   rn   ro   rg   rL   r  r   r   r   r   r   r  )rv   r  r   r  r  r  r  r  s          rT   _new_constantz#new_constant.<locals>._new_constant
  s     $u...z><06dEMM22gfX4FGU#4q{{})1<<>*./Qa //Z55 0s   =B)r   )r  r  s   ` rT   new_constantr  
  s    tDT	6 rV   r  c                l    || j                         }|| j                         }t        |d ||||      S Nr  r   r   r  )rv   r  r   r  r  r  s         rT   	new_emptyr    s<    }~d%v* rV   c                X   t        | t        t        f      sJ t        |t        t        t        d       f      sJ t	        | d       t	        |d t
        j                  fv d|        t        |      xs t        j                         }|xs t        j                  d      j                  }t        d|||       }|j                          |j                  j                  }dgt        |       z  |j                  _        t        |t         j"                        sJ | D cg c]  }t%        j&                  |       } }|r#|D cg c]  }t%        j&                  |       c}nt         j(                  j+                  |       }t!        j,                  ||| |      |_        |S c c}w c c}w )Nr  r  r  r   )r  r  r   r  )r  r   r  r  )rW   rn   ro   r  rg   rL   r  r   r  r  r  r  r<  r/  r   r  r&   rY  r   r   r  r  r  r  )	r  r  r   r  r  r  	pointwiser  r  s	            rT   r  r    se    dT5M***ftUDJ7888:~|,v$..'&0BC<5#:#:#<E/u||C(//F6TJI^^  Fs4yFKKfb//000%)*ELLO*D*  #))Qa)11$7 
 NN	FM  +)s    F"F'c                l    || j                         }|| j                         }t        ||||||      S r  r  )rv   r  r  r   r  r  r  s          rT   new_empty_stridedr  0  s>     }~fE&J rV   c                 
   |D cg c]+  }t         j                  j                  j                  |      - }}t	        t        t        |            |j                        }t        j                  j                  | |      S c c}w )N)key)r@   r   r   r  sortedr   r   __getitem__r&   r/  r0  )rv   r  r  r2  s       rT   copy_stridedr  =  sa    5;<agg((+<F<%F,&2D2DEL??//<@@ =s   0B c                 V    |j                  d      J d        t        |      | fi |S )Nr   z(dtype should be handled by decomposition)r!  r  )r  r  r   s      rT   fullr  D  s5    ::g*V,VV*)j)$9&99rV   c                    t        | t              sJ |j                         dk(  rt        | |j	                               S |j                         t        j                  k(  sJ | j	                         t              dk(  }t        | |      |rt        | dg      } dg| j                         |j                         fd}t        j                  | j                         | j                         ||j	                               S )Nr   r#   c                     t        |       } t        j                   |                }t        |       dk(  r|g} n|| <    |       S r   )rn   r?   indirect_indexingr   )r  
gather_idxr   index_loaderr  r  s     rT   rH   zgather.<locals>.fn_  sL    3i**<+<d3iH
s8q=,C!CH}rV   r  )rW   r3   	get_numelr  r   r   rL   r  r   r  r   r  r0   r   r   )	rv   r   r  sparse_gradr  rH   r  r  r  s	    `    @@@rT   gatherr  J  s     a###AENN,--??+++::<DY!^F
3
'C1qcNs}}H$$&L ||~kkm~~	 rV   c                   	
 |rJ t        | t              sJ t        |t              sJ dt        |j                               v sJ | j	                         	|j	                         t        |j                               | j                         
g |j                         
dd  	
fd}t        j                  | j                         | j                         |      S )Nr   r#   c                     t        |       t              k(  sJ |  d         | d        }t        j                  |d         gg | d  z   } |      S )Nz != r   )r   r?   r  )r  	var_index
weight_idxindices_loaderindices_ndimr  weight_loaderweight_sizes      rT   rH   zembedding.<locals>.fn}  s{    3x3x=(@SEhZ*@@("3}#56	++I{1~FG K
K
 

 Z((rV   r  )
rW   r3   strr   r  r   r   r0   r   r   )weightr   padding_idxscale_grad_by_freqsparserH   r  r  r  r  r  s         @@@@@rT   	embeddingr  p  s    :fi(((gy)))C))+,,,,&&(M((*Nw'')*L//#K6!!#6k!"o6H) )   " 	 rV   c           
         t        d | D              s)J d| D cg c]  }||j                          c}        t        d | D              rt        d      t	        |       D cg c]  \  }}t        |t              s| }}}t        |      dkD  sJ d       d gt        |       z  }t        |t        |D cg c]  }| |   	 c}       D ](  \  }}|j                         |k7  rt        d      |||<   * ||fS c c}w c c}}w c c}w )Nc              3      K   | ]T  }|P|j                         t        j                  t        j                  t        j                  t        j
                  fv  V y wr]   )r   rL   r  r  r   rR  rj   r   s     rT   rl   z.check_and_broadcast_indices.<locals>.<genexpr>  sA      = 	
%++u{{EJJLLs   AAz)indices must be int64, byte or bool. Got c              3      K   | ]7  }||j                         t        j                  t        j                  fv  9 y wr]   )r   rL   r   rR  r  s     rT   rl   z.check_and_broadcast_indices.<locals>.<genexpr>  s1      78q}%**ekk22s   ?5?zFallback for bool indicesr   z"requires at least 1 non-None indexz.Fallback when indices is on a different device)r   r   r   rd   r   rW   r3   r   r   r   r   )r   r  r   rv   
valid_idxsnew_indicess         rT   check_and_broadcast_indicesr	    s1      g 
373daVWVcAKKM3d2ef	g 
  <C  ""=>> )' 2O1jI6N!OJOz?QD DD&3w<'KJ 1
3S1GAJ3S TU 1 <<>V#%&VWWA 
""! 4e P 4Ts   C=C='D D:Dc           	      D   dt        dd        D ]  \  }}	|	|z
  dk7  sd t              D 
cg c]  \  }
}|	| |
    c}}
g | t              t              z   d  d   }rz   nd | z   |d  z   f	d}|fS c c}}
w )NFr#   Tr   c                   	 t        |       t              k(  sJ t              t        
      k(  sJ t              }g }d   }rdn|}d}t        d   dz         D ]|  }||k(  r||z  }|   *|t        |       k  sJ |j                  | |          |dz  }<|   }|J 
|   }|j                  t        j                   || |||z          |	             ~ g || |d  }|S  |      S )Nr   r  r#   r   )r   r   r   r?   r  )r  r  	new_indexfirst_tensor_indexstart_offsetnext_idxr   loaderr  r   indexed_sizer   indices_loadersnon_consecutive_tensorsoutput_sizetensor_indicestensor_sizer  s            rT   rH   z*index_output_size_and_inner_fn.<locals>.fn  sK   3x3{++++?#s<'8888;	+A.3q9K~b)A-. 	AL D qz!#c(***  X/A(+)))#A  ))s<,2EFG#	$

^
	 %,yE(92EErV   )r   r   r   )r  r   r  r  r  r  r  r   previouscurrentr   r  r  rH   r  r  s    ```````      @@rT   index_output_size_and_inner_fnr    s    " $ 1CD +'X"&*#+ ,5W+=MC6!9MKSKS&[)9C<O)O)Q"RSK'*!K/ ++,,-./ 	F FB ?] Ns   
BBc                     t        | ||      \  }}}t        j                  | j                         | j	                         ||      S r  )index_impl_helperr0   r   r   r   )rv   r   r   r  r  r   s         rT   
index_implr    sB    0GUCK1||~kkm	 rV   c           
         t        |t        t        f      sJ | j                         t	        || j                               \  }}t        |      dkD  sJ d       |D cg c]  }||j                         nd  }}t        ||d      j                               }| j                         }t        t        |            D cg c]  }||   	||    }}|rd|v rd|vrt        d      t        t        |            D cg c]  }||   	 }}t        ||||||d |      \  }	fd}
|	|
fS c c}w c c}w c c}w )Nr   z Must have at least one valid idxz0index is out of bounds for dimension with size 0r  c                        |             S r]   r   )r  index_inner_fnr  s    rT   r  z#index_impl_helper.<locals>.inner_fn  s    s+,,rV   )rW   rn   ro   r  r	  r   r   r   r   
IndexErrorr  )rv   r   r   r  r   r  r  r  r  r  r  r   r  s              @@rT   r  r    sP   ge}---}}H9'1<<>RG^~"F$FF"KRSa!-q}}TASOS w~a01::<=KZZ\F',S\':U!gaj>TF1IULUl"q';KLL',S\':;!F1I;L;"@	#K- .007 T V <s   !D1
D6D6D;c                     	 t        | |d      S # t        $ r? | j                           t        t        j
                  j                  d      | |      cY S w xY w)NTr  FrU  )r  rd   r<  r[  rh  r  r  rv   r   s     rT   r  r    sT    
!WD11 
			M

 1 1uMw
 	

s    AAAc                     t        | |d      S NFr  )r  r#  s     rT   _unsafe_indexr&  '  s    a..rV   c                 0    t        t        |       |||      S r]   )
index_put_rQ  rv   r   r   
accumulates       rT   	index_putr+  4  s    eAh<<rV   c                 4    t        t        |       |||d      S r%  )index_put_impl_rQ  r)  s       rT   _unsafe_index_putr.  9  s    58WfjNNrV   c                     |j                         | j                         k7  rt        || j                               }|rt        | |      }t        | t	        |d   ||             S r   )r   rs  rr   r  r  )r~  r   r   r*  s       rT   index_put_as_masked_fillr0  >  sV    T__..%!23D% T5UD9::rV   c                 r   t        j                         }t        |      r\|s|rX|sdnd}t        j                  j
                  j                  j                  dd       x}r| d| }|t        j                  _        t        j                  t        j                  j
                  j                  | |||       | S )Nzindex put with accumulate.zdeterministic index put.stack_trace Found from : 
 )rL   $are_deterministic_algorithms_enabledr-   r@   r   r	  r   r!  disable_cudagraphs_reasonr&   IndexPutFallbackr7  )r~  r   r   r*  deterministicrf   r2  s          rT   index_put_fallbackr8  F  s    >>@MjM ! )+ 	
 ''..3377tLL;LE*;-8C,/),,33T7FJWKrV   c                 "    t        | |||d      S )NTr  r-  r~  r   r   r*  s       rT   r(  r(  V  s    4&*DIIrV   c                 "    t        | |||d      S r%  r:  r;  s       rT   _unsafe_index_put_r=  [  s    4&*EJJrV   c           
         |j                         dk(  rt        |      dk(  r|d   j                         t        j                  t        j
                  hv r_|d   }t        t        |j                               t        | j                                     D ]  }t        |d      } t        | |g||      S t        j                         rt        | |||      S |D ]E  }||j                         t        j                  t        j
                  hv s7t        | |||      c S  | j                         }t        |      }	|rLt        | j                               r3|	dk(  rt        | dg      } t        | |||      } |	dk(  rt        | g       } | S t        || j                               }	 t        || j!                               \  }}
|D cg c]  }||j%                         nd  }}t'        | t(              sJ | j+                          |	dk(  rt        | dg      } t-        ||
d      j                               }t        t        |            D cg c]  }||   	 }}t/        |||
|||d |      \  }}t1        ||      }t3        j4                  | j!                         | j                         |j%                         |||rdnd       }t3        j6                  d t3        j8                  |       |      }t:        j<                  j?                  |      |_         t:        j<                  jC                  |       |	dk(  rt        | g       } | S # t"        $ r t        | |||      cY S w xY wc c}w c c}w )Nr#   r   r  r  
atomic_addr  r   r  r  output_indexerscatter_mode)"r  r   r   rL   r   rR  r   r   r  r0  r4  r8  r;   ri  r   r	  r   rd   r  rW   r3   r<  rn   r  r   r&   ScatterrY  MutationLayoutSHOULDREMOVEr@   r   r  r   r  )r~  r   r   r*  r   r  r   r  r  x_ndimr  r   r  r  r  expected_vals_sizer  scatterr  s                      rT   r-  r-  `  s0    	aLAAJ  "uzz5;;&??qzs4==?+S-AB 	'AT2&D	''tffjII 113!$DD  I!2uzz5;;6O!O%dGVZHHI ]]_F[FB4>>CSTQ;qc?D!$DQ;b>Dfdnn./FE"=T__&#
 LSSa!-q}}TASOSdI&&&LLN {D1# w~a01::<=K',S\':;!F1I;L;#A	$  F./F jj nn##%!%/\TG 
%%d+F
 ''))&1FKGGv&{D"~Ka  E!$DDE T <s   $M M!M&MMrU  c                    	 t        | |d      \  }}j                         | j                         		fd}t        j                  | j	                         | j                         ||      S )NFr  c                      j                   t        j                  k7  r+t        j                          t        j                        }n        }t        j
                  | fd      S )Nc                                      S r]   r   )_unsafe_index_fnr  self_loaders   rT   r   z8_unsafe_masked_index.<locals>.inner_fn.<locals>.<lambda>  s    K8H8M,N rV   )r   rL   r   r?   r   r  )r  mask_valrK  fillr  mask_loaderrL  s   ` rT   r  z&_unsafe_masked_index.<locals>.inner_fn  sK    ::#||K$4ejjAH"3'Hzz($NPTUUrV   r  )r  r  r0   r   r   r   )
r~  r  r   rN  r  r   r  rK  rO  rL  s
    ` `   @@@rT   _unsafe_masked_indexrP    sp    "3D'"OFA""$K""$KV V  nn	 rV   c           	          t        ||d      }| j                         }t        t        |            D cg c]#  }||   rt	        ||   ||    ||   dz
        nd % }}t        | ||d      S c c}w )Nr   r#   T)r*  )r  r   r   r   r  r.  )rv   r  r   r   masked_valuer  r   clamped_indicess           rT   #_unsafe_masked_index_put_accumulaterT    s    vq)LJJLE s7|$ 7>ajgaj58)U1X\2dJO  Q$OOs   (A-c                 V    t        j                  |t        j                  ||             S r]   )r?   r"  r#  r   minmaxs      rT   r  r    s    ;;sCKKQ/00rV   c                 P    t        |       }t        ||||      }t        ||       |S r]   )rQ  r  copy_)r~  r  r  r  r  r   output_views          rT   as_strided_scatterr\    s*    4[FVT6>BK	+sMrV   c                 2    t        t        |       |||fi |S r]   )scatter_rQ  )rv   r   r  r  r   s        rT   rG  rG    s    E!Hc5#888rV   r  include_selfr  r  r`  c          	      
   t        |t              }t        | ||j                         |r|j                         n
t	        |      |r|j                         j                  nd|      rt        j                  | ||||||       |S y )Nznot implr_  )rW   r3   r>   r   r  r   r&   ScatterFallback)r  r~  r   r  r  r  r`  src_is_tensors           rT   scatter_fallbackrd    s     sI.M(d3i!.J 	%	
 rV   r  c                   |dv sJ |[t        t        j                  t        j                  j
                  j                  j                        }t        || ||||      }||S |dk(  rd}n|dk(  rd}t        | ||||      S )N>   Nrr   multiplyre  rr   sumrg  prod)
r_   rh  r^  r@   r   r	  r7  _overloadnamerd  scatter_reduce_)r~  r   r  r  r  r  fallback_results          rT   r^  r^    s    ....~dmmQWW-A-A-H-H-V-VW*sE3v
 &""	:	4eS&99rV   c                 0    t        t        |       |||      S r]   )scatter_add_rQ  rv   r   r  r  s       rT   scatter_addrp  +  s    a#uc22rV   c                      t        | |||d      S )Nrh  )rk  ro  s       rT   rn  rn  0  s    1c5#u55rV   c                 4    t        t        |       ||||fi |S r]   )rk  rQ  )rv   r   r  r  reduction_typer   s         rT   scatter_reducert  5  s    58S%nOOOrV   )r`  c          
          |dv sJ t        t        j                  j                               dk(  r dt        j                  j                         v sJ d       t	        t
              rt               t        t        j                  j                   |||      }|r|S t	         t              sJ dt        |j                               v sJ t         j                               }|dk(  rt         dg       t	        t              r)t        j                               dk(  rt        dg      t	        |t              r)t        |j                               dk(  rt        |dg      }|j                         dk(  r S t                j!                          |j#                         t	        t              rj#                         nd  fd} fd	}	d
 }
|st%        j&                   j)                          j                          fd|j                         |d       }t%        j*                  d t%        j,                         |      }t.        j0                  j3                  |      |_        t.        j0                  j7                  |       t%        j&                   j)                          j                         |	|j                         | |
|            }t%        j*                  d t%        j,                         |      }t.        j0                  j3                  |      |_        t.        j0                  j7                  |       |dk(  rt         g         S )N>   Nrh  amaxaminmeanri  r#   twozKaten.scatter_reduce_.two is not the unique overload of aten.scatter_reduce_r_  r   r   c                     j                         }t        |      }t        |       }t        j                   |       |dk(  rdn|   d      |<   |S )Nr   r#   F)wrap_neg)r   r   rn   r?   r  )r  r  r  indirect_idxr   r  r~  s       rT   rA  z'scatter_reduce_.<locals>.output_indexerh  sU    5zCy11DAIq5:
S rV   c                 `    r |       S t        j                  j                               S r]   r?   r  r   )r  r~  r  r  s    rT   rH   zscatter_reduce_.<locals>.fnr  s*    c?" <<T^^%566rV   c                     | dk(  ry| J y )Nrh  r?  r   re  s    rT   backend_reduce_strz+scatter_reduce_.<locals>.backend_reduce_stry  s    U? >!>rV   c                 L    t        j                  dj                               S r   r~  )r  r~  s    rT   r   z!scatter_reduce_.<locals>.<lambda>  s    3<<4>>3C#D rV   r@  )r   rh  rk  ru   rW   r   r  rd  ry  r3   r  r   r   ri  r  r  r<  r  r&   rC  r   rY  rD  r@   r   r  r   r  )r~  r   r  r  r  r`  rl  r  rA  rH   r  zero_outr  rG  r  r  s   `` `          @@rT   rk  rk  :  s   BBBBD  **,-2T))3355U UU	6 #vc"&  !O dI&&&C)****t}}DqyD1##y!c#,,.&9Q&>3n%#ENN,<(=(BUQC A
c
"CLLN$$&L&0i&@"dJ7 ::??$.."D>>#)
 ""))$/

 gg--f5	""6*
 jj nn~~%'/G 
%%d+F
 ''))&1FKGGv&qyD"~KrV   scales_xnexactc                 T   | j                          | j                         | j                          d  | j                         d   }D cg c]+  }t        j                  j
                  j                  |      - c}t        |      k(  sJ |}t        |      D cg c]
  \  }}||z   c}}t        |      D ]  \  }}	|		d|	z  |<    fdfd}
t        j                  | j                         | j                         |
g ||      S c c}w c c}}w )Nr9  c                    t        j                  | t        j                        } r8t        j                  | t        j
                  dt        j                              } t        j                  | t        j
                  |t        j                              } t        j                  | t        j                        } t        j                  | |d      S )N      ?Fr  )
r?   r  rL   r   rr   r  r  r   r  r  )rv   r%  r  r  s      rT   scale_fnz$upsample_nearestnd.<locals>.scale_fn  s     NN1emm,3<<U]];<AGGAs||E5==9:LLEKK($$QE::rV   c                     |  d  }| d   } 
g |t        |      D cg c]  \  }}} 	|||       c}}}      S c c}}}w r]   )r   )r  rv   r   r   r  r  i_sizes
inv_scalesr  r  r  s         rT   rH   zupsample_nearestnd.<locals>.fn  s`    H!HWaW#aW:UVVJAq$8Aq$'VW
 	
Vs   Ar  )realize_hintr  r   r@   r   r   r  r   r   r   r0   r   r   r   )rv   r  r  r  r  batchr   o_sizesor%  rH   r  r  r  r  s      ``      @@@@rT   upsample_nearestndr    s    NN}}HjjlA23GJJL1"EBIJQqww55a8JGx=AG$'$9:DAq!a%:Jh' (5%KJqM(	;
 
 ||~kkm!!!	 9 K
 ;s   0D'D$c                 "    t        | ||fd      S )Nr#   r  r  rv   r  r  s      rT   upsample_nearest1dr    s    avi1==rV   c                 $    t        | ||fdd      S )Nr#   Tr  r  r  r  s      rT   _upsample_nearest_exact1dr    s    avi1DIIrV   scales_hscales_wc                 $    t        | |||fd      S )Nr!   r  r  rv   r  r  r  s       rT   upsample_nearest2dr    s     ax.BaHHrV   c                 &    t        | |||fdd      S )Nr!   Tr  r  r  s       rT   _upsample_nearest_exact2dr    s     ax.BatTTrV   scales_dc                 &    t        | ||||fd      S )Nrz   r  r  rv   r  r  r  r  s        rT   upsample_nearest3dr    s     ax8.LPQRRrV   c                 (    t        | ||||fdd      S )Nrz   Tr  r  r  s        rT   _upsample_nearest_exact3dr    s#     	;8X6!4 rV   c                 ,     t         fd|D              S )Nc              3   J   K   | ]  }t        j                  |        y wr]   r  )rj   r   r   s     rT   rl   z$_create_constants.<locals>.<genexpr>  s     6Aa'6s    #)ro   )r   r   s   ` rT   r  r  
  s    6666rV   c                     | j                         | j                         fd}t        j                  | j	                         | j                         |      S )Nc                     t        |       } t        |       t              k(  sJ D ]  }|   dz
  | |   z
  | |<     |       S r  )rn   r   )r  r   r  r  r  s     rT   r  zrev.<locals>.loader  sW    3i3x3u:%%% 	3Cc
Q#c(2CH	3 }rV   r  )r  r   r0   r   r   r   )rv   r  r  r  r  s    ` @@rT   revr    sM     }}HJJLE ||~kkm	 rV   c                    t        |      dz  dk(  sJ t        d |D              rt        |       S | j                         }t	        t        t	        t        |d d d   |dd d                           t        |      t              z
  g D ]?  \  }}j                  t        j                  j                  j                  |      |f       A t	        |d        }g t        |d        D ]C  \  \  }}}	j                  |	       |j                  t        j                  |	|z   |z                E t        |      t        |      k(  sJ  t        | j                                     fdfd}
| j!                         t#        j$                  | j'                         | j                         |
|      S )Nr!   r   c              3   &   K   | ]	  }|d k(    ywrn  r   rj   r  s     rT   rl   z"constant_pad_nd.<locals>.<genexpr>'  s     
#a16
#r   r#   c                 B    g }t         	d        D ]J  \  }\  }}}|dk7  r|j                  t        |d             |dk7  s0|j                  t        ||             L t	        j
                  t        j                  |      }t        j                  | 
fd      S )Nr   c                              S r]   r   )r  r  s   rT   r   z/constant_pad_nd.<locals>.mask.<locals>.<lambda>D  s     rV   )	r   r   range_mask_lowrange_mask_highr   r  r?   r  r  )r  r  r  rD  rE  r  boundsr  
mask_sizesr  r  s   `     rT   r  zconstant_pad_nd.<locals>.mask<  s    (+E!"Ivz(J 	:$C#tfaxN323qyOC89		:
 $/zz$ 7DDrV   c                     t        | d        }t        | d        D ]  \  }\  }}|j                  ||z
          t        |      t        |       k(  sJ  |      S r]   )rn   r   r   r   )r  r  r  rD  rE  bounds_precompr  r  s        rT   	offset_fnz"constant_pad_nd.<locals>.offset_fnF  si    rO	 #E!"I~ > 	(C#tS3Y'	(9~U+++IrV   r  )r   r   rQ  r   rn   r   r   r   r@   r   r   lookup_precomputed_sizer   r   r   r   r  r0   r   r   )rv   paddingr  r  lhr  rD  rE  r  r  r  r  r  r  r  r  s     `        @@@@@@rT   rp  rp  $  s   L1"""

#7
##QxJJLE(4GCaCL'!$Q$- @ABCFE
S[ A 68N P1qww//GGJANOP uRay/KJ qr3 <dT$5<<s
T(9:;< {s5z)))-q{{}-j9JE E }}H||~kkm	 rV   r   c                     t        j                  t        j                  | t        j                        t        j                  t        j                  |      t        j                              S r]   )r?   r  r  rL   r  r   r   )r   rD  s     rT   r  r  V  s@    66q%++&u}}S)5;;7 rV   c                     t        j                  t        j                  | t        j                        t        j                  |t        j                              S r]   )r?   r  r  rL   r  )r   rE  s     rT   r  r  ]  s7    66q%++&tU[[) rV   c                 V    t        j                  t        | |      t        | |            S r]   )r?   r  r  r  )r   rE  rD  s      rT   
range_maskr  d  s'    88q#4  rV   c                       j                          d   j                         xs dgz   fd}|S )Nr   c                 N   | d   |  d  t        j                  t        j                  t	              D cg c]  }t        |   |   
|   z   
|          ! c}      }	rt        j                  |fd      S t        j                  |fd      S c c}w )Nc                  4     t               g       S )Nr   )constant_boundary_condition)r   ihpad_fill_valueprefixrv   s   rT   r   z;constant_boundary_condition.<locals>.load.<locals>.<lambda>}  s$    O3A~3O"fNrN rV   c                       g        S r]   r   )r  r  r  s   rT   r   z;constant_boundary_condition.<locals>.load.<locals>.<lambda>  s    (>V>b>*B rV   )r   r  r?   r  r   r  r  )r  r   r  r  r  r   r  r  r  r  	padding_hrv   r  s      @@rT   r  z)constant_boundary_condition.<locals>.loadr  s    uC45\HHLQRUJWqZ1qtil2Yq\MBW
  JJ 
	
 D"BJO
	
 Xs   $B"
r   r  )	rv   r  r  r  r   r  r  r  r  s	   ````` @@@rT   r  r  k  sE     	


cTUA}}H$A39I
 
( KrV   c                 x   t        | d||   z  z   ||   dz
  z
  ||   dz
  z   ||         }|r
t        | d||   z  z   ||   dz
  z
  d||   dz
  z  z   ||         }t        j                  j                  j	                  |dz
  ||   z  | z
  ||   z
        dk\  r>|dz  }t        j                  j                  j                  d|||   z  | z
  ||   z
         t        j                  j                  j	                  ||z
        dk(  r0t        j                  j                  j                  ||       d}||fS |}||fS )Nr!   r#   r   F)r   r@   r   r   r  r  r   )rv   r   kernel_sizer  r  	ceil_modex_outx_alts           rT   pooling_sizer    sW   	A
Nk!nq01VAY]CVAYE GAJ+a.1"45VAY]8KKVTUY
 77%%uqyF1I&=&AGAJ&NOSTTQJEGG&&q%&)*;a*?'!**LM77%%eem49GG))%7I ) E)rV   c                 b    t        | d      } | d   | d   z  }|dkD  xs t        d |D              S )Nr!   r   r#      c              3   &   K   | ]	  }|d kD    ywr  r   r  s     rT   rl   z:should_fallback_max_pool2d_with_indices.<locals>.<genexpr>  s     $=qQU$=r   )r<   r   )r  dilationwindow_sizes      rT   'should_fallback_max_pool2d_with_indicesr    s;    {A.Ka.;q>1K"=$=H$=!==rV   assert_fallbackc                   |dk(  rddg}|dk(  rddg}|s|}t        |d      }t        |d      }t        |d      }t        |d      }t        | t              sJ t        |      dk(  sJ t        |      dk(  sJ t        |      dk(  sJ t        |      dk(  sJ t        | j	                               dv sJ t        ||      }|||k(  sJ |||||fS )Nr   r#   r!   rk  )r<   rW   r3   r   r   r  )rv   r  r  r  r  r  use_fallbacks          rT   max_pool2d_checksr    s     !|a&1}q6{A.K&!$F7A&GHa(Ha###{q   v;!w<1x=Aqzz|&&&:;QL"...<??rV   c                 2   t        | |d      \  }}| j                          | j                         ^ }}}	t        |d|      \  }
}t        |	d|      \  }}| j                  }|t
        j                  u rdn5|j                  rt        d      nt        j                  |      j                  }t        |      |
|gz   }d   s	d   s|s|rt        | |d      n| j                         fd}t        j                  | j!                         | j#                         t%        j&                  |d	      |
      }t        j                  | j!                         t
        j(                  t%        j&                  |d	      |
      }||fS )NFr  r   r#   -infr!   r  c                    | ^ }}}d }d }t        j                  t        d         t        d               D ]  \  }}|d   z  |z   d   z
  }	|d   z  |z   d   z
  }
 g ||	|
      }|r]t        j                  |d   z  |z   t
        j                        }||}n+t        j                  t        j                  ||      ||      }||}t        j                  ||      } |r|S |S r  )
r   productr   r?   r  rL   rQ  r  gtr"  )r  return_indexr  bhbwmaxvalmaxindexh_incw_incr  iwr  r  r  r  r  r  s                rT   rH   z/_low_memory_max_pool2d_with_offsets.<locals>.fn  s   R%--+a.!5Q#8
 	2LE5 fQi%''!*4BfQi%''!*4B,V,R,,-Cu{1~'='EuzzR#$H"yyV)<eXNH~S&1	2  OMrV   r  r  T)r  r  r   r  r   rL   r   rc  r   rf  rW  rn   r  r  r0   r   r   r   r   r   rQ  )rv   r  r  r  r  r  r   r  r  wh_out
ceil_mode1w_out
ceil_mode2r   	min_valuer  rH   r   offsetsr  s    ```                @rT   #_low_memory_max_pool2d_with_offsetsr    ss    1B	;51-K(A NN::<LUAq$Q;SE:$Q;SE:GGE EJJ 	$66eFmEKK<N<R<R  E{eU^+HqzWQZ:.q)C==?2 

||~kkm""2E:	C ||~jj""2D9	G <rV   c                     | j                         fdfd}t        j                  | j                         t        j
                  || j                               }|S )Nc                 0   t        j                  	t        j                        }t        j                  |d   z  
d   z
  t        j                        }t        j                  |d   z  
d   z
  t        j                        }|| z   }||z   }||z  |z   S r  )r?   r  rL   r  )r  r  r  r  w_inhbasewbaser  r  input_widthr  r  s            rT   increments_to_indexzF_low_memory_max_pool2d_offsets_to_indices.<locals>.increments_to_index  s    ~~k5;;7rF1I~
:EKKHrF1I~
:EKKHU]U]Dy2~rV   c                     | ^ }}} 
g |||      }t        j                  	t        j                        }||z  }|||z  z
  } ||||      S r]   r?   r  rL   r  )r  r  r  r  r  kw_constr  r  r  kernel_widthoffsets_loaders           rT   offsets_to_indiceszE_low_memory_max_pool2d_offsets_to_indices.<locals>.offsets_to_indices  sf    R 1& 1" 1b 12<<ekk:("%(*+"5%R88rV   r  )r  r0   r   r   rL   r  r   )	r  r  r  r  r  r  r   r  r  s	    ````  @@rT   )_low_memory_max_pool2d_offsets_to_indicesr    sX     ((*N9 !!#kk#!	G NrV   c                 P   dk(  rddg|dk(  rddg}st        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |      dk(  sJ t        |j                               dv sJ | j	                          	 | j                         }t        |t              rt        |j                  j                  t              r|j                  j                  }	t        j                  d t        j                  |	j                         |	j                         |	j                               |	      }
|
j                          |
j                         }n	 |j                         }|d uxr |d   dk(  xs |d uxr |d   dk(  }t        d |D              rt!        | ||||      S |j                         ^ }}| j                         ^ }|j#                         | j#                         t%        |j                               }t'        fdt)        d   dz        D              t'        fd	t)        d   dz        D              z  }|d
kD  rt!        | ||||      S |j                         fd}t        j*                  | j                         | j                         ||      }|rt        j,                  j/                  |      S |S # t        $ r d }Y `w xY w# t        $ r d }Y w xY w)Nr   r#   r!   rk  )r  r   r  )r   r  r/  c              3   &   K   | ]	  }|d k7    ywr  r   r  s     rT   rl   z3max_pool2d_with_indices_backward.<locals>.<genexpr>h  s     
$a16
$r   c           
   3   t   K   | ]/  }t        |d    z  t        d |d    z
  d    z        z
  d       1 ywr   r#   NrX  rj   r  r  r  s     rT   rl   z3max_pool2d_with_indices_backward.<locals>.<genexpr>u  F       	ANSQQ%7F1I$EFFJ   58c           
   3   t   K   | ]/  }t        |d    z  t        d|d    z
  d    z        z
  d        1 ywr#   r   Nr  rj   r  r  r  s     rT   rl   z3max_pool2d_with_indices_backward.<locals>.<genexpr>y  r  r  r  c                    | ^ }}}t        j                  |z  |z   t        j                        }|d   z   }|d   z   }t        j                  t	        |d   z
  d   z   d         t        j                        }t        j                  t	        |d   z
  d   z   d         t        j                        }t        j                  t	        |d         dz   t        j                        }t        j                  t	        |d         dz   t        j                        }t        j
                  |t        j                  dt        j                              }t        j
                  |t        j                  dt        j                              }t        j                  |t        j                  t        j                              }t        j                  |t        j                  t        j                              }d }	t              D ]3  }
t              D ]!  }t        j                  |t        j                  |
t        j                              }t        j                  |t        j                  |t        j                              }g |t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d   d      t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d   d      } |      } |      }t        j                  ||      }|	;t        j                  ||t        j                  dt        j                              }	t        j                  t        j                  t        j                   ||      t        j                   ||            |      }t        j                  |t        j                  |	|      |	      }	$ 6 |	J |	S )Nr   r#   Fr  r  r  )r?   r  rL   r  r   r"  r  r#  r   rr   r  r4  r  r  r   r  r  )r  r  r  r  
index_testphstartpwstartphendpwendgradientph_pw_phpw
grad_indexindex_actual	grad_partr   r  grad_loaderh_window_sizer  indices_sizer  r  pooled_heightpooled_widthr  w_window_sizewidths                      rT   rH   z,max_pool2d_with_indices_backward.<locals>.fn  sA   A^^AIM5;;?

N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM' #	WC]+ "WWWWcll3&DEWWWcll3&DE))Bs||Au{{7S(TU$R(# ))Bs||Au{{7S(TU$R(#
  .j9'
3	|Z8#"yyy#,,sEMM*J H 88FF2u-FF2u- D  #yyswwx/KXVHE"W#	WH ###rV   r  )rW   r3   r   r   r  ro  rx  r/  r0   r&   rY  r  r   r   decide_layoutr   )fallback_max_pool2d_with_indices_backwardr  rn   rX  r   r   r/  r{  )grad_outputrv   r  r  r  r  r  r   	gO_strider/  x_bufferx_strideis_channels_lastr  heightr   r  r  rH   r   r  r  r  r  r  r  r  r   s     ```               @@@@@@@@rT    max_pool2d_with_indices_backwardr)  8  s+    !|a&1}q6a###{q   v;!w<1x=Aqzz|&&& **,	
 !YJqvv{{I$Fvv{{$$$$(nn&]]_
 
 	 &&(	||~H !,A!1A 3)A,!"3  
$8
$$8K(Iw
 	
 JJLUFE&1&:&:&<#Q|((*N))+KAJJL!H {1~)* M  {1~)* M
  -/KR8K(Iw
 	
 ##%L9 9v 

%%'##%	C 44S99
}   	(  	H	s$   "L 6L LLL%$L%c                 ^    | j                         ^ }}}| j                         fd}|S )Nc           
         	
 |\  	|\  
|\  }}t        j                  t        j                  t        j                  z   t        j
                        t        j                  |t        j
                              t        j                  t        j                  
	z   t        j
                        t        j                  |t        j
                                    }t        j                  |	 
fd      S )Nc                  ,     g  z   z         S r]   r   )h_start_indexr  r  r  w_start_indexr  s   rT   r   z3pad_adaptive_loader.<locals>.load.<locals>.<lambda>  s&    HNvN}r'9N=2;MNO rV   )r?   r  r  r  rL   r  r  )r  
incrementsstart_indicesend_indicesh_end_indexw_end_indexr  r-  r  r  r.  pad_valr  s   `      @@@@rT   r  z!pad_adaptive_loader.<locals>.load  s    B'4$}#. [xxFF}r15;;?{EKK8 FF}r15;;?{EKK8	
 zzOO
 	
rV   r  )rv   r4  r   r  r  r  r  s    `    @rT   pad_adaptive_loaderr5    s+    zz|HQ1}}H
, KrV   c                     t        j                  | ||      }t        j                  |||      }t        j                  | ||      }t        j                  |||      }	||||	fS )N)out_diminp_dim)r   r   )
start_index	end_indexh_inr  r  r  r-  r2  r.  r3  s
             rT    compute_indices_adaptive_poolingr<    sa    %%k5$OM##IudKK%%k5$OM##IudKK+}kAArV   c                 d    |\  }}|\  }}	t        | |||||	      \  fd}
|
S )Nc                    | ^ }}} |      } |      } |      } |      }d }	t        j                  t        d         t        d               D ]$  \  }
} |||
|g||g||g      }|	|}	 ||	      }	& |	S r  )r   r  r   )r  r  r  r  r  r-  r2  r.  r3  r  r  r  r  h_end_index_fnh_start_index_fnkernel_maxes
pooling_fnw_end_index_fnw_start_index_fns                rT   rH   z _adaptive_pooling_fn.<locals>.fn  s    R(,$R((,$R(''l1o(>lSTo@VW 
	1FBR.k*	C ~#C0
	1 rV   r<  )r9  r:  rA  in_sizes	out_sizesrB  r;  r  r  r  rH   r?  r@  rC  rD  s     `  `     @@@@rT   _adaptive_pooling_fnrH    sP     JD$LE5 	)YdE5	
 . IrV   c                 h   
 |\  }|\  }}t        | ||||      \  

fd}	|	S )Nc                    | ^ }}} |      } |      } |      } |      }d }	d }
t        j                  t        d         t        d               D ]  \  }} ||||g||g||g      }t        j                  ||z   z  |z   |z   t
        j                        }|
|}
n+t        j                  t        j                  ||	      ||
      }
|	|}	| ||	      }	 |
S r  )	r   r  r   r?   r  rL   r  r  r  )r  r  r  r  r  r-  r2  r.  r3  r  r  r  r  r  r  r?  r@  rA  rB  rC  r  rD  s                  rT   rH   z)_adaptive_pooling_fn_with_idx.<locals>.fn.  s   R(,$R((,$R(''l1o(>lSTo@VW 	1FBR.k*	C NN#t+m;b@%++E  99SVVC%8%J~#C0)	1, rV   rE  )r9  r:  rA  rF  rG  rB  r;  r  r  rH   r?  r@  rC  r  rD  s     `  `    @@@@@rT   _adaptive_pooling_fn_with_idxrK    sQ     JD$LE5 	)YdE5	
! !F IrV   c                     t         t              sJ t        |      dk(  sJ  j                           j	                         ^ }}}t
        j                  j                  j                  |      }t
        j                  j                  j                  |      }|\  }}||k(  r||k(  rt               S |dk(  s|dk(  r2g |||}t        | j                          j                               S ||z  dk(  r||z  dk(  r||z  ||z  g}t         |      S t        ||z   dz
  |      }	t        ||z   dz
  |      }
t        |      ||gz   } j                         }|	|
z  }|dkD  rt!         |      S d }d }t#        |||	|
g||g||gt$        j&                        t)        t+                      fd	}t-        j.                   j                         |||
      }|S )Nr!   r   r  r#   r  c                      t        | |z  |      S r]   r   r  r7  r8  s      rT   r9  z)_adaptive_avg_pool2d.<locals>.start_index|      733rV   c                 2    t        | dz   |z  |z   dz
  |      S r  rN  rO  s      rT   r:  z'_adaptive_avg_pool2d.<locals>.end_index  "    g-7!;WEErV   r9  r:  rA  rF  rG  rB  c                 ^    t        j                   | t                     |             S r]   )r?   truedivr5  )r  fn_sumones_loaderrv   s    rT   rH   z _adaptive_avg_pool2d.<locals>.fn  s-    {{3+A./[1I
 	
rV   r  )rW   r3   r   r  r   r@   r   r   r  rQ  r  r   r   
avg_pool2dr6   rn   fallback_adaptive_avg_pool2drH  r?   rr   r5  	ones_liker0   r   )rv   r  r  r;  r  r  r  o_sizer  h_kernel_maxw_kernel_maxr  r   r  r9  r:  rH   rvrV  rW  s   `                 @@rT   _adaptive_avg_pool2dr_  Y  s   a###{q   NNUD$7711$7D7711$7DLE5 u}QxzUaZ'5'%''V1;;=HHe|qTE\Q.u}dem4![))D5L1,u5LD5L1,u5LE{eU^+HKKME-KR+A{;;4F ""L1%.77F &il3K

 
		||~	
B IrV   c                     t         t              sJ t        |      dk(  sJ  j                           j	                         ^ }}}t
        j                  j                  j                  |      }t
        j                  j                  j                  |      }|\  }}|dk(  s|dk(  r\g |||}t        | j                          j                               t        |t        j                   j                               fS ||z  dk(  r^||z  dk(  rV||z  ||z  g}t        |ddg      rt         |      S t!         ||ddgddgd      \  }	}
t#        |
|d   ||ddg      }|	|fS t%        ||z   dz
  |      }t%        ||z   dz
  |      }t'        |      ||gz   } j                         }||z  }|d	kD  rt)         |      S d
 }d }t+        ||||g||g||gt,        j.                        t1        ||||g||g||gt,        j.                         fd} fd}t3        j4                   j                         |||      }t3        j4                   j                         t        j                  ||      }||fS )Nr!   r   r  r#   )r  F)r  r  r  r  )r  r  c                      t        | |z  |      S r]   rN  rO  s      rT   r9  z(adaptive_max_pool2d.<locals>.start_index  rP  rV   c                 2    t        | dz   |z  |z   dz
  |      S r  rN  rO  s      rT   r:  z&adaptive_max_pool2d.<locals>.end_index  rR  rV   rS  c           	      <     | t        t        d                  S Nr  r5  r   )r  inner_func_max_valrv   s    rT   inner_fn_max_valz-adaptive_max_pool2d.<locals>.inner_fn_max_val      !#':1eFm'LMMrV   c           	      <     | t        t        d                  S rd  re  )r  inner_func_max_idxrv   s    rT   inner_fn_max_idxz-adaptive_max_pool2d.<locals>.inner_fn_max_idx  rh  rV   r  )rW   r3   r   r  r   r@   r   r   r  r  r   r   rL   r  r  max_pool2d_with_indicesr  r  r6   rn   fallback_adaptive_max_pool2drH  r?   r"  rK  r0   r   )rv   r  r  r;  r  r  r  r[  r  r  r  r   r\  r]  r  r   r  r9  r:  rg  rk  r^  rirj  rf  s   `                      @@rT   adaptive_max_pool2dro    s   a###{q   NNUD$7711$7D7711$7DLE5zUaZ'5'%''V1;;=H%%++allnK
 
 	
 e|qTE\Q.u}dem42;!QP*1k::<"AQJAw @Q{QFG g:D5L1,u5LD5L1,u5LE{eU^+HKKME-KR+A{;;4F ."L1%.;; 7"L1%.;;NN 
		||~!	
B 
		||~kk!	
B r6MrV   c                               t        z
  dz
         j                          fd}|S )Nr#   c                 J    g |       }t        j                  |j                               }t        j                  j                               }t        j                  ||z   |z        t        j                  ||z        z
  }t        j                  |t
        j                        }t        j                  |t        j                  dz
  t
        j                              }t        j                  ||t        j                  	
z
  t
        j                              S r  )	r?   r  r   r  r   rL   r  r  r  )r  r   samplei_expr
alpha_exprseq_ir  r  r   in_sz	kernel_szout_szsamplessamples_loaders          rT   r  z)_fractional_pooling_offsets.<locals>.load	  s    &#/7#4#4#67^^E7+<+<+>?
		6F?j89CIIZ=
 
 UEKK0vvNN6A:u{{3
 yyucnnUY5F&TUUrV   )r   r  )ry  rv  rx  rw  r   r  r  rz  s   ````` @@rT   _fractional_pooling_offsetsr{    sT    C[F#JE#Iuy(&1*5E((*NV V KrV   c                 ~   | j                          | j                         ^ }\  }}|\  }}||z  dk\  rt        | ||      S t        j                  t
        |g|      }	 |	d       |	d      | j                         fd}
t        |      ||gz   }t        j                  | j                         | j                         t        j                  |
d      |	      }t        j                  | j                         t        j                  t        j                  |
d
      |	      }||fS )Nr  )ry  rv  rx  rw  r   r  r#   c           	      t   | ^ }}}t        j                   ||            }t        j                   ||            }d }d }t        j                  t	        d         t	        d               D ]  \  }	}
 g |||	z   ||
z         }|rt        j
                  ||	z   z  |z   |
z   t        j                        }||}nRt        j                  t        j                  t        j                  ||      t        j                  |            ||      }||}t        j                  ||      } |r|S |S r  )r?   r  r   r  r   r  rL   r  r  or_r  r  r"  )r  r  r  r  r  r-  r.  r  r  r  r  r  r  
h_index_fninp_hinp_wr  
w_index_fnr  s                rT   rH   z!fractional_max_pool2d.<locals>.fn3  s:   R--j.DeL--j.DeL''k!n(=u[QR^?TU 	2FBLVL]R%7L9KLMC"R'50=@2Eu{{ #$H"yysF 3SYYs^DeX H ~S&1	2  OMrV   Fr  r  T)r  r   fallback_fractional_max_pool2dr   r   r{  r  rn   r0   r   r   r   rL   r  )rv   r  r  random_samplesr  kernel_hkernel_wr  r  gen_offsets_for_dimrH   r  r^  rn  r  r  r  r  r  s    `            @@@@@rT   fractional_max_pool2dr    s/   NN::<UE5$HhLE5(b -{K
 	
 $++#en %+J$+J}}H : E{eU^+H			||~kkm""2E:	
B 
		||~kk""2D9	
B r6MrV   c                 b     j                           j                         ^ }}}t        j                  j                  j                  |      }t        j                  j                  j                  |      }|^ }}}	||z  dk(  r||	z  dk(  rt         ||z  ||	z  gd      S t        ||      }
t        ||	      }d fd}t        ||
|g||g||	gt        j                         fd}t        j                   j                          j                         |t        |            }|S )	Nr   r#   )divisor_overridec                 F    t        | |z  t        j                  |            S r]   )r   r   r  rO  s      rT   r9  z0upsample_nearest2d_backward.<locals>.start_indexs  s    uwg(>??rV   c                      | dz   ||      S r  r   )r  r7  r8  r9  s      rT   r:  z.upsample_nearest2d_backward.<locals>.end_indexv  s    EAI99rV   rS  c                 (     | t                    S r]   )r5  )r  rV  rv   s    rT   rH   z'upsample_nearest2d_backward.<locals>.fn  s    c.q122rV   r  )r  r   r@   r   r   r  rX  r6   rH  r?   rr   r0   r   r   r   rn   )rv   r  
input_sizer  r  r  r  r  out_hout_wr\  r]  r:  rH   r^  rV  r9  s   `              @@rT   upsample_nearest2d_backwardr  a  s"    NN::<UE5GG2259EGG2259E%UE5u}eemq0!euneun=PQRR5%(L5%(L@: ""L1%.77F3 
		||~kkmJ	
B IrV   c           
      (    t        | ||||||d      S )Nr!   r  _avg_poolndrv   r  r  r  r  count_include_padr  s          rT   rX  rX    )     		 	rV   c           
      (    t        | ||||||d      S )Nrz   r  r  r  s          rT   
avg_pool3dr    r  rV   c                 L   ssdgz  t              t              t              t        | t              sJ t              k(  sJ t              k(  sJ t              k(  sJ t        | j	                               dz   dz   fv sJ | j                          | j	                         d   }| j	                          d  t        t              D 	cg c]  }	t        |	   |	|       c}	 \  }
}t              st        |      rt        | d      d}n| j                         d}t        |      t        |
      z   }| j                         t        j                  t         j"                        }|dkD  r4dk(  rt$        }nd	k(  rt&        }nt)        d
        || ||      S fd|r|r|rd|z  nd|z  fd}nfd}t+        j,                  | j/                         ||      }|S c c}	w )Nr   r#   r!   r  r  TFr  rz   zUnknown dim: c           	      T   | d 	  }| 	 d  }d }t        j                  t        	      D cg c]  }t        
|          c} D ]V  }t        	      D cg c]  }||   |   z  ||   z   |   z
   }} |g ||      }||}At        j                  ||      }X |S c c}w c c}w r]   )r   r  r   r?   rr   )r  r  r  r   totalr   r  r   r  r   r  r  r  s            rT   rV  z_avg_poolnd.<locals>.fn_sum  s    UsdJ##U3Z%PeKN&;%PQ 	,BBG*MQ1Q4&)#be+gaj8MCM6C)C}U+	,  &QMs   B B%r9  c                 f    t        j                   |       t        j                              S r]   )r?   r  r  )r  r   rV  r%  r  s    rT   rH   z_avg_poolnd.<locals>.fn  s&    776#x0#,,ue2LMMrV   c                     | d 
  }| 
 d  }g }t        
      D ]  }||   |   z  |   z
  }t        j                  ||   z   |   |   z         }	s/t        j                  |d      }t        j                  ||         }t	        j
                  ||z
  t        j                        }|j                  |        t        j                  t        j                  |      }t	        j                   |       |      S r   )r   r   MinMaxr?   r  rL   r  r   r   r  r  rU  )r  r  r  divide_factorsr   hstarthendfactordivide_factorr  r   rV  r  r  r  r  r  s            rT   rH   z_avg_poolnd.<locals>.fn  s    3$ZFcTUBN3Z .A*WQZ7yy+a.!8!A$:KL("YYvq1F 99T1Q40Dvu{{C%%f-. &,,SWWnEM;;vc84mDDrV   r  )r<   rW   r3   r   r   r  r   r   r  r   r  r  rn   r   r   r  operatorr  fallback_avg_pool2dfallback_avg_pool3d
ValueErrorr0   r   r   )rv   r  r  r  r  r  r  r   r  r   r  
ceil_modeshad_paddingr  r  fallbackrH   r^  r   rV  r  r%  r  s    ``` ` `          @@@@@rT   r  r    sD    #){C0K&#&F7C(Ga###{s"""v;#w<3qzz|q#' 2222NNJJL3$E	

cTUA 3Z

 1q+vw	J

E: 7|s:.q#3?==?E{T%[(HKKME""8<<=KR!8*HAX*H}SE233
 	
 *((E+%E	N 	N
	E 	E  
		||~	
B Ig

s   ,H!c                    dk7  sJ d       ssddgt        | t              sJ t        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |j                               dv sJ | j	                          |j                         ^ }t        d|      \  }	}
t        d|      \  }}| j                         d   xs d   xs |
xs || j                         ^ }t        |j                               }|j                         }t        fdt        d   dz        D              t        fdt        d   dz        D              z  }|dkD  rt        | ||      S fd	fd
}t        j                  | j                         |||      }|S )Nr   divisor must be not zeror!   rk  r#   c           
   3   t   K   | ]/  }t        |d    z  t        d |d    z
  d    z        z
  d       1 ywr  r  r  s     rT   rl   z&avg_pool2d_backward.<locals>.<genexpr>b  r  r  c           
   3   t   K   | ]/  }t        |d    z  t        d|d    z
  d    z        z
  d        1 ywr	  r  r
  s     rT   rl   z&avg_pool2d_backward.<locals>.<genexpr>f  r  r  r  c           	      @   t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  t        j
                  | |      |      }t        j                  t        j
                  ||      |      }	t        j                  t        j                  ||      t        j                  t        j                  t        j                        |            }
t        j                  t        j                  |	|      t        j                  t        j                  t        j                        |            }t        j                  |t        j                  dt        j                              }t        j                  |	t        j                  dt        j                              }	t        j                  |
t        j                  t        j                              }
t        j                  |t        j                  t        j                              }t        j
                  t        j                  |
|      t        j                  ||	            }|S )z{
        This computes the scaling factor that we will divide an element
        by when `count_include_pad=False`
        r   r#   )
r?   r  rL   r  r4  r  r#  rr   r  r"  )r  r  stride_hstride_wpad_hpad_wr  r  r  wstartr  wendr  r(  r  r  r  r   s                rT   !compute_pool_size_without_paddingz>avg_pool2d_backward.<locals>.compute_pool_size_without_paddingy  s   
 <<q	5;;7<<q	5;;7WQZ5WQZ5<<A<<<A<X.6X.6{{GGFH%GGCNN65;;7?
 {{GGFH%GGCNN5%++6>
 VS\\!U[[%ABVS\\!U[[%AB{{4!DE{{4u{{!CDf 5swwtV7LMrV   c                 8   | ^ }}}|d   z   }|d   z   }t        j                  t        |d   z
  d   z   d         t        j                        }t        j                  t        |d   z
  d   z   d         t        j                        }t        j                  t        |d         dz   t        j                        }t        j                  t        |d         dz   t        j                        }t        j
                  |t        j                  dt        j                              }t        j
                  |t        j                  dt        j                              }t        j                  |t        j                  t        j                              }t        j                  |t        j                  t        j                              }d }t              D ]+  }	t              D ]  }
t        j                  |t        j                  |	t        j                              }t        j                  |t        j                  |
t        j                              }}nssd   d   z  }n	 ||      }t        j                   g |t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d      t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d            |      }t        j                  t        j                  ||      t        j                  ||            }|;t        j                  ||t        j                  dt        j                               }t        j                  |t        j                  ||      |      } . |J |S )Nr   r#   Fr  r  )r?   r  r   rL   r  r"  r  r#  r   rr   rU  r  r4  r  r  r  r   )r  r  r  r  r  r  r  r  r  r  r  r  r  r%  partr  r  r  r  r  r  r  r  r  r  r  r  r  s                   rT   rH   zavg_pool2d_backward.<locals>.fn  s2   A
N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM' *	RC]+ )RWWWcll3&DEWWWcll3&DE#/,E&k'N[^;E=b"EE{{#11 #$&s||Au{{7S(T!" !.&+  11 #$&s||Au{{7S(T!" !-&+& ). xxFF2u%FF2u% #"yytS\\#u}}5UVH"yyswwx/FQHS)R*	RV ###rV   r  )rW   r3   r   r   r  r  r  rn   r   rX  r   fallback_avg_pool2d_backwardr0   r   r   )r#  rv   r  r  r  r  r  r  r  r  r  r  r  r   r  r   r  rH   r^  r  r  r  r  r(  r  r  r  r   s     ``` ``           @@@@@@@@@rT   avg_pool2d_backwardr  ;  s    #'71'<X>XX<a&k9---a###{q   v;!w<1qzz|&&&JJLUFE$VQVWiXE:$UA{FGYWE:))+K!*F
FjFJK&1&:&:&<#Q|AJJL!HKKME {1~)* M  {1~)* M
  -/KR+	
 		
 8? ?B 
		%%'	
B IrV   c                     dk7  sJ d       ssg dt        | t              sJ t        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |j                               dv sJ | j	                          |j                         ^ } t        d|      \  }	}
t        d|      \  }}t         d|      \  }}| j                         t              xs
 |
xs |xs || j                         ^ }t        |j                               }|j                         }fdt        d      D        \  z  z  }|d	kD  rt        | ||      S  fd
fd}t        j                  | j                         |||      }|S )Nr   r  )r   r   r   rz   )r{   r|   r#   r!   c              3   j   K   | ])  t        fd t           dz        D               + yw)c           
   3   t   K   | ]/  }t        |   z  t        d |   z
     z        z
  d       1 ywr  r  )rj   r  r   r  r  s     rT   rl   z0avg_pool3d_backward.<locals>.<genexpr>.<genexpr>  sF      
 VAYQ[^);q	(I!JJAN
r  r!   N)rX  r   )rj   r   r  r  s    @rT   rl   z&avg_pool3d_backward.<locals>.<genexpr>  s;      3
 	 	 
;q>A-.
 	
3s   /3}   c           	         d D        \  }}}d D        \  }}}d D        \  }	}
}d t        | ||g|||g|||g      D        \  }}}d t        |||g|	|
|gg|||g      D        \  }}}d |||fD        \  }}}d t        |||gg      D        \  }}}t        j                  t        j                  t        j                  ||      t        j                  ||            t        j                  ||            }|S )Nc              3   d   K   | ](  }t        j                  |t        j                         * y wr]   r  )rj   r  s     rT   rl   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>$  s     'UQ(D'U   .0c              3   d   K   | ](  }t        j                  |t        j                         * y wr]   r  r  s     rT   rl   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>%  s     Ms||Au{{;Mr  c              3   d   K   | ](  }t        j                  |t        j                         * y wr]   r  )rj   rg  s     rT   rl   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>&  s$      (
-.CLLEKK((
r  c              3   x   K   | ]2  \  }}}t        j                  t        j                  ||      |       4 y wr]   )r?   r4  r  )rj   r  r  pads       rT   rl   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>*  s3      "
1c GGCGGAqM3'"
s   8:c           
   3      K   | ]i  \  }}}}t        j                  t        j                  ||      t        j                  t        j                  |t        j
                        |             k y wr]   )r?   r#  rr   r  rL   r  )rj   r  rg  r   r  s        rT   rl   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>0  sU      

 #q#s KKq!3773>>#u{{+KS#Q

s   A/A1c              3      K   | ]<  }t        j                  |t        j                  d t        j                               > ywrn  r?   r"  r  rL   r  )rj   r  s     rT   rl   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>;  s2      "
 KKs||Au{{;<"
   AAc              3      K   | ]?  \  }}t        j                  |t        j                  |t        j                               A y wr]   r?   r#  r  rL   r  )rj   r  r   s      rT   rl   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>?  s6      
S KKS^^C=>
   AA)r   r?   r  r4  )pdr  r  stride_dr  r  pad_dr  r  kernel_dr  r  dstartr  r  dendr  r  r  depthr(  r  r  r  r   s                      rT   r  z>avg_pool3d_backward.<locals>.compute_pool_size_without_padding#  sH   'Uf'U$(HMWMue(
2=(
$(H"
 Rx8<ueU>S"


 '*(8X.&u%	'	

dD"
 &&1"

tT 2UFE4JK
dD GGCGGD&)3774+@A3774QWCX
 rV   c                 .   | ^ }}}}d t        |||g      D        \  }}}d t        |||g!      D        \  }}}d t        |||g!      D        \  }}	}
d |||fD        \  }}}d t        ||	|
g g      D        \  }}	}
d }t              D ]{  }t              D ]i  }t        "      D ]W  }d t        |||g|||g      D        \  }}}}n ssd   d   z  d	   z  }n
 |||      }t        j                   g |t        j                  t        j
                  |t        j                  |t        j                  dt        j                                    d
      t        j                  t        j
                  |t        j                  |	t        j                  dt        j                                    d
      t        j                  t        j
                  |t        j                  |
t        j                  dt        j                                     d
            |      }t        j                  t        j                  t        j                  ||      t        j                  ||	            t        j                  ||
            }|;t        j                  ||t        j                  dt        j                              }-t        j                  |t        j                  ||      |      }Z l ~ |J |S )Nc              3   ,   K   | ]  \  }}||z     y wr]   r   )rj   r  r  s      rT   rl   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>J  s     Avq#1s7As   c              3      K   | ]<  \  }}}t        j                  t        ||z
  |z   |      t        j                         > y wr]   r?   r  r   rL   r  )rj   r  rg  r  s       rT   rl   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>L  s<      %
1a NN8AEAIq15;;?%
r  c              3      K   | ]8  \  }}t        j                  t        ||      d z   t        j                         : ywr  r  )rj   r  r  s      rT   rl   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>Q  s6      
1 NN8Aq>A-u{{;
s   >A c              3      K   | ]<  }t        j                  |t        j                  d t        j                               > ywrn  r  )rj   pstarts     rT   rl   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>V  s2      %
 KKQ <=%
r  c              3      K   | ]?  \  }}t        j                  |t        j                  |t        j                               A y wr]   r  )rj   pend
pooled_dims      rT   rl   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>Z  s6      
 j KKcnnZEF
r  c              3      K   | ]?  \  }}t        j                  |t        j                  |t        j                               A y wr]   )r?   rr   r  rL   r  )rj   r  p_s      rT   rl   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>f  s6      "&FB R(EF"r  r   r#   r!   Fr  r  )r   r   r?   rU  r  r#  r4  r  rL   r  r  r  r  r   rr   )#r  r  r  r  r  pdstartr  r  pdendr  r  r  pd_r  r  r  r  r  r%  r  r  r  r  d_window_sizer  r  r  r  r  r  pooled_depthr  r  r  r  s#                        rT   rH   zavg_pool3d_backward.<locals>.fnH  s   AqAaAY)@A1a%
1ay+v>%
!'

Q1Iv.
ue
%
"GW5%
!'
$'u%m\'R%
ue ' 8	VC]+ 7V / 6VC"*-$gw7#sC+"JBB (3 0*+ +AQ ?+a. P A"b" M;;#!' # 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!" !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %2*/!"  !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!"!4 7D< 88E!2CFF2u4EFr5)D  '#&99 $S%--(H$ $'99T3778T3JH#Um6V7V8	Vr ###rV   r  )rW   r3   r   r   r  r  r  r   rn   r   r   fallback_avg_pool3d_backwardr0   r   r   )!r#  rv   r  r  r  r  r  r  r  d_outceil_mode_dr  ceil_mode_hr  ceil_mode_wr   r  r   r  rH   r^  r  r  r  r  r  r  r(  r  r  r  r  r   s!     ``` ``             @@@@@@@@@@@@rT   avg_pool3d_backwardr    s    #'71'<X>XX<k9---a###{q   v;!w<1qzz|&&&#$::< UE65%eQVWiXE;%;E; &eQVWiXE;))+Kg,K+KKK4?4H4H4J1Qm\AJJL!HKKME3
 q3/M=-  -/-?KS+	
 		
# #JU U Un 
		%%'	
B IrV   c                    | j                         }t        |t              r|g}n|st        t	        |            }t	        |      dk(  rt        |      dv s
J d|        g S t        |      }t        t	        |            D ]_  }||   dk  r#||xx   t	        |      rt	        |      ndz  cc<   d||   cxk  rt	        |      k  rFn t	        |      dk(  r	||   dk(  r_J  t	        t        |            t	        |      k(  sJ d       |S )Nr   )r   rD  r  zinvalid axis: r#   zreduction axis not unique)r   rW   r   r   r   ro   rn   rp   )rv   r  r  r   s       rT   _validate_reduction_axisr    s    ::<D$vSY
4yA~T{//H>$1HH/	:D3t9 M7Q;GCIs4y14GDG'c$i'CINtAw!|LLM s4y>SY&C(CC&KrV   c                `  
 |t        | |      } | j                         t        t        | |            }g }g g }g t	        t                    D ]Q  }||v r&j                  |       |j                  |          -j                  |       |j                  |          S 
fd}r+t              }	D ]  }t        j                  d      |	|<    n|}	| j                         
t        | j                         |xs | j                         | j                         ||	|      S )Nc                    t        |      t        	      k(  sJ r+t        |       t        
      k(  sJ D cg c]  }| |   	 } }t        |       t              k(  sJ d gt        |       t        |      z   z  }t        j                  t        |       t        	|            D ]
  \  }}|||<     |      S c c}w r]   )r   r   chainr   )r  reduction_indexr   r  r  varinner_loaderkeepdimskept_idxreduced_idxr  s         rT   r  z%_make_reduction_inner.<locals>.loader  s    ?#s;'7777u:T***'/0!U1X0E05zS]***Fc%j3+??@	!% #k?"C
 	!HC !IcN	! I&& 1s   B;r#   )r  	dst_dtyperN  r  r  reduction_ranges)r   r   rp   r  r   r   r   rn   r   r   r  r   r   r   )rv   r  r  r   r   
kept_sizesreduced_sizesr   r  r  r  r  r  r  s     `       @@@@rT   _make_reduction_innerr    s$   Q::<D'401DJHMK3t9 '9q!  a)OOAd1g&'' ' : 	+A--*HQK	+ ==?L||~'81;;=++-& rV   rs  c                      dd d fd}|S )Nr   c                    t        | |||      }t        j                  d| d|}t        |j                  j                  t              r|j                          |S )Nr  r  r   r   )rs  
input_noder   )r  r1   r   rW   r/  r<  )rv   r  r  r   r   r  r   rs  s         rT   r(  zmake_reduction.<locals>.inner  sa    &"7
 !!XAXQWXKKi
 NNrV   r   r   )rs  r   r(  s   `` rT   make_reductionr    s    T  LrV   c                    |t        | |      } | j                         }t        | |      }t        | j	                         | j                         f| j                         f| j                         |      S )N)r  dtypes	inner_fnsr  r  )r   r   r  r   r   r   r  )rv   r  r   r  s       rT   _make_scan_innerr    se    Q::<DD!D||~==?$ZZ\ rV   r   c                   |t        | |      } | j                         t        | |      }| j                         }|t        j
                  t        j                  fv rt        | t        j                        } t        | ||      }t        fd|D              }t        j                  || j                         | j                               }t        j                  |t        |j                                     }t        t!        ||      |      S )Nc              3   (   K   | ]	  }|     y wr]   r   rj   r   r  s     rT   rl   zmean.<locals>.<genexpr>       0a$q'0   )r   r   r  r   rL   r#  r"  r   sum_r=   r&   r,   r   r+   r   rn   div)rv   r  keepdimr   output_dtype
sum_resultdenomr  s          @rT   rx  rx    s    Q::<D#At,D;;=Lu~~66Q$aw'J0400Eq{{}allnEEeT**=*=*?%@AEC
E*L99rV   c                 0  
 |d}| j                         
t        | |      }t        | |d      }|r|j                          t	        t        | |            }t        |||      }t        
fd|D              }|rt        j                  ||z
  d      }t        j                  || j                         | j                               }t        j                  |t!        |j                                     }t#        ||      }	|s|	fS |r|nt%        ||      }|	|fS )Nr#   T)r  c              3   (   K   | ]	  }|     y wr]   r   r	  s     rT   rl   z var_mean_sum_.<locals>.<genexpr>+  r
  r  r   )r   r  rx  r<  squarer4  r  r=   r   r  r&   r,   r   r   r+   r   rn   r  r  )rv   r  
correctionr  return_meanx_meandiffsr  r  x_varr  s             @rT   var_mean_sum_r    s    
::<D#At,D!T4(F3q&>"EeT7+J0400E		%*,a0q{{}allnEEeT**=*=*?%@AE
E"ExVGFD$9F&=rV   c                     t        | |      }t        | ||d d       }|d   }t        |d         }t        |t        j
                        xr, t        |      t        j                  k  xr t        |      dk7  S )Nr   r  r  r#   )	r  r  r=   rW   r   r   r   r$   unroll_reductions_threshold)rv   r  r  r   r  reduction_numels         rT   use_two_step_variancer  8  s|    #At,D"	wd$F HF#F+=$>?O?EMM2 	' 6#E#EE	'&!Q&rV   c                   dt        | ||d d       }|j                  d      }|j                  d       |j                  d       t        j                  j                  d|fd| j                         d|\  }}}	|j                          | j                         | j                         t        | |      }t        fd|D              d	 fd
}
 t        |
      |      }|r|j                          ||fS |fS )Nr#   r   r  r  rN  welford_reduce)r  rs  r   c              3   (   K   | ]	  }|     y wr]   r   r	  s     rT   rl   z$var_mean_welford_.<locals>.<genexpr>^  s     1q471r  c                     t        | t        j                        rD| j                  s8t	        j
                  t	        j                  | t        j                        |      S t	        j                  | |      S r]   )
rW   r   r   	is_numberr?   r   r  rL   r  r  r   s     rT   get_constant_or_index_exprz5var_mean_welford_.<locals>.get_constant_or_index_expr`  sG    a$Q[[<<q%++ >FF||Au%%rV   c                            }       }t        j                  d      }| t        j                  |||z
        z  S r   )r?   r  r"  )r/  cNzeror  r   r$  rnumels       rT   r  z#var_mean_welford_.<locals>.scale_fne  sE    &z59&vu5||Au%ckk$A...rV   r   )r  r&  r&   WelfordReductionr   r   r<  r   r  r=   r*  )rv   r  r  r  r  r   r  rx  m2r   r  r  r   r$  r)  r  s     `         @@@@rT   var_mean_welford_r,  H  s    
"	wd$F ZZ
#F
JJ{
JJ{%%,, )'kkm 	KD"a JJLKKME::<D#At,D1D11F&
/ #.
"2
&CDy6MrV   c                    | j                         t              }t        | |d      } t        | ||||      }t	        | ||      rt        di |n
t        di |}t        fd|D              }|s|d   S |S )NFrW  )rv   r  r  r  r  )r  r  c              3   :   K   | ]  }t        |d         yw)FrW  N)r   )rj   rv   	out_dtypes     rT   rl   z#var_mean_helper_.<locals>.<genexpr>  s     F!8Ayu55Fs   r   r   )r   r   r   r   r  r  r,  ro   )	rv   r  r  r  r  compute_dtyper   r   r/  s	           @rT   var_mean_helper_r1  s  s    I))4MM.A
F !w? 	(( 
 FvFFF'6!93V3rV   )r  r  c                "    t        | |||d      S )NFr  r  r  r  r1  rv   r  r  r  s       rT   var_r6    s    	W% rV   c                "    t        | |||d      S )NTr3  r4  r5  s       rT   var_meanr8    s    	W$ rV   c                    |dk  r!t        t        j                  |       | |      S |dk(  rt        j                  d|      S |dk(  r| S t        | |dz  |      }t        j                  ||      }|dz  dk(  rt        j                  ||       }|S )Nr   r#   r!   )pow_recursiver?   r!  r  r  )rv   r   r   r  s       rT   r:  r:    s    1uS^^A.E::Av||Au%%Av1a1fe,FWWVV$F	A!|#MrV   c                 .    t        j                  | |      S r]   )r?   powr   r   s     rT   
pow_nativer>    s    771a=rV   )r   c                 v    t        t              r#t              k(  rt         t                    S t        t              rdk(  rt	               S t        t              rdk(  rt               S t        d  fD              }t        |      }t        t              xr dcxk  xr dk  nc xs	 |xr dk\  }|rZ j                          fd}t        j                   j                          j                         | j                               S t         t              r: dk(  rt        d      S  d	k(  r$t!        j                               rt#              S |rDt         t              rt%               S t        t              rt'               S t)               S t+               S )
Nr  r#   c              3   r   K   | ]/  }t        |t        j                        s|j                          1 y wr]   )rW   r&   r3   r   r   s     rT   rl   zpow.<locals>.<genexpr>  s#     N1*Q2MNs   77i    r   c                 F    t         |       j                               S r]   )r:  r   )r  r   r   r  s    rT   rH   zpow.<locals>.fn  s     a??rV   r  r!   )rW   r   r   r<  sqrtrQ  r   r   r  r0   r   r   r   r   r   r  r   exp2fallback_pow_scalarfallback_pow_tensor_scalarfallback_pow_tensor_tensorr>  )r   r   r   is_integer_powembed_exponentrH   r  s   ``    @rT   r<  r<    sx   !USV1c!f~	Au	!s(Aw	As	QQx NANNE%e,N  3' a"32AF  	@ <<>++-::<	
 	
 !V6Q?"6nQ[[]37Na &q!,,6"-a33-a33arV   c                    t        | t              r| j                  }n| }t        |t              r|j                  }t        |t        j                        swt        j                  | j                         | j                         |j                         | j                               j                  }t        |t        j                        sJ t        |t        j                        rg|j                         sW|j                         sGt        |j                  t        j                        s#|j                          |j                  |_        | S t        j                  j!                  |||       | S )Nr  unsafe_alias)rW   r3   r/  r&   rV  r0   r   r   r   r  r   is_input_bufferis_module_buffer	NopKernelr<  rD  realize_into)changedr  rL  changed_datas       rT   r  r    s   '9%||#y!hhc2==)%%'##%__&##%	

 $ 	 #r}}---,.$$&((*l''6 	HH!!..\ /  NrV   c                 .    t        | t        | |            S r]   )r  r  )rv   r  s     rT   fill_rT    s    Q	!Z011rV   c                     | |u r| S t        || j                               }t        || j                               }t	        || j                               }t        | |      S r]   )rs  r   r   r   r   r   r  )r_  r  r  s      rT   rZ  rZ  
  sR    
cz

C)
*C
3
(C
clln
%CS#rV   c                 .    t        j                  | |      S r]   )r?   floordivr=  s     rT   rW  rW        <<1rV   c                 .    t        j                  | |      S r]   )r?   truncdivr=  s     rT   rZ  rZ    rX  rV   c                 >   t        |       xr t        |      }t        |       xr t        |      }|dk(  r,|rJ d       |rt        | |      S t        t	        | |            S |dk(  r,|rJ d       |rt        | |      S t        t	        | |            S t	        | |      S )Nr  z5floordiv operands can not be boolean at the same timer  z5truncdiv operands can not be boolean at the same time)r   r   rW  r  r  rZ  r  )r   r   rounding_modeboth_integerboth_booleans        rT   div_moder_    s    "1%</!*<L"1%</!*<L X!XX!-x1~C5Q3CCX!XX!-x1~C5Q3CCq!9rV   c                     t        |       xr t        |      }|rt        | |      S t        t        j                  j
                        } t        |      | |      S r]   )r   logical_andr.   rh  r  rx  r*  )r   r   	both_boolrH   s       rT   r  r  /  sN    "9q'9I1a  **+!~b!!Q''rV   c                 f   t        | t        j                        rt        | j                        S t        | t        j
                        rt        | j                               S t        | t        j                        r| S t        | t        j                        syt        j                  j                  j                  | j                               }t        j                  |      5  t!        j"                  t        j$                  dd      5   | j&                  | j)                          }ddd       ddd       t        t        j                  j*                  j,                        sJ t        |j.                  t        j                        r|j.                  S y# 1 sw Y   rxY w# 1 sw Y   vxY w)z:Try convert an arbitrary IR node into an ir.Constant valueNallow_indexingT)rW   r&   
MutableBoxget_constant_valuer/  r  r  r   LoopsrL   	_inductorops_handlerExtractConstantsHandlerr   r@   set_ops_handlerr   objectr  r  inner_fn_argsvirtualizedOpsValuer   )rv   r  r   s      rT   rf  rf  9  s1   
 !R]]#!!&&))!R[[!!!--/22!R[[! a"oo))AA!,,.QG	
		7	# -U\\
+T& - ajj!//+,- -
 c5??66??@@@#))R[[)yy- - - -s$   &&F'F*F'F$	 F''F0c                 2   t        d | |fD              }|rt        | |      S t        |      x}T|j                  dk(  r*t	        j
                  t        d      |j                        }nd|j                  z  }t        | |      S d } t        |      | |      S )Nc              3   L   K   | ]  }t        |      xs t        |        y wr]   )r   r   r   s     rT   rl   zdiv_prim.<locals>.<genexpr>Z  s"     O1oa(>OA,>>Os   "$r   infr9  c                  &    t        j                  |  S r]   )r?   rU  r  s    rT   rH   zdiv_prim.<locals>.fng  s    {{D!!rV   )	r   rZ  rf  r   mathcopysignr   r  r*  )r   r   is_integraldivisorr!  rH   s         rT   div_primrx  X  s    OAOOK1~%a((5==AuU|W]]CJw}},J1j!!" >"a##rV   c                 Z    t        | |ft        j                        \  } }t        | |      S r  )r  r   INT_TO_FLOATrx  r=  s     rT   r  r  m  s0     	
A$C$P$PDAq Aq>rV   c                 h    t        |       xs t        |       }|rd }nd } t        |      | |      S )Nc                 .    t        j                  | |      S r]   )r?   modr=  s     rT   rH   zfmod.<locals>.fn  s    771a= rV   c                 .    t        j                  | |      S r]   )r?   fmodr=  s     rT   rH   zfmod.<locals>.fn  s    88Aq>!rV   )r   r   r*  )r   r   rv  rH   s       rT   r  r  y  s9    !!$:(:K	!
	" >"a##rV   c                     | j                         }t        |      st        |      rt        | t	        j
                               } d } t        |      |       S )Nc                 ,    t        j                  |       S r]   )r?   rsqrtr   s    rT   _rsqrtzrsqrt.<locals>._rsqrt  s    yy|rV   )r   r   r   r   rL   r  r*  )rv   r   r  s      rT   r  r    sK    KKME"25"9Q//12 ">&!!$$rV   c                    t        | j                               st        | j                               r|t        j                  }t        d|      } || |||      S )Nrh  rP  r   r   r   r   rL   r  r  rv   r  r  r   rH   s        rT   r  r    sK     	'+;AKKM+J
-	U	;Baxu--rV   c                    t        | j                               st        | j                               r|t        j                  }t        | j                               dk(  r(|dv sJ |xs | j                         }t        | |d      S d }t        | ||      }t        j                  j                  d	i |d|i\  }|t        | ||      S |S )
Nr   r   r  TrW  c                 @    | \  }|\  }t        j                  ||      fS r]   )r?   rr   a_tupleb_tupler   r   s       rT   
combine_fnzcumsum.<locals>.combine_fn  #    1rV   r  r   r  r   r   r   )r   r   r   rL   r  r   r   r   r  r&   Scanr   fallback_cumsumrv   r  r   r  r   r  s         rT   cumsumr    s     	'+;AKKM+J
-
1::<Aw&5t,, 
 ad%8F??J?IV~qd%88MrV   c                    t        | j                               st        | j                               r|t        j                  }t        | j                               dk(  r(|dv sJ |xs | j                         }t        | |d      S d }t        | ||      }t        j                  j                  d	i |d|i\  }|t        | ||      S |S )
Nr   r  TrW  c                 @    | \  }|\  }t        j                  ||      fS r]   )r?   r  r  s       rT   r  zcumprod.<locals>.combine_fn  r  rV   r  r  r  r   )r   r   r   rL   r  r   r   r   r  r&   r  r   fallback_cumprodr  s         rT   cumprodr    s     	'+;AKKM+J
-
1::<Aw&5t,, 
 ad%8F??J?IV~t599MrV   c                 
   d }| j                         }t        | j                               dk(  r|dv sJ t        |       S t	        | ||      }t        j                  j                  di |d|i\  }|t        | |      S |S )Nc           	      *   | \  }|\  }t        j                  ||      }t        j                  ||      }||k7  t        j                  |       z  }t        j                  |t        j
                  t        j                  ||z
              |z   |      fS r]   )r?   r#  r"  r  r  log1pexp)r  r  r   r   min_vmax_vr  s          rT   log_add_exp_helperz(logcumsumexp.<locals>.log_add_exp_helper  s{    Aq!Aq!CIIe$4#45		$		#''%%-*@ AE I1MOOrV   r   r  r  r  r  r   )	r   r   r   rQ  r  r&   r  r   fallback_logcumsumexp)rv   r   r  r   r   r  s         rT   logcumsumexpr    s    P KKME
1::<Ag~~Qxac7FGG4FGIV~$QC00MrV   c                    t        | j                               dk(  r,|dv sJ t        |       t        | t        j
                        fS | j                         }t        j                  d|d      }|t        j                  u rdnI|j                  rt	        j                  |      j                  nt	        j                  |      j                  }t        | ||      }|t        j
                  f|d<   | j                         d	 f|d
<   t        j                   j"                  di |d|i\  }}|t%        | |      S ||fS )Nr   r  r   argmaxFr   arg_break_ties_leftr  r  c                      yNrindexr   r  s    rT   r   zcummax.<locals>.<lambda>      rV   r  r  r  r   )r   r   rQ  rY  rL   r  r   r&   get_reduction_combine_fnr   rc  rd  rW  rf  r  r  r  r   fallback_cummax)rv   r  r   r  r  r   r   r   s           rT   cummaxr    s   
1::<AwQxAU[[999KKME,,5J EJJ 	 && KK""U#''  ad%8Fu{{+F8==?,>?F;ggnnEvE*EOFG~qd++7?rV   c                    t        | j                               dk(  r,|dv sJ t        |       t        | t        j
                        fS | j                         }t        j                  d|d      }|t        j                  u rdnI|j                  rt	        j                  |      j                  nt	        j                  |      j                  }t        | ||      }|t        j
                  f|d	<   | j                         d
 f|d<   t        j                   j"                  di |d|i\  }}|t%        | |      S ||fS )Nr   r  r   argminFr  Tr  r  c                      yr  r   r  s    rT   r   zcummin.<locals>.<lambda>#  r  rV   r  r  r  r   )r   r   rQ  rY  rL   r  r   r&   r  r   rc  rd  rX  rf  r  r  r  r   fallback_cummin)rv   r  r   r  	max_valuer   r   r   s           rT   cumminr    s   
1::<AwQxAU[[999KKME,,5J EJJ 	 && KK""U#''  ad%8Fu{{+F8==?,>?F;ggnnEvE*EOFG~qd++7?rV   c                    t        | j                               st        | j                               r|t        j                  }t        d|      } || |||      S )Nri  rP  r   r  r  s        rT   ri  ri  *  sK     	'+;AKKM+J
-	e	<Baxu--rV   c                 ^    t        | t        j                        }  t        d      | ||      S )Nr   r  r  )r   rL   r   r  rv   r   r  s      rT   
reduce_anyr  5  s(    EJJA >% w??rV   c                 Z    |t        | ||      t        | ||      fS t        | d |      S Nr  )reduce_amaxreduce_argmaxr  s      rT   
reduce_maxr  ;  :    
g6!#8
 	

 qtg66rV   c                 Z    |t        | ||      t        | ||      fS t        | d |      S r  )reduce_aminreduce_argminr  s      rT   
reduce_minr  F  r  rV   xor_sumrX  rW  r  rP  r  
logical_or)r%  r  stabler   
descendingc          	         |d}| j                         }| j                         }t        t        |      |      }t        |      dk(  r't	        |       t        d|t        j                  |      fS t        |      r||   nd}t        j                  j                  j                  |t        j                  t        j                        j                        st        | |||      S t!        |ddt        j                  |d      }dgt        |      z  }t        |      r|||<   t#        ||      }t%        ||      }t&        j(                  j+                  || j,                  |j,                  f| j/                         |j/                         f||||      \  }	}|	t        | |||      S |J |	t1        |t        j                        fS )NFr   r#   r  )r  r  r   r  r  )r  r  r  r  r  r  r  )r   r   r   r   rQ  r  rL   r  r@   r   r   statically_known_ltrf  int16rX  sort_fallbackr  ri  r   r&   Sortr   r   r  r   )
rv   r  r   r  r  r  r  r   
view_shaper   s
             rT   sort_stabler  b  s   ~JJLE\\^F
3u:s
+C
5zQQxq&%++u=== ZuSzQH77//%++ekk:R:V:VWQv3:NNVSXG s5z!J
5z"
37J'GWe$Gggnn'==?G$7$7$9: % OFG ~Qv3:NN8GU[[111rV   c                      t        | d||      S )NFr  )r  )rv   r   r  s      rT   sortr    s    qCJGGrV   c                 <    t        | |t        j                  |      S )Nr   r   r'  r{  r   rz  )r   r   r'  s      rT   register_pointwise_numericr    s!    
;HH'	 rV   c                 :    t        | t        j                  d      S )NT)r   ry  r  rq  s    rT    register_pointwise_numeric_ldf64r    s    
;HH" rV   r  logical_not)r  )r   r   r   identity)r:  pointwise_overrides_datac              #   F  K   t         |   t        | j                  d       }|y fd}t        |t        j
                  j                        r9|j                         D ]%  }t        ||      }|j                   ||      f ' y |j                   ||      f y w)Nc                 4    j                   t        |       S y r]   )tritonr[  )r   r/  s    rT   make_triton_fallbackz6_get_pointwise_overrides.<locals>.make_triton_fallback  s    ;;#B'' rV   )	r  r_   r   rW   rL   rX   rs   ru   r   )nsr   r   r  olnamer  r/  s         @rT   _get_pointwise_overridesr    s     #D)D	TYY	%B	z( "ejj112lln 	IFV$Bd..0DR0HHH	I $**,@,DDDs   BB!r  c                 d    | t         |<   t        j                  |        fd}t        | |       y )Nc                       | i |}g }t        | d   |      D ]"  \  }}|j                  t        ||d             $ |S )Nr   TrK  )r   r   r  )r   r   resultsmut_resultsr   r  outplace_ops         rT   rH   z$register_foreach_inplace.<locals>.fnL  sV    t.v.tAw0 	JKCyf4HI	J rV   )rG   rF   rr   r   )aten_opoutplace_aten_opr  rH   s     ` rT   register_foreach_inplacer  H  s.    07,-G$ w+rV   c                 2    t        | d       fd       }|S )NrS  c                  l     | i |}t        || d   j                               }t        | d   |      S r   )r   r   r  )r   r   r  r  s      rT   rH   zregister_inplace.<locals>.fnl  s<    d-f-&$q'"3"3"56a&))rV   )r   )r  r  rH   s    ` rT   register_inplacer  k  s#    wD9* :*
 IrV   c                      y r]   r   rV  s      rT   sym_constrain_ranger        rV   c                     t         j                  j                  j                  d   }t	        |t
        j                        sJ |j                  j                  S r^  	r@   r   r	  r   rW   rL   r  r@  r  r   r   r  s      rT   sym_sizer    s@    
''


#
#E
*C c5<<(((88==rV   c                     t         j                  j                  j                  d   }t	        |t
        j                        sJ |j                  j                  S r^  r  r  s      rT   
sym_strider    s>    
''


#
#E
*Cc5<<(((88==rV   c                 "    | j                         S r]   )r  )r   s    rT   	sym_numelr    s    ;;=rV   c                     t        d      )NzHelpful for debuggingrc   )r~  r   r   s      rT   foobarr    s    
5
66rV   c                 8    | j                          t        |       S r]   )r<  rQ  r   s    rT   _realizer    s    IIK8OrV   c                 R    | j                          t        j                  | |       | S r]   )r<  r&   ResizeStorageBytes)variabler  s     rT   resize_storage_bytes_r    s#    (H-OrV   c                     | j                          |j                          t        j                  t        j                  | |            S r]   )r<  r3   r   r&   SetSourceTensorKernelr~  source_tensors     rT   set__source_tensorr    s5    LLNB44T=IJJrV   set_c                 p    | j                          |j                          t        j                  | |       y r]   )r<  r&   r  r   s     rT   	fsdp_set_r    s'    
  }5rV   c                  	
 t        | t              sJ t        |t        t        f      sJ |t        j
                  }|t        j                  k(  rt        d|       |t        j                  k(  rt        |      dk(  sJ |t        j                  k(  rt        |      dk(  sJ | j                         
| j                         }| j                         }t        | j                  t        j                         r| j                  j#                         | _        t	        j$                         rit        j&                  j(                  j*                  rEt-        |      rt/        d      n0t1        |      r t	        j2                  |      j4                  ndndt6        j8                  j:                  j=                  
d      rt?        |||      S tA        | 
gd	g      }|jC                         	t        jD                  jG                  ||      }t        jH                  ||||      jK                         	
fd
}tM        jN                  |||t        |            }|S )Nzunsupported memory format: r{   r|   nanTr  r   r  r#   c                     |       t        j                  t        j                        }t        j                  t        j                        }t        j                  ||      }t        j
                  |fd      S )Nc                        g      S r]   r   )
flat_indexflat_loaders   rT   r   z*resize.<locals>.inner_fn.<locals>.<lambda>  s    ZL(A rV   )r?   r  rL   r  r  r  )	r  flat_index_exprlimitr  r
  r  	old_numelout_indexeruninitalized_vals	       @rT   r  zresize.<locals>.inner_fn  sZ     %
..U[[Ay%++6vvou-zz$ ACSTTrV   r  )(rW   r3   rn   ro   rL   contiguous_formatpreserve_formatr  channels_lastr   channels_last_3dr  r   r   r/  r&   r  r  r4  utilsr7  fill_uninitialized_memoryr   r   r   rf  rX  r@   r   r   statically_known_equalsr  r  r  r   stride_ordered_for_memory_formatr  r  r0   r   )rv   r  r  r   r  x_flat
out_strider  r   r  r  r  r  s            @@@@rT   resizer    s   a###dT5M***//---8HII+++4yA~~...4yA~~IKKME\\^F!&&"++&##% 	224KK%%??% $U|e$${{5155# ww//	1=D*%GG		
 	
F $$&K""CCD-XJ..jANNPKU 

UXd4jC JrV   )auto_functionalizedc                     ddl m} |j                  |      }t        j                  | |i ||       |j                         D ci c]  \  }}t        |t              s|| c}}S c c}}w )Nr   )kernel_side_table)
kernel_idxgridkernel_args)*torch._higher_order_ops.triton_kernel_wrapr  get_constant_argsr&   UserDefinedTritonKernelr8  rW   r3   )r  constant_args_idxr   r   r  constant_argsr  r  s           rT   triton_kernel_wrap_r'  %  sd    L%778IJM/v//
 &,\\^Rcz#y7QCHRRRs   A) A)c                 |   t        |       st        t        t         |            rTd}t        j                  j
                  j                  j                  dd       x}r| d| }|t        j                  _        t        j                  j                  | |||      }t        t        t        j                  |            S )Nz"control flow operator: torch.cond.r2  r3  )r-   r   r  r@   r   r	  r   r!  r5  r&   Conditionalr   rn   r3   )predtrue_fnfalse_fnoperandsrf   r2  r  s          rT   re   re   2  s    #c)X672''..3377tLL;LE*;-8C,/)^^""4(HEFI$$f-..rV   c                 l   t        t        t        ||z               rTd}t        j                  j
                  j                  j                  dd       x}r| d| }|t        j                  _        t        j                  j                  | |||      }t        t        t        j                  |            S )Nz(control flow operator: torch.while_loop.r2  r3  )r   r  r-   r@   r   r	  r   r!  r5  r&   	WhileLoopr   rn   r3   )cond_fnbody_fncarried_inputsadditional_inputsrf   r2  r  s          rT   
while_loopr4  >  s    
3y.+<<=>8''..3377tLL;LE*;-8C,/)\\  '>CTUFI$$f-..rV   r  c                   
 ddl m}m} t        j                  ||      D cg c](  } ||j                         |j                               * }} || |      

fd}t        |d   |d       }t        d |D              |d<   t        d	 |D              |d
<   t        j                  j                  d|dd|}	|	d   t        d      |	S c c}w )Nr#   )InputDescriptorlower_pointwise_subgraphr  c                 d     g t        j                  |       t        j                  |       S r]   )r  r  )lhsrhslowered_combine_fns     rT   wrapped_combine_fnz,associative_scan.<locals>.wrapped_combine_fnT  s6    ! 
$
$
 	
rV   r   r  c              3   <   K   | ]  }|j                           y wr]   )r   r   s     rT   rl   z#associative_scan.<locals>.<genexpr>[  s     :qQ[[]:r   r  c              3   <   K   | ]  }|j                           y wr]   )r  r   s     rT   rl   z#associative_scan.<locals>.<genexpr>\  s     ?A?r   r  F)r  can_fallback_to_atenz/Unable to generate code for associative_scan opr   )subgraph_loweringr6  r7  r   r  r   r   r  ro   r&   r  r   r  )r  rC  r   r6  r7  rv   subgraph_inputsr<  r   r  r;  s             @rT   associative_scanrB  J  s    L . 	akkmALLNCO  2*oN
 eAhS=F:E::F8???F;WW^^ %" F
 ayLMMM-s   -Cc                      y r]   r   )tokenss    rT   _sink_tokensrE  g  r  rV   c                 Z   t        j                  j                  |g|i |}ddlm}  ||||      }|J t
        j                  j                  |   }||fS t        j                  t         j                  t        j                  |      }t        |t        t        f      s||fS |g|S )Nr   )get_effect_key)r&   EffectfulKernelr   torch._higher_order_ops.effectsrG  r@   r   effectful_opsr  rU  MultiOutputr3   rW   rn   ro   )tokenr   r   r   r  rG  effect_typeeffectful_kernels           rT   with_effectsrO  l  s    &&r;D;F;F> T62K"""ww,,[9~ ""!!"..)2B2BFKFftUm, &)) *6**rV   c                 >   t        |       } t        j                  rG| j                          t        j
                  j                  j                  | j                                t        j                  j                  t        j                  j                  | ||       | S r]   )rQ  r$    reorder_for_compute_comm_overlapr<  r@   r   no_fuse_buffer_namesrr   r<  r&   _CollectiveKernelcreate_inplacer   all_reduce_r^  r   	reduce_op
group_names      rT   _all_reducerY    sm    Cj22 KKMGG((,,S\\^<
++((00#y*	
 
rV   c                 z    t         j                  j                  t        j                  j
                  | ||       | S r]   )r&   rS  rT  r   rU  r^  rV  s      rT   _all_reduce_r[    s2    
++((00#y*	
 
rV   c                     | D cg c]  }t        |       } }t        j                  j                  t        j
                  j                  | ||       | S c c}w r]   )rQ  r&   rS  rT  r   all_reduce_coalesced_r^  )r   rW  rX  r   s       rT   _all_reduce_coalescedr^    sQ    (./%*//
++22::		
  0s   Ac                 z    t         j                  j                  t        j                  j
                  | ||       | S r]   )r&   rS  rT  r   r]  r^  )r   rW  rX  s      rT   _all_reduce_coalesced_r`    s5    
++22::		
 rV   c                     t         j                  j                  t         j                  j	                  t
        j                  j                  | ||            S r]   )r&   r3   r   rS  create_out_of_placer   all_gather_into_tensorr^  )r   
group_sizerX  s      rT   _all_gather_into_tensorre    sE    ||""  44 77??	
 	
rV   c           	          t        j                  t        j                  j                  t        j
                  j                  t        j                  j                  | ||            S r]   )
r  r  r&   r3   r   rS  rb  r    all_gather_into_tensor_coalescedr^  )r   rd  rX  s      rT   !_all_gather_into_tensor_coalescedrh    sL    LL  44 AAII	
 	
rV   c                ~    t         j                  j                  t        j                  j
                  | |||       |S )N)r   )r&   rS  rT  r   all_gather_into_tensor_outr^  )r   rd  rX  r   s       rT   _all_gather_into_tensor_outrk    s=    
++77?? 	, 	
 
rV   c           	          t         j                  j                  t         j                  j	                  t
        j                  j                  | |||            S r]   )r&   r3   r   rS  rb  r   reduce_scatter_tensorr^  )r   rW  rd  rX  s       rT   _reduce_scatter_tensorrn    sH    ||""  44 66>>
 	
rV   c           
          t        j                  t        j                  j                  t        j
                  j                  t        j                  j                  | |||            S r]   )
r  r  r&   r3   r   rS  rb  r   reduce_scatter_tensor_coalescedr^  )r   rW  rd  rX  s       rT    _reduce_scatter_tensor_coalescedrq    sO    LL  44 @@HH	
 		
rV   c           	          t         j                  j                  t         j                  j	                  t
        j                  j                  | |||            S r]   )r&   r3   r   rS  rb  r   all_to_all_singler^  )r   output_split_sizesinput_split_sizesrX  s       rT   _all_to_all_singlerv    sH    ||""  44 22::"!
 	
rV   c                     t        |       } t        j                  j                  t        j
                  j                  | ||       | S r]   )rQ  r&   rS  rT  r   
broadcast_r^  r   r  rX  s      rT   
_broadcastrz    s;    Cj
++''//c:	
 
rV   c                 z    t         j                  j                  t        j                  j
                  | ||       | S r]   )r&   rS  rT  r   rx  r^  ry  s      rT   _broadcast_r|     s2    
++''//c:	
 
rV   c                 v    t         j                  j                  t        j                  j
                  |        | S r]   )r&   _WaitKernelcreate_waitr   wait_tensorr^  )r   s    rT   _wait_tensorr    s(    
""#3#?#?#G#GM
rV   c           	          t         j                  j                  t         j                  j	                  t
        j                  j                  j                  j                  | |||            S r]   )
r&   r3   r   rS  rb  rL   r?   _dtensorshard_dim_alltoallr^  )r   
gather_dim	shard_dimrX  s       rT   _shard_dim_alltoallr    sR    ||""  44		""55==
 	
rV   zRInductor support for distributed collectives depends on building torch.distributed)r  )quantized_lowerings)mkldnn_lowerings)jagged_lowerings)NN)NNNNFN)Fr]   )r   r   l            r#   TrD  )r   r   r#   )r   Tr  )Tr   )r   NNr#   )r  FF)r!   F)NNN)Nr9  N)r  )NNNN)r   r   FTNr   )r  F(  r   r   loggingrt  r  r  r  collectionsr   typingr   r   r   r   r   r	   r
   r   unittest.mockr   r   rL   $torch.ao.quantization.fx._decomposedtorch.fxtorch.utils._pytreer  _pytreer  (torch._higher_order_ops.associative_scanr   r"  r   torch._prims_commonr   r   r   r   r   r   r   r   r   r   r   torch.fx.experimental.sym_noder   r   torch.utils._sympy.functionsr   r   r   r   r    _dynamo.utilsr"    r$   r%   r&   r'   decompositionr(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   rn  r?   r@   	getLoggerrx  r  rA   rX   rY   __annotations__rB   rp   rC   rh  tr_c10dr\  rD   rE   rF   rG   quantized_decomposedr[   rZ   rg   rq   ry   r  r  rX  r  bmmconvolutionconvolution_backwardrl  r)  mmr  r  _int_mmrR  rQ  r  r  r  r#  r   r  	complex32	complex64r   r"  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r*  rK  r   r   r]  r`  rm  ri  rp  r  rs  
device_putru  r{  r  r  r  r   aliasdetachdetach_liftview_ofr  r   r  r  r  r  r  r  r  r  r  r^  r  r   r  r  r  _unsafe_viewreshaper  slicer  r  r  quantize_per_channelr/  dequantize_per_channelr6  quantize_per_tensorr   r<  dequantize_per_tensorr?  r  rJ  rM  ro  r  r  r  r  r  r  r  r  r  r  r  r  r[  	lru_cacher  r  r  r  r  r  r  r  r  rngprimsr  r  r  	bernoullir  r  r  r  r!  r  r'  r$  r%  r,  r+  r*  randintforce_stride_orderr3  r  r5  r=  r8  lookup_seedr>  randomr  rC  rI  rP  rX  rZ  r{  rP   FALLBACK_ALLOW_LISTr|  _adaptive_avg_pool3dadaptive_max_pool3dfractional_max_pool3dmax_pool3d_with_indicesuniformexponential_pdist_forwardsoft_margin_loss_backwardsearchsorted_cdist_forward_cdist_backwardmax_unpool2dmax_unpool3d
_trilinearsegment_reduce_segment_reduce_backwardhistc	histogrambin_ct_histogramdd_bin_edges_histogramdd_from_bin_ctsaddbmm_addmm_activation
_cudnn_rnn_cudnn_rnn_backward_embedding_bag_embedding_bag_forward_only_embedding_bag_backward*_embedding_bag_per_sample_weights_backward_fused_moving_avg_obs_fq_helper*_fused_moving_avg_obs_fq_helper_functional max_pool3d_with_indices_backward_adaptive_avg_pool2d_backward_adaptive_avg_pool3d_backwardadaptive_max_pool2d_backwardadaptive_max_pool3d_backwardfractional_max_pool2d_backwardfractional_max_pool3d_backwardreplication_pad1d_backwardreplication_pad2d_backwardupsample_linear1d_backwardupsample_bicubic2d_backwardupsample_trilinear3d_backwardgrid_sampler_2d_backward_pdist_backwardr  r  kthvaluetopkr@  median	nanmedianrandpermresize_
resize_as__linalg_detlinalg_householder_productlinalg_inv_exlinalg_ldl_factor_exlinalg_ldl_solve	linalg_lulinalg_lu_factor_exlinalg_lu_solvelinalg_matrix_exp	linalg_qr_linalg_slogdet_linalg_solve_exlinalg_solve_triangular_linalg_svd	lu_unpackormqr_linalg_check_errorslinalg_pinvatol_rtol_tensor_linalg_eightriangular_solvelinalg_cholesky_excholesky_inversecholesky_solvegeqrf_fft_r2cnonzerogcd_thnn_fused_lstm_cell_prims	rng_primsrun_and_save_rng_staterun_with_rng_statemasked_scattermasked_scatter_backwardr  angle_efficientzerotensor(_sparse_coo_tensor_with_dims_and_tensors	to_sparse
_to_sparser   r  '_scaled_dot_product_efficient_attention0_scaled_dot_product_efficient_attention_backward#_scaled_dot_product_flash_attention,_scaled_dot_product_flash_attention_backward#_scaled_dot_product_cudnn_attention,_scaled_dot_product_cudnn_attention_backward+_scaled_dot_product_flash_attention_for_cpu4_scaled_dot_product_flash_attention_for_cpu_backward_flash_attention_forward_flash_attention_backward_efficient_attention_forward_efficient_attention_backwardindex_reducerR  rQ  r  r  r  r  r  r  scalar_tensorr  
LongTensorr  r  r  r  r  r  r  r  r  rY  rZ  
zeros_liker  r  r  r  r  r  r  r  r	  r  r  r  r  r&  r+  r.  r0  r8  r(  r=  r-  rP  fallback__unsafe_masked_indexrT  ,fallback__unsafe_masked_index_put_accumulater  r\  rG  rd  r^  rp  rn  rt  rk  r  r  r  r  r  r  r  rp  r   r  r  r  r  r  r  r  r  r  r"  r5  r<  rH  rK  r_  rY  ro  rm  r  r  r{  r  r  r  r  r  r  r  r  r  r  r  r  rx  r  r  r,  r1  r  r6  r8  r:  r>  r<  Tensor_TensorrG  ScalarrE  Tensor_ScalarrF  r  rT  rZ  rW  rZ  r  r_  r  r|  r   rf  rx  true_dividerz  r  r  rh  r  r  r  r  r  r  r  r  r  r  r  ri  r   r  rX  r  rW  r  r  rv  r  rw  r  r  r  r  r  rr   r  r  r  r  r  rD  expm1relur  rC  r  r4  cossinabsbitwise_andbitwise_left_shiftbitwise_not
bitwise_orbitwise_right_shiftbitwise_xorlgammaerfspecial_erfr  tantanhra  r  r  logical_xorr"  r#  	clamp_min	clamp_maxnegr!  	remaindersignsignbit	_neg_viewler  r  r  r  necoshsinhacosacoshasinasinhatan2atanatanhru  erfcerfinvhypotlog10log2	nextaftercodegen.commonr:  r  r  r   r   r   r'  _foreach_addforeach_add_listforeach_add_scalar_foreach_mulforeach_mul_listforeach_mul_scalar_foreach_sub_foreach_neg_foreach_abs_foreach_powScalarAndTensor_foreach_divforeach_div_listforeach_div_scalar_foreach_sqrt_foreach_maximum_foreach_minimum_foreach_clamp_min_foreach_clamp_max_foreach_reciprocal_foreach_sign_foreach_copyr  _foreach_add__foreach_mul__foreach_div_r  add_bitwise_and_bitwise_left_shift_bitwise_not_bitwise_or_bitwise_right_shift_bitwise_xor_mul_div_Tensor_modelogical_and_logical_not_logical_or_logical_xor_sub_relu_sigmoid___and__
__lshift____or__
__rshift____xor____iand____ilshift____ior____irshift____ixor__r  r  r  r  r8  methodfuncr  _inductor_testr<  r  inductorr  r  source_Tensorr  fsdpr  r  *torch._higher_order_ops.auto_functionalizer  r'  higher_orderre   r4  SubgraphrB  rE  rO  )torch.distributed._functional_collectivesr   
all_reducerY  rU  r[  all_reduce_coalescedr^  r]  r`  rc  re  rg  rh  rj  rk  rm  rn  rp  rq  rs  rv  r   rz  rx  r|  r  r  r  r  r  rx  ImportErrorr  r  r  register_quantized_opsregister_woq_mm_opsr  register_onednn_fusion_opsr  register_jagged_opsr   rV   rT   <module>r     s6        	  # I I I    +  $ $ H U    M  - 8 8 =   
 
 
   g!=?	4

%%xS'99: ?  4	JJ8HS#X$677  ),	3uzz$$% -yy~~
))

		47E s5::001 9*-%S&&' /25% S../ 7NP ejj33UZZ5J5JJK Pyy55 -#s(!3 -#s(AS8T -6MF

3   !!$$--&&* {{zz{{{{{{}}}}}}

. "#3R  "J20j 7??	$#0)\  $!%WtM`E	 E%++ E 5--4H)Y )u{{ ) I) @E 5	 5%++ 5( 499??=&9 &U[[ & >& ;@ = =ELL = 5##>+9 +ell + ?+ 
7?? $)X>    4::DI J0 4))UPTU V" DJJT\\499emmTU V 4&doo&s+ 4<<T:B ;B, 4$$$?" @" DMM?# $ 4::C C 4::C C 499! ! 4::! ! 4::%%&% '% 4::! ! 4;;D9> :>2 5))tD
 E
 4>>t<# =# 4;;.  .b 4$$$?499$74<<T:1 ; 8 @1 4<<T:> ;> 4::48V 9V 4??=> >>$ 4##> ? 4''TB C
BJ '<<RVW/// / 	/
 / / ;;/ / X/d //T))) ) 	)
 ) ) ;;) ))X ,,44$!!! ! 	!
 ! ;;! !!H ..66D  	
  ;; < ,,33*** * 	*
 * ;;* **Z ..554&&& & 	&
 & ;;& &&R 488v: v:r 4==d;3EC 3E3 3E# 3E <3El 4%%4@6 6 6s 6 A6 4((dC  s  D 4;;D96 :6
 4::48 9$ 4((dC- D- 4;;D9 : 4;;D9D :D4 4>>t< = 4??= > 488  T  3 3"< "<J,N^	, 599%%11tL#+ M#+L 4&&DAG BG 4??= > 4>>##>' ?' 4<<  ! QB B/
 )):):; *499+>+>? )$***<*<= +DJJ,@,@A  dll  499J J 4::J J >44$OL PL
 >&&DAK%,, K BK >''TBJ CJ
 >--4H	 I	 >((dCRS $s) 9 C C  D6 >))tDLM	#Cy09FI E8 4>>t<
 --- 	-
 - =-`"  
<@ d'' ( d&& ' d(( ) d** + dll ' d&&U 3 d!! " d,,5 9 d   d!! " d"" # d   d   doo  d!!)) * d++33 4 djj  dnn## $ d))11 2 d,,44 5 dkk  d$$5 1 d'')@ A doo} - d&&(: ; d!!#5 6 d..0B C d** + d== > d== > d22 3 d== > d33 4 d00- @ d00 1 d// 0 d// 0 d11 2 d11 2 d-- . d-- . d-- . d..0B C d00 1 d++] ; d"" # dii  dii  dmm  dii  dii  dkk  dnn  dmm  dll  doo  d  d-- . d   ! d'' ( d## $ dnn  d&& ' d"" # d$$ % dnn  d"" # d## $ d** + d  dnn  djj  d'' ( d// 0 d   d## $ d%% & d## $ d!! " djj  dmm  dll"" # dhhU + d((- 8 ell$$;; < ell$$77 8 d!! " d** + d""$6 7 djj  d'' ( d;; < dnn  doo  djj  0088	
 99AA	
 ,,44	
 55==	
 ,,44	
 55==	
 44<<	
 ==EE	
 d++33_ E d,,44o F d//77 I d0088/ J d  
 499$7
 8
 4::"  ( 4"#+d**+E2 5:: ( 4&&DA C  B: 4%%4@@ A@F ELL$"4"456tDU 5 75p 5??#4 $4 5##$+ %+ 4++,*" -*"Z 4&&' (< 4>>t<K =K8 EKK,- 
 .$.> 0t/0B50IJ
1!45	 21 56
 4>>" $T$4  # 4%%&Td '< 4))*"4	 +	 5%%--.A /A EJJ		*+: ,:
 4;;D9" :"J 4>>t< =8#0DN!1H 4::48
 9
 4%%4@/ A/ 4>>"= #= 4))*O +O;  4??=J >J >44$OK PKXv !1%%5!  0@,,44%0 ,
 4,,$G H( 4;;QUV	P W	P 1 1 4**E F 4<<T:9C 9 ;9 !&& 
 SM B 4==d;DH : :HSM : <:" 4##>3 3 ?3 4$$$?6 6 @6 4&&DAP3 P BP 4''TBPT ms m m Cmh , HUOS(), 	,
 ,^ 4**223>x > 4> 41199:Jhuo J ;J 4**223RVI&uoI@HI 4I 41199:RVU&uoU@HU ;U 4**223 !% $ $S uoS uo	S
 uoS 4S 41199: !% $ $	 uo	 uo		
 uo	 ;	7 599$$% &* 4''TB. C.bejj uUZZ_'= uzz  %** EJJ UZZ  :><,> CG@: 5<<RVW E XEP 	33B d** + -=))11- ) 488dSS TSl:B&R2j  0%%5  
 4,,-? .?D  0$$%  
 4++,X -Xv "2&&E" 
2 4--.B /BJ 433;;<BF* =*Z 'OO  'OO 
 4??=  >* 4??=  >*pf  0$$%  
 4++F ` G`F  0$$%  
 4++F @ G@F$.b3 & 499:t : : 4 (V4( DHHeii()T5  * 4==!u  "   .HH  'txxER -HH 
 488t,, -,^ F 4::2 2 4::48 9     488t, - DHH:.( /(")) (= > EII;$/$ 0$( 	txx'7DD

 DIIuzz*d;$ <$  4::% % DHHeii(). . *. #4;;#6#67#DLL$8$89 ():):)B)BC "4;;#6#67"4;;#6#67 4;;  . 4<<  !. 4$$% &* 4;;D9 :: 4;;D9 :: 499. . . 488@ @
 48867 77 48867 77 ! %--  	!: ;*		*>%+@A*		*>%+@A.!$++.85;;? /!$++.85;;? HH$L !!1!1uM 499##>!re $2 ?$2N 499$$$?H @H 'txx0!$)),"4::.$))$
*4<<
8'		2	DKK	(t4   *   *" !1!12'(?(?@  -  0
()A)AB  !1!12 4;; ' * *I*V*V 4:: & 488 $ 499 %   * **	 !**	  OO**	
 !**	 T\\
*
T\\
* ! $.. !' * ! $.. !' *""'8
 4>> "$))L 499  4<<uzz B ! $.. !# & 477%** = 477%** = 477%** =uzzB 477%** = 477%** = 499 % 499 % 499 % 4:: & 499 % 4:: & 4:: & 499 % 4:: & 4== ) 499 % 4;; ' 4:: & 4:: & 499 % 4>> * D DE$ % 
D4Ld5 
0 	 3+		

 5Mt5 
0 	 3+		


, .CT  0ct  4,,33Sd K-d.?.?.D.DcJ /0A0A0H0H#N  4,,113 7 4,,33S 9 4,,44c : 4,,44c : 4,,33S 9 4,,<<c B-d.?.?.D.DcJ /0A0A0H0H#N  4--t 4 40055w ? 40077 A 40055w ? 40077 A 42277 A 422997 C 42277 A 422997 C 433Z @ 4--t 4 4--t 4, T..335E t00779K T..335E t00779K T..335E t00779K
 C   ""K 0 ))+= > ""K 0 !!: . **,? @ ""K 0 C   !!3 ' && 1 ""K 0 ""K 0 !!: . ""K 0 C   T "  (   $,,  , " $// "#5 6  $++ z * " $// "#6 7  $,,  ,  - !!4?? 3 t{{ + !!4?? 3  - 4++, - 4==$$% &  4??&&' ( 4>>" # (M'') 8LFD1(01$78 4<< 7 !7 599++334 5
 599%%;;< = 599>>&&445K 6K 599>>6"uyy~~**2236 46 599>>(()%) = *=@ K J ! " 12	S 3	S 599))..// 0/ 599))445/ 6/ &DA #  B8 599??//778 9 599))667+ 8+&[4yy11'223 4 '334 5 '<<= > '==> ? '>>?
 @
 'HHI	
 J	
 'BBC D '==>	
 ?	
 'GGH

 I

 '99:	
 ;	
 '112 3 '223 4 '334 5 uyy))<<=	
 >	
"     ! ! +  * * , '  ' ' )   ,  + + -   %  $ $ &3 	$ HH\s   [F9Cd dCd"d!Cd"