
    ɯwg~                      d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$Z%ddl%m&Z& ddl'm(Z( dd	l)m*Z*m+Z+ dd
l,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z> ddl?m@Z@ ddlAmBZB ddlCmDZE ddlFmGZG e!rddlHmIZI ddlJmKZK eLZMeLZNeLZO e"dd      ZPe%j                  j                  j                         r	ddlTmUZVmWZW n G d d      ZVdOdZWe%j                  j                  eZd      Z[ddl\m]Z]  ej                  d       G d d             Z_dPd Z`ej                  dQd!       Zbej                  dQd"       Zcej                  dQd#       ZddRd$Ze G d% d&      Zf e
j                         Zgi eg_h         ee
j                        eg_j         G d' d(      Zke%j                  j                  d)egj                         e%j                  j                  d*egj                         dPd+ZndPd,ZodSd-ZpdTd.Zq	 dU	 	 	 	 	 dVd/Zr	 	 	 	 	 	 	 	 	 	 	 	 dWd0Zs	 dXdd1d1d1d2	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dYd3Zt G d4 d      ZudZd5Zv	 	 	 	 d[d6Zwej                  	 	 	 	 	 	 	 	 d\d7       Zxd]d8ZyeeLeLf   Zzeee{      Z|eee}      Z~ G d9 d:      ZeZeZeZ G d; d<      Z G d= d>e      Z e       Z G d? d@e      Z G dA dBe      Z G dC dD      Zd^dEZdUd_dFZd`dGZ	 	 	 	 	 	 	 	 dadHZ G dI dJe      Z G dK dLe      Z G dM dN      Zy)ba  
CUDA graph trees are a safety abstraction over CUDAGraphs, similar to make_graph_callables,
which share the same memory pool.  Sharing a memory pool is an extremely
important optimization when chaining multiple CUDA graphs together, as it
prevents you from needing to copy intermediate tensors from one graph to the
next, and reduces overall memory usage by allowing dead memory from the first
pool to be reused in the second.

The standard graph/make_graph_callables support sharing memory pool, but
with a lot of caveats.  CUDA graph trees remove these restrictions:

* Previously, if you recorded graphs A, B, you had to replay A, B in that
  order.  With CUDA graph trees, after replaying A, you can change your
  mind and record/replay a different graph B'; we will support efficient
  execution of both A, B and A, B', using only max(mem(A, B), mem(A, B')).  In
  other words: we support arbitrary trees of CUDA graph operations, not just
  sequences (this is why this feature is called CUDA graph trees.)

* Previously, if you executed graph A, some non-CUDA graph code, and then
  graph B, after executing graph B, it was not safe to retain any references
  to intermediates produced by A.  With CUDA graph trees, we track if any
outputs of graph A are still live by the time graph B is run, and make
  sure graph B doesn't clobber there memory when reusing the CUDA graphs
  pool.  You'll get a separate recording of B depending on what tensors
  stay live or dead.

CUDA graph trees are flexible enough to be used in Dynamo across graph breaks,
which is their primary use case.

The ability to switch from replay to record is fairly nontrivial: remember that
when you replay a CUDA graph, you only replay CUDA operations; no CPU side state
is updated.  In particular, the CPU-side book-keeping for the allocator is not
reconstructed.  However, to record a new child CUDA graph, we must restore this
book-keeping.  This is what checkpoint pool state is used for.
    )annotationsNdefaultdict)autoEnum)AnyCallablecastContextManagerDict	GeneratorIteratorListOptionalSequenceSetTupleTypeTYPE_CHECKINGTypeVarUnion)Tensor)GenerationTracker)counterspreserve_rng_state)align_inputs_from_check_idxscopy_misaligned_inputsget_expanded_dimsget_input_idxs_to_checkindex_expanded_dimsremove_unaligned_input_idxsstatic_input)
check_for_mutationCheckInvariantStatus
FunctionID#log_cudagraph_skip_and_bump_counterlog_data_ptr_mismatch"maybe_warning_due_to_dynamic_shape	ModelType
OutputTypePlaceholderInfoWrappedFunction)StorageWeakRef)UntypedStorage)_pytree)TensorWeakRef)	InputType)_boolSStorageWeakRefWrapper)bound)"_cuda_CUDAAllocator_AllocatorState_set_cached_tensors_enabledc                      e Zd Zy)AllocatorStateN__name__
__module____qualname__     d/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/torch/_inductor/cudagraph_trees.pyr9   r9   w   s    r?   r9   c                     y Nr>   enableds    r@   r7   r7   z   s    r?   
cudagraphs   )configT)frozenc                      e Zd ZU dZded<   y)GraphIDz(Unique counter of a cuda graph recordingintidN)r;   r<   r=   __doc____annotations__r>   r?   r@   rJ   rJ      s
    .Gr?   rJ   c                 @    t         j                  j                          y)a}  
    Cublas keeps a persistent workspace allocation for running matmuls. This poses a problem for
    doing warmup within a CUDAGraph private pool because we do not want persistent allocations from
    one one run to the next. When we begin a new run of a cudagraphs path (generation), all tensors
    from the previous generation are freed. This frees them the memory pool, but not elsewhere.
    A tensor in the cublas workspace would continue to be in use the workspace but would also get allocated
    in the next run. The memory would be in use in two places.

    To solve this, we clear cublas caches before and after warming up or recording. If a workspace is required
    it will be allocated to the cudagraph private pool and accounted for in the allocator for the duration of the
    program. There is no overhead to this on replay since cudagraphs removes allocation overhead.
    N)torch_C_cuda_clearCublasWorkspacesr>   r?   r@   clear_cublass_cacherS      s     
HH((*r?   c               #  \   K   t                	 d t                y# t                w xY ww)zOContext manager around clearing cublas caches that will clear on enter and exitN)rS   r>   r?   r@   clear_cublas_managerrU      s#      s   , ,),c               #    K   t         j                  j                         } t         j                  j                  d       	 d  t         j                  j                  |        y # t         j                  j                  |        w xY wwNF)rP   rQ   $_cuda_get_conv_benchmark_empty_cache%_cudnn_set_conv_benchmark_empty_cache)prevs    r@   disable_conv_cache_emptyingr[      sV     8888:D	HH2259=66t<66t<s   >B	A%  B	%!BB	c               #  `  K   t         j                  j                         } 	 | s(t         j                  j                  j                          d | s*t         j                  j                  j                  d       yy# | s*t         j                  j                  j                  d       w w xY ww)z8Turns on history recording in the CUDA Caching AllocatorN)rP   rQ   _cuda_isHistoryEnabledcudamemory_record_memory_historyrC   s    r@   enable_history_recordingra      s{      hh--/G;JJ446JJ44T: wJJ44T: s   B..A= -B.=.B++B.c                 r    t         j                  j                  st        j                         S t               S rB   )rG   triton!cudagraph_trees_history_recording
contextlibnullcontextra   r>   r?   r@   get_history_recordingrg      s'    ==::%%''#%%r?   c                  @    e Zd ZdZd	dZd
dZd
dZd
dZddZddZ	y)TreeManagerContainera  
    Manages the lifetime of the tree manager. Like `PrivatePool` in cuda caching allocator,
    the tree and its corresponding memory pool should be kept alive as long as any outstanding
    graph or tensor which is an output of a graph remains alive.

    There is a single tree manager container per device.

    The lifecycle of a tree_manager is:
    -  Is constructed, no graph, no fns, no tensors
    -  Tree manager is fetched, resulting in tree manager being allocated
    -  We generate a bunch of functions, calling add_strong_reference
    -  These functions die, calling finalize_reference
    -  When all the functions die, we finalize_tree_manager.

    TODO: in the future, we would like to do the following once storage weak refs land
    -  We look for all the live storages and add references to THOSE
    -  We count as storages die
    -  All the storages are dead, we deallocate the tree manager
    c                |    d | _         d| _        || _        d| _        d | _        t        j                         | _        y Nr   )tree_managerlive_cudagraphify_fnsdevice_indexlive_storages_countgraph	threadingLocklockselfrn   s     r@   __init__zTreeManagerContainer.__init__   s>     =A &'"(
 $% 59
NN$	r?   c                    | j                   5  | xj                  dz  c_        | j                  dk(  rd | _        | j                  dk(  rd | _        d d d        y # 1 sw Y   y xY wNrF   r   )rs   ro   rp   rm   rl   ru   s    r@   _finalize_tensorz%TreeManagerContainer._finalize_tensor   s\    YY 	-$$)$''1,!
 --2(,D%	- 	- 	-s   AAA!c                    | j                   5  | xj                  dz  c_        | j                  dk(  r| j                          d d d        y # 1 sw Y   y xY wrx   )rs   rm   _finalize_tree_managerry   s    r@   finalize_cudagraphify_fnz-TreeManagerContainer.finalize_cudagraphify_fn   sI    YY 	.&&!+&))Q.++-	. 	. 	.s   5AAc                J    | j                   j                         sJ d | _        y rB   )rs   lockedrl   ry   s    r@   r|   z+TreeManagerContainer._finalize_tree_manager   s     yy!!! r?   c                    | j                   5  | xj                  dz  c_        d d d        t        j                  || j                         y # 1 sw Y   *xY w)NrF   )rs   rm   weakreffinalizer}   )ru   fns     r@   add_strong_referencez)TreeManagerContainer.add_strong_reference  sG    YY 	,&&!+&	, 	T::;	, 	,s   AAc                    | j                   5  | j                  t        | j                        | _        | j                  cd d d        S # 1 sw Y   y xY wrB   )rs   rl   CUDAGraphTreeManagerrn   ry   s    r@   get_tree_managerz%TreeManagerContainer.get_tree_manager  sF    YY 	%  ($89J9J$K!$$	% 	% 	%s   2A		ANrn   rK   returnNoner   r   )r   zCallable[..., Any]r   r   )r   r   )
r;   r<   r=   rM   rv   rz   r}   r|   r   r   r>   r?   r@   ri   ri      s%    (%&	-.!0<%r?   ri   c                      e Zd ZdZy)MarkStepBoxr   N)r;   r<   r=   mark_step_counterr>   r?   r@   r   r   (  s    r?   r   tree_manager_containerstree_manager_locksc                 6    t         xj                  dz  c_        y)zJIndicates that a new iteration of inference or training is about to begin.rF   Nr   r   r>   r?   r@   mark_step_beginr   2  s     !!Q&!r?   c                 t   t        t        d      } t        t        d      }|j                         D ]S  \  }}|5  | j                  |      }|r|j                  s
	 ddd       2|j                  j                          ddd       U t        d       | j                          dt        _	        y# 1 sw Y   xY w)zClear all cudagraph treesr   r   NFr   )
get_objlocalitemsgetrl   shutdownr7   clearr   r   )container_dict
locks_dictdevicers   	containers        r@   reset_cudagraph_treesr   9  s     U$=>N 45J"((* . 	.&**62II$:$:	. 	.
 ""++-	. 	..  &$%K!	. 	.s   !B.#B..B7	c                    t        | |      rt        | |      S t        j                  j	                  |      sJ t        j                  j                  |      S rB   )hasattrgetattrrP   rQ   _is_key_in_tls_get_obj_in_tls)r   	attr_names     r@   r   r   L  sF    ui ui((xx&&y111xx''	22r?   c                    t        t        d      }t        t        d      |    }|5  | |vrt        |       || <   ||    cd d d        S # 1 sw Y   y xY w)Nr   r   )r   r   ri   )rn   r   rs   s      r@   get_containerr   T  sX    U$=>N5./=D	 ,~-+?+MN<(l+	, , ,s   AAc                b    |rt        |       j                         S t        |       j                  S rB   )r   r   rl   )rn   create_if_none_existss     r@   get_managerr   _  s,     \*;;==&333r?   c                    	
 i 	t        |      D cg c]  \  }}t        |t              s| }}}|rt        j                  | nd 
d~d	
 fd}|S c c}}w )Nc                     y rB   r>   _s    r@   <lambda>z#cudagraphify_impl.<locals>.<lambda>r      r?   Fc                Z    |       }j                  |      }| ||       S |t        j                  d       nt        j                  d|       	st        |      	t	        |       }t        |       }t        | |       t        | |gi 
\  }}t        ||      }||<   |S )Nz2recording cudagraph tree for graph without symintsz*recording cudagraph tree for symint key %s)inputs_to_check)	r   loginfor(   r   r!   r   cudagraphifyr   )inputsint_keyr   check_input_idxsnew_static_input_idxsoutargsfn_cacheget_intshas_warnkwargsmodelstatic_input_idxss         r@   deferred_cudagraphifyz0cudagraphify_impl.<locals>.deferred_cudagraphifyx  s     6"\\'">f:?HHIJHHA7K9(GLH 36;LM ;FDU Vv'78uf.CUdUfUC)">NO
r?   )r   List[InputType]r   r*   )	enumerate
isinstancerK   operator
itemgetter)r   r   r   r   r   ivr   r   r   r   r   s   ` ```    @@@r@   cudagraphify_implr   g  sf     ;=H 'v.ETQ*Q2DqEGE5<H''1.HH 8 ! G Fs
   A A r>   )stack_traces	constantsplaceholdersmutated_input_idxsc          
         t        |      j                         }
|r|rJ |rt        j                  n!|rt        j                  nt        j
                  }|
j                  | |||||||	      S rB   )r   r   CompilationModeBACKWARD	INFERENCEFORWARDadd_function)r   r   r   rn   is_backwardis_inferencer   r   r   r   managermodes               r@   r   r     sw     L)::<G--  	  +7o''_=T=T 	 	 	r?   c                      e Zd ZU dZg dZded<   	 d	 	 	 	 	 ddZe	 d	 	 	 	 	 	 	 	 	 dd       ZddZ	dd	Z
dd
ZddZddZddZy)r4   z[
    Wrapper around a storage weak ref. Will deallocate it upon expiration if invoked.
    )ref	_data_ptrextra_ref_checkzOptional[StorageWeakRef]storage_refNc                    t        |t              r|j                         }nt        |t              sJ |}t	        |      | _        |j                         | _        || _        y)z
        extra_ref_check is an additional check we need to run to check if the
        weak ref has expired. in checking storage use count we assume extra_ref_check
        will hold an additional reference to the storage.
        N)	r   r   untyped_storager.   r-   r   data_ptrr   r   )ru   inpr   stors       r@   rv   zStorageWeakRefWrapper.__init__  sR     c6"&&(Dc>222D!$'.r?   c                x    | j                  |       }||_        t        j                  |      |_        ||_        |S rB   )__new__r   r-   from_weakrefr   r   )clscdatar   r   instances        r@   from_weakref_and_data_ptrz/StorageWeakRefWrapper.from_weakref_and_data_ptr  s9     ;;s#%%2259#2 r?   c                P    | j                         ry | j                  j                  S rB   )expiredr   r   ry   s    r@   __call__zStorageWeakRefWrapper.__call__  s    <<>xx~~r?   c                Z    | j                   j                          || j                   _        y rB   )r   __del__r   )ru   r   s     r@   swap_weakrefz"StorageWeakRefWrapper.swap_weakref  s    r?   c                    | j                   S )z8NB: returns the data ptr even if the storage has expired)r   ry   s    r@   r   zStorageWeakRefWrapper.data_ptr  s    ~~r?   c                    d | _         y rB   r   ry   s    r@   remove_extra_referencez,StorageWeakRefWrapper.remove_extra_reference  s
    #r?   c                    | j                   | j                         syt        j                  j                  | j                  j
                        }|| j                   d uz
  dk(  S )NFr   )r   rP   rQ   _storage_Use_Countr   r   )ru   
stor_counts     r@   r   zStorageWeakRefWrapper.expired  sU    +D4H4H4J XX00@
d22$>?AEEr?   c                    | j                   | j                   j                         rd| j                          dS d| j                          dS )NzStorageWeakRefWrapper to z; deadz; alive)r   r   r   ry   s    r@   __repr__zStorageWeakRefWrapper.__repr__  sG    88txx//1.t}}.?vFF.t}}.?wGGr?   rB   )r   zUnion[Tensor, UntypedStorage]r   Optional[Callable[[], bool]]r   r   )
r   zType[S]r   r   r   rK   r   r   r   r4   )r   zOptional[StorageWeakRefPointer])r   r   r   r   r   rK   r   r   bool)r   str)r;   r<   r=   rM   	__slots__rN   rv   classmethodr   r   r   r   r   r   r   r>   r?   r@   r4   r4     s     8I))
 9=/*/ 6/ 
	/& 
 9=	


 
 6	

 

 
$FHr?   c                    t        |       d uS rB   )maybe_deref)weak_refs    r@   is_liver    s    x ,,r?   c                @    | y  |        }|y || j                         fS rB   )r   )r  rs     r@   r  r    s0     
Ayh!!!r?   c              #    K   t         j                  j                          |j                  t         j                  j	                                t         j                  j                  |      5  t        j                  |       5  t         j                  j                  | |       	 d t         j                  j                  | |       t         j                  j                  | |       	 ddd       ddd       t         j                  j	                         j                  |       y# t         j                  j                  | |       t         j                  j                  | |       w xY w# 1 sw Y   xY w# 1 sw Y   xY ww)a}  
    Context manager to use cuda graph pool for new allocations. If you use this manager
    all cudagraph tensors in use should be reflected in the allocator or they will be overwritten.
    existing_graph should already have been used in a capture, and the mem_pool must already exist,
    because this manager will not preserve a reference to the pool which keeps it alive.
    N)rP   r^   synchronizewait_streamcurrent_streamstreamr   rQ   &_cuda_beginAllocateCurrentStreamToPool$_cuda_endAllocateCurrentStreamToPool_cuda_releasePool)r   mem_poolr
  s      r@   _use_cuda_memory_pool_managerr    s     
JJ
uzz0023			6	" 9ELL$8 977I	9HH99&(KHH&&vx89 9 
JJ++F3 HH99&(KHH&&vx89 9 9 9sP   A+F-E9!E-%D()A E-*E926F(AE**E--E6	2E99F>Fc                V    t        | t        j                        s| J y t        |       S rB   )r   rP   r   r4   )ts    r@   
map_to_refr  )  s(    a&yy ##r?   c                  v    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	dZd
dZe	 	 dd       ZddZddZ	ddZ
y)CUDAWarmupNodeaJ  
    Simplified Wrapper around A CUDA Model that wraps outputs in storage refs and exposes
    apis to get the live storages in the current chain of warmup.

    A CUDAWarmupNode may have either CUDAGraphNode or CUDAWarmupNode as a parent, but may only have
    CUDAWarmupNode as children, because we cannot record or execute with tensors which do not have stable
    memory addresses.

    CUDAWarmupNode and CUDAGraphNode have a number of differences that make it easier to use separate classes.
    - Much of the CUDAGraphNode logic & initialization is based on the tensor properties of first recording. In the
    first instance of warmup, these are not finalized yet.
    - All Inputs to the RecordedFunction must be copied over to the cuda graph memory pool, this is unnecessary in warmup.
    - CUDAWarmup is only used once and so does not need to optimize as much bookkeeping. It is much simpler.

    NB: this class and CUDAGraphNode need to expose `path_live_weakrefs`, `all_outputs_are_dead`, and
    `self.outputs_weakrefs`, `stack_traces`, and `tensor_weakrefs` for compatibility.
    c
                    || _         || _        || _        g | _        g | _        || _        d| _        || _        || _        || _	        || _
        |	| _        y rW   )wrapped_functionparentcuda_graphs_pooloutputs_weakrefstensor_weakrefsexisting_cuda_graphhas_runrn   r   r
  already_warmrL   )
ru   r  r  r  r  rn   r   r
  r  rL   s
             r@   rv   zCUDAWarmupNode.__init__M  s`     !1FL 0GI>@#6 (((r?   c           
          j                   rJ d        j                         D ch c]  } |       s|j                          c}d fd} |       }t        j                  j
                  rF j                  s:t         j                               }t         j                   j                  |       t        j                  j                   j                        5  t               5  t               5  t!         j                   j                   j"                        5  t%               5   j&                  j)                        }d d d        d d d        d d d        d d d        d d d        t+               |D ]'  } |       }|j-                  |j.                         ) t1              dk(  sJ dfd}	 j2                  j5                  D 
cg c]  }
 |	|
      rt7        |
      nd  c}
        j8                  j5                  |D 
cg c]  }
 |	|
      rt;        |
      nd  c}
       t        j                  j
                  rF j                  s:t         j                               }t         j                   j                  |       |S c c}w # 1 sw Y   UxY w# 1 sw Y   ZxY w# 1 sw Y   _xY w# 1 sw Y   dxY w# 1 sw Y   ixY wc c}
w c c}
w )Nz*Wrapped function should never be run twicec                 F   g } t        j                  j                  j                        D ]p  }t	        |t
        j                        s|j                         j                         vs?| j                  t        j                  |j                                      r | S rB   )	itertoolschainr  r   r   rP   r   r   r   appendr   r   )non_cudagraph_inpsr  existing_path_data_ptrs
new_inputsru   s     r@   get_non_cudagraph_inpsz2CUDAWarmupNode.run.<locals>.get_non_cudagraph_inpso  s    !#__Z1F1F1P1PQ Pq%,,/))+446>UU&--gkk!:K:K:M.NOP &%r?   r   c                    t        | t        j                        xrM | j                  xr? | j	                         j
                  vxr! | j	                         j                         dk7  S rk   )r   rP   r   is_cudar   _cdatar   )onon_cudagraph_inps_storage_ptrss    r@   add_refz#CUDAWarmupNode.run.<locals>.add_ref  s`    1ell+ 8II8%%'..6UU8 %%'002a7	r?   )r   z+List[weakref.ReferenceType[UntypedStorage]])r*  r   r   r   )r  path_live_weakrefsr   rG   rc   slow_path_cudagraph_assertsr  listcheck_memory_poolrn   r  rP   r^   r   r[   rU   r  r
  rg   r  r   setaddr)  lenr  extendr  r  r0   )ru   r%  r  r&  non_cudagraph_inps_storagesrefsr   storagesr,  r*  out_refsr$  r+  s   ``          @@r@   runzCUDAWarmupNode.runf  sp   <<M!MM
 #'"9"9";#
qsAJJL#
	& '=&>#==44T=N=N//12Dd//1F1FMZZ
 	:&(	:*>*@	:B_t44dkkC
	: !"		:
 ''--j9C	: 	: 	: 	: 	: +.%'2 	>G	A}/33AHH=	>
 :!###	 	$$<?@qgajZ]d2@	
 	##?BC!]15C	
 ==44T=N=ND3356Hd//1F1FQ
{#
(	: 	: 	: 	: 	: 	: 	: 	: 	: 	:> A Ds   JJ$K/K:,J:&J-	1J J-	J:K%KK!K& J*%J-	-J72J::K?KK	KKc              #     K   g }| }|r |j                  |       |j                  }|r t        |      E d {    y 7 wrB   )r"  r  reversed)ru   nodesnodes      r@   _path_from_rootzCUDAWarmupNode._path_from_root  s>      59LL;;D  E?""s   &??=?c              #  p   K   | j                   D ]#  }|j                  D ]  }t        |      s|  % yw)zEReturns all live storages weakrefs that created by nodes in this pathN)r?  r  r  )ru   r>  outputs      r@   r-  z!CUDAWarmupNode.path_live_weakrefs  s<     (( 	!D// !6? L!	!s   *6	6c                6    t        | j                                S rB   )r/  r-  ry   s    r@   all_outputs_are_deadz#CUDAWarmupNode.all_outputs_are_dead  s    //1222r?   c                    | j                         D ]3  }|j                         j                         |j                         k(  s3 y yNTF)r-  r   r   )ru   r  storage_weak_refs      r@   _is_cuda_graph_recorded_tensorz-CUDAWarmupNode._is_cuda_graph_recorded_tensor  sG     $ 7 7 9 	  "++-1A1J1J1LL	 r?   N)r  r,   r  .Optional[Union[CUDAGraphNode, CUDAWarmupNode]]r  Tuple[int, int]r  zOptional[torch.cuda.CUDAGraph]rn   rK   r   Optional[StackTraces]r
  torch.cuda.Streamr  r   rL   rJ   r   r   )r%  r   r   r*   )r   z;Generator[Union[CUDAGraphNode, CUDAWarmupNode], None, None]r   zIterator[StorageWeakRefWrapper]r   r  torch.Tensorr   r   )r;   r<   r=   rM   rv   r:  propertyr?  r-  rC  rG  r>   r?   r@   r  r  :  s    $) ? *	
 <  , "   
2BH 	#	D	# 	#!3r?   r  c                      e Zd Zy)OutputAliasInfoNr:   r>   r?   r@   rQ  rQ    s    r?   rQ  c                      e Zd ZdZy)_UnaliasedStoragezISingleton to mark that the graph output constructs a new alias or is NoneN)r;   r<   r=   rM   r>   r?   r@   rS  rS    s    Or?   rS  c                  *    e Zd ZU dZdgZded<   ddZy)AliasesPriorGraphOutputz>Marks that the graph output aliases an output of a prior graphindexPathOutputIndexc                6    t        |t              sJ || _        y rB   )r   tuplerV  ru   rV  s     r@   rv   z AliasesPriorGraphOutput.__init__  s    %'''
r?   NrV  rW  r   r   r;   r<   r=   rM   r   rN   rv   r>   r?   r@   rU  rU    s    D	Ir?   rU  c                  *    e Zd ZU dZdgZded<   ddZy)AliasesNewOutputzIMarks that the graph output aliases an index in the new, returned outputsrV  rK   c                6    t        |t              sJ || _        y rB   )r   rK   rV  rZ  s     r@   rv   zAliasesNewOutput.__init__  s    %%%%
r?   N)rV  rK   r   r   r\  r>   r?   r@   r^  r^    s    O	IJr?   r^  c                  2   e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d)dZ	 	 	 	 	 	 d*dZd+dZd,dZd,dZd-dZ		 	 	 	 	 	 d.dZ
	 	 d/d	Zd0d
Zd1dZd2dZ	 	 	 	 	 	 d3dZd4dZd0dZd5dZed6d       Zed7d       Zed7d       Zd8dZ	 	 	 	 d9dZe	 	 	 	 	 	 d:d       Zd;dZe	 	 	 	 	 	 d<d       Ze	 	 	 	 d=d       Z	 	 	 	 	 	 d>dZd0dZd0dZ d?dZ!d@dZ"d0dZ#d0d Z$d0d!Z%e	 dA	 	 	 	 	 dBd"       Z&	 dC	 	 	 	 	 dDd$Z'dEd%Z(	 	 	 	 dFd&Z)	 	 	 	 dGd'Z*dHd(Z+y#)ICUDAGraphNodear  
    A single recording of a function into a CUDA Graph. Recordings of CUDA Graphs share a single memory pool
    and are structured into a tree, where there is a single recording that can precede it (parent) and multiple
    subsequent recordings that may follow (children). A node will have no parent if it is the first recording
    in a tree; i.e., when it is first recorded, there are no live tensors from a previous recording which
    would force a dependency.

    On first recording, all of the live tensors in the current CUDA Graph Node path will be
    reflected in the corresponding private pool. On subsequent executions, the caching allocator
    is unaffected when the graph is replayed.

    In order to support recording a subsequent cuda graph recording after execution of this graph,
    we checkpoint the state of the memory pool so that it may later be resumed.

    WrappedFunction should have already been warmed up prior to invocation.

    See [setCheckpointPoolState] for further explanation, as well as
    https://user-images.githubusercontent.com/13564/222815509-374f3400-f83d-4f7d-8fa6-4a092b3250bb.png
    c	                T
   t        |t        t        f      sJ || _        || _        || _        || _        || _        t        j                  j                  j                  xs. t        j                  j                  j                  j                  | _        |t!        j"                  |      nd | _        || _        t)        t              | _        g | _        | j.                  D 	cg c]  }	|	j,                   c}	| _        | j.                  D 	cg c]  }	|	j                   c}	| _        g | _        t7        |      D 
cg c]2  \  }
}t        |t        j8                        r| j;                  |      r|
4 c}}
| _        t        t?        |j@                        t?        | j<                        z        | _         tC        tE        |            D cg c]  }|| j@                  vs| c}| _#        tH        d   dxx   tE        | jF                        z  cc<   |j@                  D cg c]  }|| j<                  vr| c}| _%        	 	 	 	 	 	 	 	 dd}tC        tE        |            D cg c]  } |||| j@                         c}| _&        t7        |      D 
cg c]:  \  }
}t        |t        j8                        r|
| j@                  vrtO        |      ng < c}}
| _(        g | _)        g | _*        g | _+        g | _,        g | _-        | j\                  Q| j\                  jT                  }| j_                  | j0                        }| ja                  ||      }|| _)        || _+        | jc                  |      }|je                          ~t        jf                  ji                         | _5        |D cg c]>  }t        |t        j8                        r | jm                  | jo                  |            n|@ c}| _8        d | _9        g | _:        g | _;        g | _<        g | _=        | j}                  |j~                  |      | _@        g | _A        | j                  J | j                  D ]  }t        |t        j8                        r-| j                  j                  | jo                  |d             Jt        |t        t        d       f      sJ t        |             | j                  j                  |        | jj                  j                          y c c}	w c c}	w c c}}
w c c}w c c}w c c}w c c}}
w c c}w )Ninductor$cudagraph_recorded_non_static_inputsc                j    ||    }t        |t        j                        r| |v r|j                         S y rB   )r   rP   r   r   )idxr   r   r   s       r@   maybe_get_static_data_ptrz9CUDAGraphNode.__init__.<locals>.maybe_get_static_data_ptrV  s2    
 +C#u||,8I1I||~%r?   F)ignore_storage_offset)rf  rK   r   List[Union[torch.Tensor, int]]r   	List[int]r   zOptional[int])Fr   r/  rY  r  rL   r   r   r
  rP   _dynamorG   inline_inbuilt_nn_modules	_inductorrc    cudagraph_support_input_mutation rerecord_if_static_inputs_changer   r   _parentr  r   childrenr  r?  path_weakrefspath_stacktracesr  r   r   rG  cudagraph_managed_idxsr1  r   ranger3  non_static_input_idxr   non_managed_static_input_idxsstatic_input_data_ptrsr   expanded_dimsrecorded_liveness_before_graphrecorded_liveness_after_graph"expected_dead_indices_before_graph!expected_dead_indices_after_graphlive_indices_after_graphr  _get_liveness_get_different_indices#_allocate_and_copy_recording_inputsr   r^   	CUDAGraphrp   !_reconstruct_from_tensor_metadata_tensor_metadatareconstructed_inputscheckpointed_caching_stateoutput_storage_aliasunaliased_in_all_pathscached_tensor_outputsstatic_output_tensors_recordr   recording_outputsoutputs_metadatar"  rK   typereplay)ru   r  rL   r  r   r  rn   r   r
  r>  rf  r  r   rg  xprevious_livenesscurr_livenessdifferent_indicesrecording_inputsr   s                       r@   rv   zCUDAGraphNode.__init__  s    &4-000 0"(
 MM  :: N%%,,MM 	- /5.@w{{6*d 0 @K4?P NP.2.B.BV
&*D!!V
 +/*>*>C
"&DC
 EG
 $F+2
Q!U\\*t/R/RST/U 2
# -1 223c$:U:U6VV-

 S[)5
Qd6L6L-LA5
! 	CD%%I
 	
D &77>
333 >
*		2	  )	 		 3v;'A
 &a1G1GHA
#$ $F+	/
 Q !U\\*s$:P:P/P a /
 LN+JL*
 JL/HJ. @B%;;" $ I I ..t/A/ABM $ ; ;!=! 3@D/6GD3CCFK 6;ZZ5I5I5K
 &	6
  !U\\* 2243H3H3KL6
!& EI' LN! 9;#CE"
 DF" 8<||""$48
 OQ
 %%111)) 	2C#u||,%%,,))#U)K "#T$Z'89D49D9%%,,S1	2 	

OV
C
2
5
>
 A
/
\6
s7   #T T<7T
TT T!T?T#AT%c                <   g }g }| j                   D ]p  }t        ||   t        j                        s!| j                  |   }|j                  t        ||   |             |j                  t        ||   |             d ||<   r |rt        j                  ||       y y rB   )rv  r   rP   r   ry  r"  r    _foreach_copy_)ru   dstssrcsdst_tensorssrc_tensorsrf  ry  s          r@    _copy_inputs_and_remove_from_srcz.CUDAGraphNode._copy_inputs_and_remove_from_src  s     ,, 	Cd3i6 ..s3M249mLM249mLMDI	   k: r?   c                H   | j                   st        j                  j                  || j                  | j
                        s_t        | j                  j                  || j                  | j
                  t        j                        t        j                  dfd       y y y )NFc                      S rB   r>   )	error_msgs   r@   r   z>CUDAGraphNode.check_static_inputs_are_stable.<locals>.<lambda>   s    	 r?   )ro  rP   rQ   #_tensors_data_ptrs_at_indices_equalrx  rw  r'   r  r   r$   StaticInputIdxMismatch_check)ru   r%  r  s     @r@   check_static_inputs_are_stablez,CUDAGraphNode.check_static_inputs_are_stable  s     55HH@@++22 .%%22++22$;;I LL 12 6r?   c                    t         j                  j                  r| j                          t	        |      dk(  sJ | j
                  }d | _        |J |S rk   )rG   rc   fast_path_cudagraph_asserts(debug_check_invariants_before_invocationr3  r  ru   r%  outputss      r@   run_first_inputszCUDAGraphNode.run_first_inputs  sS    ==4499; :!###((!%"""r?   c                   | j                  |       | j                  | j                  |       |j                          | j	                          | j                         }t        j                  j                  r| j                          t        j                  j                  rt        j                  j                          d| _        |S rW   )r  r  r  r   	run_graphreconstruct_outputsrG   rc   r  'debug_check_invariants_after_invocationforce_cudagraph_syncrP   r^   r  static_inputs_stabler  s      r@   r:  zCUDAGraphNode.run  s    ++J7--d.G.GT**,==4488:==--JJ""$ %*!r?   c           	        | j                   s| j                          g }t        t        | j                  | j
                              D ]  \  }\  }}t        |t              s/t        |t        t        d      f      sJ |j                  |       I| j                   |   }|%|j                  d|_        |j                  |       | j                  |   }|#| j                  |   J |j                  |       | j                  ||      }t        |t              s|| j!                  ||      }nMt        |t              sJ | j!                  |t#        t$        j&                  ||         j)                               }|j                  |       | j                  |   }	|	J |	j+                  |j)                         j-                                 |S )zRReconstruct output tensors according to their saved metadata and alias informationN)r  _initialize_cached_tensorsr   zipr  r  r   dictrK   r  r"  _backward_hooksr  r  *prepare_alias_info_for_tensor_constructionr.   r  r
   rP   r   r   r   	_weak_ref)
ru   r  r   storage_infometadatacached_tstatic_tr7  r   ws
             r@   r  z!CUDAGraphNode.reconstruct_outputs#  s    ))++- +4))4+@+@A,
 +	>'A'h h-!(S$t*,=>>>x(11!4H# ++7/3H, x(11!4H#,,Q/777x(EEhG '>2go<<XwO!'3///<<d5<<1ABRRT NN3%%a(A= =NN3..0::<=W+	>Z r?   c                8   t        |t        t        d       f      s|t        u ry t        |t              rI|j
                  \  }}| j                  |   |   }|J t        j                  j                   |             S t        |t              sJ |j
                  S rB   )r   rK   r  UnaliasedStoragerU  rV  rr  rP   r.   _new_with_weak_ptrr^  )ru   out_alias_infor  depthexisting_output_indexr   s         r@   r  z8CUDAGraphNode.prepare_alias_info_for_tensor_construction]  s     x#tDz!23!11n&=>+9+?+?(E($$U+,ABC?"?''::35AA.*:;;;###r?   c                    g }t        | j                  | j                        D ]&  \  }}|j                  | j	                  ||             ( |S rB   )r  r  r  r"  r  )ru   output_storagesr  r  s       r@   !prepare_storages_for_constructionz/CUDAGraphNode.prepare_storages_for_constructionq  s_     .1%%t'<'</
 	* ( ""??((	 r?   c                T    | j                   J | j                   j                          y rB   )rp   r  ry   s    r@   r  zCUDAGraphNode.run_graph  s"    zz%%%

r?   c                f    | j                   D ]"  \  }}t        | j                  |   |         s" y y)z;All outputs of the path from this node to its root are deadFT)r~  r  rr  )ru   r  output_indexs      r@   rC  z"CUDAGraphNode.all_outputs_are_dead  s<    #'#@#@ 	E<t))%0>?	 r?   c           
     ,    d fd}t        j                   |        j                  j                        D ci c]*  }|j	                         j                         t        |      , }}t        j                  j                  rω j                  g n"t         j                  j                               }|t              D cg c]c  \  }}t        |t        j                         rD| j                  j"                  vr,|j	                         j                         dk7  rt        |      e c}}z  }t%         j&                   j(                  |       t+               5  t        j,                  j'                   j&                        5  t/               5  t        j,                  j1                   j0                   j2                   j(                  d      5  t5               5   |      }	ddd       ddd       ddd       ddd       ddd       t7              dk(  sJ t        	t        t8        f      s|	f}	 j;                  |	|       |	S c c}w c c}}w # 1 sw Y   qxY w# 1 sw Y   uxY w# 1 sw Y   yxY w# 1 sw Y   }xY w# 1 sw Y   xY w)zRecord the modelc               3     K   j                   j                  D ]8  } |    }t        |t        j                        s#j                  |      r5| : y wrB   )r  r   r   rP   r   rG  )r   _inpr   ru   s     r@   static_input_iterz0CUDAGraphNode._record.<locals>.static_input_iter  sO     **<< ay%,,==dCJs   9AAANr   thread_local)r
  poolcapture_error_mode)r   z#Generator[torch.Tensor, None, None])r   r!  r  r   r   r   r4   rG   rc   r.  r  r/  r-  r   r   rP   r   r   r0  r   r  r   r^   rU   rp   r
  rg   r3  rY  _add_first_outputs)
ru   r   r   r  r   $static_input_persistent_storage_ptrsr_   r   elemstatic_outputss
   ` `       r@   r  zCUDAGraphNode._record  s=   	 !!#T%:%:%D%DR
 !**,.CC.HHR
, R
 ==44 kk)tDKK4R4R4T/U  (0AtdELL1T22DDD((*335:	 &d+ F dkk4+@+@&I! 	+5::#4#4KK$
 	+!	+#(::#3#3JJ;;&&-	 $4 $
	+ !"	+ #6]N	+ 	+ 	+ 	+ 	+ 6{a.4-8,.N0TUOR
	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+sy   /IA(I*J
	I>AI2I&	!	I*I&	2I2:I>J
I#I&	&I/+I22I;7I>>J	J

Jc                
     j                   } j                   j                        } j                  ||      }| _        t         j                        dk(  sJ i }t        t        |            D cg c]  }d c} _        t        t        |            D cg c]  }d c} _	        t        |      D ]  \  }|t        |t        j                        s  j                  j                  t                Ct        j"                  |j$                  xs! |j'                         j)                         dk(   fd      f |j+                  |j'                         j)                         d      }	|j'                         j)                         dk(  }
|	r |	       |
r, j                  j                  d       | j                  <    j-                  |      }|7 j/                  |        j                  j                  t1        |             [|j'                         j)                         |v rV||j'                         j)                            }d j                  |<    j                  j                  t3        |             щ||j'                         j)                         <    j                  j                  t                d j                  <   "  j4                  (t        t        |            D cg c]  }d c} _        n(t         j4                        t        |      k(  sJ d        j                  rJ t7        | j                        D ]  \  }}t        |t        j                        r|7 j                  j                  d        j8                  j                  d       Y j                  j                  t;        |              j8                  j                  t=        |               j                   j                         _        t        j@                  jC                   jD                   jF                         _$        t        t         j                              D ]a  }t        t         j                  |               D ];  }tK         j                  |   |         s jL                  j                  ||f       = c  jO                          tP        jR                  jT                  r9tW         jD                   jF                  tY         j[                                      yyc c}w c c}w c c}w )zIAdd the outputs from the first invocation of the node and set up metadatar   FNc                 H    dj                   rj                        S d S )NzHExpected all cuda outputs in cuda graph recording. Non cuda output from z	(unknown))r   )r   ru   s   r@   r   z2CUDAGraphNode._add_first_outputs.<locals>.<lambda>  s=    484E4ED--a0WY KVWY r?   Tz&Wrong number of stack traces passed in).rz  r  rr  r  r}  r3  r  ru  r  r  r   r   rP   r   r  r"  r  r  r(  r   r   r   !_is_alias_of_live_recorded_tensor#_mark_prior_graph_output_as_aliasedrU  r^  r   r  r  r4   r0   r{  rQ   _cuda_getCheckpointStater   r  r  r  r~  r  rG   rc   r.  r0  r/  r-  )ru   r  r  prev_livenessr  deltaoutput_new_storages_indexr   r*  r   is_empty_storagepath_refrV  r   static_output_tensorr  r  r   s   `                @r@   r  z CUDAGraphNode._add_first_outputs  s    ;;**4+=+=>++M=I16.4(()Q...?A!6;CL6I&Ju&J#49#g,4G%Hqd%H"g& &	2DAqy
1ell ;))001ABLL		@Q..099;q@  7::!!#,,.C
  !002;;=B).>))00601**1-==a@H#88B))001H1RS  "++-1JJ1!2C2C2E2N2N2PQ5:++E2))001A%1HIHI%a&7&7&9&B&B&DE%%,,-=>-1D''*M&	2P $/4S\/B C! CDt(()S.  878  ((((),Wd6P6P)Q 	@%C%c5<<04H4T%%,,T2$$++D1%%,,-B3-GH$$++M#,>?	@ .2-?-?@R@R-S**/((*K*KKK..+
'
 3t1123 	PE %c$*<*<U*C&D E P4--e4\BC1188%9NOP	P
 	446==44T22D9P9P9R4S 5O 'K%HV !Ds   5	T=	U=	Uc                    |\  }}t        | j                        |   }d|j                  |<   | j                  |   |   }|J |j	                          y)zLRemove a graph output from the unaliased, cached tensors in an ancestor nodeFN)r/  r?  r  rr  r   )ru   rV  r  r  r>  r  s         r@   r  z1CUDAGraphNode._mark_prior_graph_output_as_aliased  s\    #|D(()%049##L1u%l3}}	  "r?   c                  	 t        | j                        t        | j                        k(  sJ t        t	        | j
                  | j                  | j                              D ]  \  }\  }}}|s| j                  j                  d        (|t        u sJ t        |t              sJ | j                  |      }| j                  ||      }t        j                  j!                  |       t#        j$                  |       	d	fd}t'        j(                  ||      }t+        ||      | j                  |<   | j                  j                  |        y )N)r7  c                @            }|y|j                  |       dk(  S )NF   )get_output_refcount)r   self_locself_refs     r@   check_refcountz@CUDAGraphNode._initialize_cached_tensors.<locals>.check_refcountJ  s)    #:# 33A6!;;r?   )r   r   )r   rK   r   r   )r3  r  r  r   r  r  r  r  r"  r  r   r  create_storager  rP   rQ   _add_cached_tensorr   r   	functoolspartialr4   )
ru   r   r  r  make_cachedr8  r   r  checkr  s
            @r@   r  z(CUDAGraphNode._initialize_cached_tensors(  s4    4(()S1F1F-GGGG8A))%%++9
 &	34A4h **11$7#3333h---##H-A8818MC HH'',{{4(H< %%n:E'<SRW'XD!!!$&&--c2M&	3r?   c                F    t        j                  | j                  |         S rB   )sysgetrefcountr  rZ  s     r@   r  z!CUDAGraphNode.get_output_refcountU  s    t99%@AAr?   c                >    | j                   | j                         S dS )zunwraps the weakref to _parentN)rp  ry   s    r@   r  zCUDAGraphNode.parentX  s     "&!9t||~CtCr?   c              #  <   K   | }|r| |j                   }|ryyw)zAReturns all nodes in the path starting at self and ending at rootN)r  ru   r>  s     r@   _path_to_rootzCUDAGraphNode._path_to_root]  s#      J;;D s   c              #  `   K   t        t        | j                              }|E d{    y7 w)zEReturns all nodes in the path starting at the root and ending at selfN)r<  r/  r  ru   r=  s     r@   r?  zCUDAGraphNode._path_from_roote  s&      d0012s   $.,.c                    | j                   D ]@  }|D ]9  }||j                         }|j                         j                         |k(  s8  y B y)z/Is this tensor an output of a node in this pathTF)rr  r   r   )ru   r  output_refsrF  r   s        r@   rG  z,CUDAGraphNode._is_cuda_graph_recorded_tensork  sb    -- 	 K$/   #+ ,446$$&//1X= 	  r?   c                    t        | j                        D ]T  \  }}t        |      D ]A  \  }}t        |      x}|\  }}||j                         j	                         k(  s;||fc c S  V y rB   )r   rr  r  r   r   )	ru   r  r  r  r  r   storage_and_ptrr7  ptrs	            r@   r  z/CUDAGraphNode._is_alias_of_live_recorded_tensory  s     #,D,>,>"? 	5E;-6{-C 5)k'2;'??OL#2LGSa//1::<< %|44	5	5 r?   c                B    | D ]  \  }}||   |   }|J  |        y y)z;Check that all of the indices specified are dead referencesFTr>   )indicesr  r  r  r  s        r@   _check_livenesszCUDAGraphNode._check_liveness  s@     $+ 	E<E"<0A= =s		
 r?   c                @    | j                   |   j                  |       y)zAdds node as a a child of selfN)rq  r"  )ru   function_idr>  s      r@   	add_childzCUDAGraphNode.add_child  s    k"))$/r?   c                    g }t        |       t        |      k  sJ t        t        | |            D ]Z  \  }\  }}t        |      t        |      k(  sJ t        t        ||            D ]!  \  }\  }}||k7  s|j                  ||f       # \ |S )z(Find indices where the two lists differ.)r3  r   r  r"  )	rZ   currdead_indicesr   outputs1outputs2joutput1output2s	            r@   r  z$CUDAGraphNode._get_different_indices  s    
 4yCI%%%'0T4'A 	0#A#(x=CM111)23x3J)K 0%%GWg% ''A/0	0 r?   c                |    t        |       dk(  rg S | D cg c]  }t        j                  t        |       c}S c c}w )zCMaps weakrefs to true if the reference is alive and false otherwiser   )r3  pytreetree_mapr  )weakrefsr  s     r@   r  zCUDAGraphNode._get_liveness  s4    
 x=AIAIJg1JJJs   !9c                   t         j                  j                  sy t        | j                        D ]"  \  }}| j
                  |   |j                  u r"J  t        | j                        }t        | j                        }t               }t               }t        |      D ]  \  }	}
t        |
      D ]x  \  }}| j
                  |	   |   }t        |      x}&|sJ |\  }}||v ||v k(  sJ |j                  |       |j                  |       ||	   j                  |   d u}|ss||vrxJ   |D ]"  \  }	}t        | j
                  |	   |         s"J  y rB   )rG   rc   r  r   r?  rr  r  r/  get_block_addrsr  r1  r  r2  r  r  )ru   expected_liveness
newly_deadr   r>  r=  live_blockslive_storage_data_ptrslive_storage_weak_ptrsr  outputs_liveness
output_idxoutput_livenessr  stor_weak_ptr_and_data_ptrstor_weak_ptrstor_data_ptris_persistent_aliasr  s                      r@   debug_assert_invariantsz%CUDAGraphNode.debug_assert_invariants  s    }}88 !5!56 	BGAt%%a(D,A,AAAA	B T))*%d&;&;<!$!$'01B'C 	@#E#/89I/J @+
O&&u-j92=a.@.M**?3M0M=)-CC%)??   +..}=*..}= e:::FdR ( +,K???#@	@( $. 	HE<t11%8FGGG	Hr?   c                P    | j                  | j                  | j                         y rB   )r  rz  r|  ry   s    r@   r  z6CUDAGraphNode.debug_check_invariants_before_invocation  s!    $$//1X1X	
r?   c                P    | j                  | j                  | j                         y rB   )r  rz  r}  ry   s    r@   r  z5CUDAGraphNode.debug_check_invariants_after_invocation  s!    $$//1W1W	
r?   c                    | j                  | j                        }| j                  | j                  |      }t	        | j
                        }g }|D ])  \  }}|j                  ||   j                  |   d          + |S )z
        Since this node was invoked, return data ptrs of all tensor outputs that have died
        in the current executing tree path.
        r   )r  rr  r  r{  r/  r?  r"  r  )ru   r  r  pathptrs_to_deallocater  r  s          r@   data_ptrs_dead_since_invocationz-CUDAGraphNode.data_ptrs_dead_since_invocation  s    
 **4+=+=>!%!<!<.."
 D(()#9 	E<%%U,,\::F	
 "!r?   c              #  ~   K   | j                   D ]*  \  }}| j                  |   |   }|t        |      s'| , y wrB   )r~  rr  r  )ru   r   r  r   s       r@   r-  z CUDAGraphNode.path_live_weakrefs  sD     11 	DAq$$Q'*C73<		s   '===c                $   | j                   D ]$  }|t        j                  j                  |       & | j                   j	                          t        | j                        D ]+  \  }}|s	| j                  |   }|J |j                          - y rB   )	r  rP   rQ   _remove_cached_tensorr   r   r  r  r   )ru   r  r   	unaliasedns        r@   remove_node_cached_tensorsz(CUDAGraphNode.remove_node_cached_tensors  s    ++ 	2A}..q1	2 	""((*%d&A&AB 	+LAy))!,}$}((*		+r?   c                F    | j                   D ]  }|j                           y rB   )r?  r%  r  s     r@   remove_path_cached_tensorsz(CUDAGraphNode.remove_path_cached_tensors  s#    (( 	.D++-	.r?   c                     y)z3Clear the path state in this current executing nodeNr>   ry   s    r@   clear_path_statezCUDAGraphNode.clear_path_state	  r   r?   c                >   t        | t        j                        sJ | j                         j	                         | j                         j                         | j                  | j                         | j                  | j                  |s| j                         dS ddS )Nr   )nbytesr   sizestridedtyper   storage_offset)r   rP   r   r   r+  r   shaper-  r.  r   r/  )r  rh  s     r@   r  zCUDAGraphNode._tensor_metadata  s     !U\\*** '')002))+446GGhhjWWhh8Ma..0
 	
 TU
 	
r?   Nc                l    || j                  |      n|}t        j                  j                  ||      S rB   )r  rP   rQ   0_construct_CUDA_Tensor_From_Storage_And_Metadata)ru   r  r7  r8  s       r@   r  z/CUDAGraphNode._reconstruct_from_tensor_metadata  s2     .5_D)'xxHHSTUUr?   c                V    t         j                  j                  |d   |d   |d         S )Nr   r   r+  )rP   rQ   $_construct_storage_from_data_pointer)ru   r  s     r@   r  zCUDAGraphNode.create_storage$  s/    xx<<Z (8"4hx6H
 	
r?   c           	     D   t         j                  j                          | j                  j	                  t         j                  j                                g }t        j                  d      5  t         j                  j                  | j                        5  t        | j                  | j                  | j                        5  t        |      D ]}  \  }}t        |t         j                        s$t        |t              sJ |j                  |       D|| j                   vr|j                  t#        |             m|j                  |        | j%                  ||       ddd       ddd       ddd       |S # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   |S xY w)z
        Allocate inputs for non static, non cudagraph managed tensors in the memory pool
        and copy over the tensor values.
        Trecord)r  r
  N)rP   r^   r  r
  r  r	  warningscatch_warningsr   r  r  r   r   r   rK   r"  r   r"   r  )ru   r   r  r   r   s        r@   r  z1CUDAGraphNode._allocate_and_copy_recording_inputs)  s[    	

 

 9 9 ;<,.$$D1 	L5::3D3DKK4
 	L(KK**;;
	L $F+ 13!#u||4%c3///$++C0d444$++L,=>$++C01 112BFK#	L 	L 	L&  '	L 	L 	L 	L 	L&  s=   .*F-F	BE=#F	+F=FF		F	FFc                z   t        j                  t        | j                  j                  || j
                        }t        j                  j                  || j
                  | j                        s5t        j                  t        j                  || j                        }|fS | j                  | j                  | j                        st        j                  fdfS | j                   rjt        j                  j                  || j
                  | j"                        s5t        j$                  t        j                  || j"                        }|fS | j                  D ]  }d||<   	 t        j&                  | j                  | j(                  | j                        d        t        j*                  d fS )z
        Checks if this node can be run. The same pattern of tensor liveness, static inputs,
        and tensors managed in the cudagraph private pool must remain stable.
        c                 
      S rB   r>   )statuss   r@   r   z0CUDAGraphNode.check_invariants.<locals>.<lambda>m  s	    fX r?   Nc                      y)NzTODO: graph recording observed an input tensor deallocate during graph  recording that did not occur during replay. Please file an issue.r>   r>   r?   r@   r   z0CUDAGraphNode.check_invariants.<locals>.<lambda>  r   r?   c                 $    t         j                   S rB   )r$   SUCCESSr>   r?   r@   r   z0CUDAGraphNode.check_invariants.<locals>.<lambda>  s    8L8T8T7U r?   )r  r  r'   r  r   rx  rP   rQ   r  rt  r$   CudagraphManagedIdxMismatchr  r|  rr  &ExpectedDeadIndicesBeforeGraphMismatchro  r   r  r  r}  r?  )ru   r   _loggerrf  r<  s       @r@   check_invariantszCUDAGraphNode.check_invariantsJ  s    ##!!!..''	
 xx;;''''

 *EEF''++G
 7?"##33T5G5G
 *PPF... 11HH@@++&& *@@F''&&G
 7?"
 .. 	CF3K	 	  668J8JQ		
 $++-VVVr?   c                    d}| j                   j                         D ]!  }|D ]  }|dz  }||j                         z  } # |S )z(Total number of descendents of this noder   rF   )rq  valuesnum_descendants)ru   num_descrq  childs       r@   rF  zCUDAGraphNode.num_descendants  sU    ,,. 	4H! 4AE11334	4 r?   )r  r,   rL   rJ   r  Optional[CUDAGraphNode]r   r   r  rI  rn   rK   r   rJ  r
  rK  r   r   )r  r   r  r   r   r   )r%  r   r   r   )r%  r   r   r*   )r   r*   )r  zOptional[OutputAliasInfo]r  z Union[Dict[str, Any], int, None]r   z Union[UntypedStorage, None, int])r   z&List[Union[UntypedStorage, None, int]]r   r   )r   r)   r   r   r   r*   )r  r*   r  z Dict[int, StorageWeakRefWrapper]r   r   r[  )rV  rK   r   rK   )r   rI  )r   z$Generator[CUDAGraphNode, None, None]rM  )r  rN  r   zOptional[PathOutputIndex])r  List[PathOutputIndex]r  +List[List[Optional[StorageWeakRefWrapper]]]r   r   )r  r%   r>  ra  r   r   )rZ   List[List[bool]]r   rL  r   rJ  )r
  rK  r   rL  )r  rL  r  rJ  r   r   )r   rj  rL  T)r  rN  rh  r   r   Dict[str, Any]rB   )r  rN  r7  zOptional[UntypedStorage]r   r   )r  rN  r   ztorch.types.Storage)r   r   r   ri  )r   r   r   z/Tuple[CheckInvariantStatus, Callable[..., str]]r   ),r;   r<   r=   rM   rv   r  r  r  r:  r  r  r  r  rC  r  r  r  r  r  rO  r  r  r?  rG  r  staticmethodr  r  r  r  r  r  r  r  r-  r%  r'  r)  r  r  r  r  rC  rF  r>   r?   r@   ra  ra    s   (X)X X (	X
  X *X X ,X "X 
Xt;#;+:;	; 3(
*8t$1$ 3$ 
*	$(	/3j]] /O] 
	]~#+3ZB D D    


	"
 
&
@
 

 
0 &6	  K=K	K K%H!1%H?T%H	%HN



"&
+.> 7;

04
	
 
" MQV&V1IV	V

 % 	' BFW%FW	8FWPr?   ra  c                z    t         j                  j                         }|D cg c]  }|d   | k(  s| c}S c c}w )Nsegment_pool_id)rP   r^   memory_snapshot)pool_idsegmentssegments      r@   get_cudagraph_segmentsrV    s5    zz))+H#+Uw7H/IW/TGUUUs   88c                    g }t        |       D ]4  }|d   }|d   D ]%  }|d   dk(  s|s|j                  |       ||d   z  }' 6 |S )Naddressblocksstateactive_allocatedr,  )rV  r"  )rS  	live_onlyrY  rU  addrblocks         r@   r  r    si    F)'2 "y!X& 	"EW~!339d#E&M!D		"" Mr?   c           	         g }| D ]1  }|j                  t        j                  |d   |d   |d                3 dj                  t        j                  |            S )Nfilenamelinename )r"  	tracebackFrameSummaryjoinformat_list)framesformatted_tracebackentrys      r@   	format_tbrk    sa     
""""5#4eFmU6]S	


 779(()<=>>r?   c                   t        d |D              sJ |D ch c]  } |       s|j                          c}t        j                  j	                  |       ry t        j                          t              }i }|D ]<  }|d   }|d   D ]-  }|d   dk(  r|vr|||<   nj                  |       ||d   z  }/ > t        j                  t              dk(  fd       t        |      dk7  rmg }	|j                         D ]7  \  }
}t        |j                  d	g             }|	j                  d
|
 d|        9 dj                  |	      }d| }t!        |      y c c}w )Nc              3  <   K   | ]  }t        |t                y wrB   )r   r4   ).0r  s     r@   	<genexpr>z$check_memory_pool.<locals>.<genexpr>  s      48
4./s   rX  rY  rZ  r[  r,  r   c                     d  d S )Nz2These storage data ptrs are not allocated in pool z but should be r>   )rS  unique_storagess   r@   r   z#check_memory_pool.<locals>.<lambda>  s    DWI_]l\mn r?   rh  zData Pointer: z, history: 

zpThese live storage data ptrs are in the cudagraph pool but not accounted for as an output of cudagraph trees: 

)allr   rP   rQ   _cuda_checkPoolLiveAllocationsgccollectrV  remover  r3  r   rk  r   r"  rf  RuntimeError)r   rS  live_storages_ptrsr   rT  allocated_not_in_live_storagesrU  r]  r^  	formatteddptraceformatted_smsgrq  s    `            @r@   r0  r0    s   
  <N    4FP4t}}PO xx..vwP JJL%g.H%'" 	"y!X& 	"EW~!33.;@248#**40E&M!D	"	" 
LLO!n
 )*a/	7==? 	HIBeii"56E~bTugFG	H ii	*BBMP 	 3 0? Qs
   E#E#c                  H    e Zd ZdZ e       Z e       Z e       Z e       Zy)ExecutionStatez
    Represents the state of the CUDAGraph Tree. Will be None if there is no live current memory allocated
    in the cuda graph pool. Otherwise will reflect the state of the most recently executed node.
    N)	r;   r<   r=   rM   r   NONEWARMUP	RECORDING	EXECUTIONr>   r?   r@   r  r    s%    
 6DVFIIr?   r  c                  6    e Zd Z e       Z e       Z e       Zy)r   N)r;   r<   r=   r   r   r   r   r>   r?   r@   r   r     s    fGvHIr?   r   c                     e Zd ZdZd%dZd&dZd'dZd(dZd)dZ	 	 	 	 	 	 d*dZ	d+dZ
	 	 	 	 	 	 d,d	Zd&d
Zd'dZ	 	 	 	 	 	 d&dZ	 	 	 	 	 	 d-dZ	 	 	 	 	 	 d&dZd)dZd.dZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d/dZed0d       Zed0d       Zd1dZed2d       Zej2                  	 	 	 	 d3d       Zd'dZed4d       Zed0d       Zd0dZd0dZd5dZ d'dZ!d5dZ"d5dZ#d'd Z$d'd!Z%d'd"Z&	 	 d6d#Z'y$)7r   a  
    Groups individual recordings or executions of cuda graphs into a tree of recordings,
    and checks required invariants, and manages warmups of graphs.

    When graphs are recorded in the same tree, it enforces subsequent execution
    to follow the same order and have the same output tensor livespans. To remove
    unnecessary coupling of cuda graphs (and additional imposed invariants),
    the tree manager will end a currently recording tree whenever it is valid - when
    the memory pool no longer has any live allocations.

    We ignore outputs from a previous generation that correspond to prior model outputs.
    Currently this is hardcoded `GenerationTracker.generation` tracked in torch dynamo.
    # TODO: make generation increment configurable, warn on overwrite.

    We run graph warmups in the cudagraph memory pool and return the result on the first invocation
    of a function. For many models it is important to reclaim activations as you run the backward.
    If we were to warm up the model and keep an extra copy of the inputs around to subsequently
    use for recording, we would incur a memory penalty. Additionally, if we are part way through training
    your model and need to recompile, memory will be allocated to the cuda graph pool, so we run this
    warmup run in the cuda graph memory pool. As for recording, warm up needs the state of live tensors
    to be accurately reflected so we checkpoint the allocator state if we need to warm up following graph
    replay.
    c                >   t        t              | _        i | _        i | _        t               | _        t               | _        t        j                  j                  d       t               | _        t        j                  j                  |      5  t        j                  j                          t        j                  j                         | _        | j                   j#                  t        j                  j%                                t        j                  j'                         | _        t        j                  j+                         | _        t/        j0                  d      5  t        j                  j)                  | j(                  | j,                  | j                   d      5  	 d d d        d d d        d d d        t3        j4                  d      | _        t3        j4                  d      | _        t        t:              | _        t3        j4                  dd      | _        t        d       | _         tB        jD                  | _#        || _$        d | _%        d| _&        d| _'        d| _(        i | _)        d	| _*        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)
NTr6  r  )r  r
  r  r   )startstepc                     t        d       S )Nc                      yrk   r>   r>   r?   r@   r   zACUDAGraphTreeManager.__init__.<locals>.<lambda>.<locals>.<lambda>R  r   r?   r   r>   r?   r@   r   z/CUDAGraphTreeManager.__init__.<locals>.<lambda>R  s    K	* r?   F)+r   r/  rootsids_to_funcsids_to_stack_tracesr1  warmed_up_functionswarned_functionsrP   rQ   r7   warned_mutationr^   r   r  Streamr
  r  r	  r  rp   graph_pool_handlecuda_graphs_thread_poolr8  r9  r   countgraph_counterfunc_counterr  #non_cudagraph_managed_mutation_hintwarmup_node_counternum_rerecordr  r  
path_statern   current_nodecurrent_gendebug_fail_counterdebug_checkpointing_counter
id_to_mode'running_forwards_with_pending_backwardsrt   s     r@   rv   zCUDAGraphTreeManager.__init__  s   
 =H<M
 @BLN 47E  25,,T2 14 ZZ|, 	JJ""$**++-DKKK##EJJ$=$=$?@ :?9M9M9ODJ+0::+G+G+ID(((5 uzz7G7G

11{{#1	 8H 8   	" '__Q/%OOA.  	0 $-??"#E  MX*M
 )--( MQ
 !# #$+,(=?  8=4}   	 	s>   CJ#AJ%I;'J/J;J JJ	JJc                    | j                   J d       | j                  ||      }| j                  |   }|t        j                  k(  r	d| _        |S |t        j                  k(  rd| _        |S )Nz Running CUDAGraph after shutdownTF)rp   _runr  r   r   r  r   )ru   r%  r  r   r   s        r@   r:  zCUDAGraphTreeManager.run}  sr    zz%I'II%ii
K0 {+?***;?D8 
 _---;@D8
r?   c                    d| _         y rW   )r  ry   s    r@   set_to_running_backwardz,CUDAGraphTreeManager.set_to_running_backward  s
    7<4r?   c                t    t        | j                  t        t        f      r| j                  j                  S d S )Nc                     yrW   r>   r   s    r@   r   zNCUDAGraphTreeManager._get_cuda_graph_recorded_tensor_checker.<locals>.<lambda>  r   r?   )r   r  ra  r  rG  ry   s    r@   '_get_cuda_graph_recorded_tensor_checkerz<CUDAGraphTreeManager._get_cuda_graph_recorded_tensor_checker  s9     $++m^-LM <<	
 !	
r?   c                >    t        t        | j                              S rB   )rJ   nextr  ry   s    r@   new_warmup_node_idz'CUDAGraphTreeManager.new_warmup_node_id  s    tD44566r?   c                ,   | j                         }t        | j                  |   || j                               x}rHd| j                  |   |<   || j
                  v ry | j
                  j                  |       t        |       y d| j                  |   |<   y rE  )_get_node_idr#   r  r  r  r  r2  r&   )ru   r  r   node_idmaybe_mutation_strs        r@   &_update_non_cudagraph_managed_mutationz;CUDAGraphTreeManager._update_non_cudagraph_managed_mutation  s     ##%!3k*88:"
 
 

 NRD44W=kJd222  $$[1/0BCMRD44W=kJr?   c                    | j                   y t        | j                   t        t        f      r| j                   j                  S t        dt        | j                                )NzUnknown node type )r  r   ra  r  rL   rx  r  ry   s    r@   r  z!CUDAGraphTreeManager._get_node_id  sW    $))M>+JK$$'''!3D9J9J4K3LMNNr?   c                    t         j                  j                  j                  ry| j                  |   |   t         j
                  j                  j                  j                  kD  S rW   )rP   rk  rG   rl  r  rm  rc   #cudagraph_unexpected_rerecord_limit)ru   r  r  s      r@   exceed_rerecord_limitz*CUDAGraphTreeManager.exceed_rerecord_limit  sS     ==99 g&{3oo$$++OOP	
r?   c                   | j                   r| j                  |       | j                  r| j                  |       | j	                         }|| j
                  |   vr| j                  ||       | j
                  |   |   s| j                  ||      r| j                  |   j                  |      S || j                  v st        j                  j                  r&| j                  st        j                  j                  r?| j                  t         j"                  k(  r| j%                          | j'                  ||      S t)        | j*                  t,              rJ | j*                  | j.                  n| j*                  j0                  }| j                   s2dd }}||   D ]h  }|j3                  |      \  }}	|t4        j6                  k(  r| j9                  ||      c S |t4        j:                  k(  s|t4        j<                  k(  sed}|	}j | j*                  <|| j.                  v r.| j?                          | j*                  | jA                  ||      S tC        | j                  |   jD                        dkD  rP| j                  ||       | j
                  | j	                            |   r| j                  |   j                  |      S |r| j	                         }
| jF                  |
   |xx   dz  cc<   | j                  |
|      r|
r|
jH                  nd }tK        d|jH                   dtL        jN                  j                  j                  jP                   d| d	 |        d
	       | j                  |   j                  |      S | xjR                  dz  c_)        | j?                          | j*                  | j%                          | jU                  ||      S )NFc                      y)Nrc  r>   r>   r?   r@   r   z+CUDAGraphTreeManager._run.<locals>.<lambda>  r   r?   Tr   rF   z#skipping cudagraph due to function z$ exceeding max re-recording limit (=z) on cudagraph node z due to .)+in_recordingtry_end_curr_recording	in_warmuptry_end_curr_warmupr  r  r  r  r  r   r  rG   rc   skip_cudagraph_warmupforce_cudagraphs_warmupr  r  r  -apply_checkpoint_execution_state_in_allocator	run_eagerr   r  r  r  rq  rC  r$   r?  execute_noder  r@  try_end_curr_executionr:  r3  r   r  rL   r&   rP   rm  r  r  record_function)ru   r%  r  r  child_nodesunexpected_rerecordunexpected_rerecord_reasonrH  r<  status_loggercurr_node_id_ids               r@   r  zCUDAGraphTreeManager._run  s   
 ''4>>$$[1##%dFFwOO77ZP
 33G<
''=$$[177
CC  4#;#;;}}:: ~~}}44 .":"::BBD>>*k::d//@@@++3DJJ9J9J9S9S 	   >CZ!;$[1 ? ).(>(>z(J%1999,,UJ?? 2III!5!Q!QQ*.'1>.?&   ,

1J++- $$,88J<<4$$[1DDEI;;KT;;D<M<M<OP  ,,[9??
KK
 ##002!!,/<A<--lKH-9,//tC7=knn=M N"__33::^^_ `--0E:T:V9WWXZ  ,,[9??
KK ##q(#'')  ,BBD ##J<<r?   c                J   g }| j                   j                         D ]  }|j                  |        |rZ|j                         }|j                  j                         D ]  }|j                  |        |j                          d|_        |rZd| _        d| _         d| _        y)a  
        Remove all cached tensors in all nodes. Because cached tensors can hold gradients which in turn
        might reference a backward which invokes a CUDA Graph Node, we have to manually clear them on shutdown
        to avoid a reference cycle.
        N)r  rE  r4  poprq  r%  rp   r  )ru   r=  r  r>  rq  s        r@   r   zCUDAGraphTreeManager.shutdown+  s     ZZ&&( 	 ELL	  99;D MM002 'X&'++-DJ  

 r?   c           
        t        | j                  t              rJ | j                         }t        j                  d|j                  |j                         t        j                  j                          t        | j                  |   || j                  || j                  | j                  | j                  |   | j                        }| j                  | j                   |   j#                  |       n| j                  j%                  ||       || _        t&        j(                  | _        | j-                          t        j                  j                          |j/                  |      S )Nz.Recording function %d of graph recording id %d)r   r  r  new_graph_idr   debugrL   rP   r^   r  ra  r  r  rn   r  r
  r  r"  r  r  r  r  update_generationr  )ru   r%  r  graph_idr>  s        r@   r  z$CUDAGraphTreeManager.record_function@  s"    d//@@@$$&		<NNKK	

 	

 k*(($$[1KK	
 $JJ{#**40''T: (22 

 $$Z00r?   c                |    || _         t        j                  | _        | j	                          |j                  |      S rB   )r  r  r  r  r  r:  )ru   r>  r%  s      r@   r  z!CUDAGraphTreeManager.execute_node_  s4     !(22 xx
##r?   c                .   || j                   v }|s!t        j                  d|j                         n t        j                  d|j                         | j                   j	                  |       t        | j                  |   | j                  | j                  | j                  | j                  | j                  |   | j                  || j                         	      }|| _        t        j                  | _        | j#                          |j%                  |      S )NzRunning warmup of function %dz?Running eager of function %d because ancestor needed to warm up)r  r   r  rL   r2  r  r  r  r  rp   rn   r  r
  r  r  r  r  r  r:  )ru   r%  r  r  r>  s        r@   r  zCUDAGraphTreeManager.run_eagerg  s    
 #d&>&>>II5{~~FIIQ 	  $$[1k*((JJ$$[1KK##%

 !(// xx
##r?   c                >    t        t        | j                              S rB   )rJ   r  r  ry   s    r@   r  z!CUDAGraphTreeManager.new_graph_id  s    tD../00r?   c                >    t        t        | j                              S rB   )r%   r  r  ry   s    r@   new_func_idz CUDAGraphTreeManager.new_func_id  s    $t00122r?   c	           	     h   | j                         }	|| j                  |	<   t        |t        |      |	t	        d |D              ||      | j
                  |	<   || j                  |	<   t        j                  | j                  |	      }
t        | j                        j                  |
       |
 |
|      fS )Nc              3  p   K   | ].  }t        |t        j                        s|j                  s+| 0 y wrB   )r   rP   r   r(  )rn  r  s     r@   ro  z4CUDAGraphTreeManager.add_function.<locals>.<genexpr>  s#     T*Q*E!))!Ts   666)r  )r  r  r,   r/  rY  r  r  r  r  r:  r   rn   r   )ru   r   r   r   r   r   r   r   r   rL   r   s              r@   r   z!CUDAGraphTreeManager.add_function  s     '3  $ /"#TYTT!
" #txxR8 	d''(==bA2f:~r?   c                <    | j                   t        j                  k(  S rB   )r  r  r  ry   s    r@   r  z!CUDAGraphTreeManager.in_recording  s    .":":::r?   c                <    | j                   t        j                  k(  S rB   )r  r  r  ry   s    r@   r  zCUDAGraphTreeManager.in_warmup  s    ."7"777r?   c              #  b   K   | j                   j                         D ]  }|E d {     y 7 wrB   )r  rE  r  s     r@   	get_rootszCUDAGraphTreeManager.get_roots  s.     ZZ&&( 	E	s   #/-/c                    | j                   S rB   )_current_nodery   s    r@   r  z!CUDAGraphTreeManager.current_node  s    !!!r?   c                B    || _         |t        j                  | _        y y rB   )r  r  r  r  )ru   values     r@   r  z!CUDAGraphTreeManager.current_node  s#     #=,11DO r?   c                .    | j                         | _        y rB   )get_curr_generationr  ry   s    r@   r  z&CUDAGraphTreeManager.update_generation  s    335r?   c                 h    t         j                  dk7  rt         j                  S t        j                  S rk   )r   r   r   
generationr>   r?   r@   r  z(CUDAGraphTreeManager.get_curr_generation  s(    ((A-000 +++r?   c                 (    t         j                  dk7  S rk   r   r>   r?   r@   user_invoked_mark_stepz+CUDAGraphTreeManager.user_invoked_mark_step  s    ,,11r?   c                `    | j                         sy| j                         ry| j                   S )NFT)in_new_torch_compile_invocationr  r  ry   s    r@   can_start_new_generationz-CUDAGraphTreeManager.can_start_new_generation  s/    335&&(????r?   c                <    | j                   | j                         k7  S rB   )r  r  ry   s    r@   r  z4CUDAGraphTreeManager.in_new_torch_compile_invocation  s    4#;#;#===r?   c                   | j                   sJ | j                  J | j                         r!| j                          | j	                          y| j                  j                         r| j	                          y| j                  |       y)a  
        Check if the current recording can be terminated, either because all outputs of the
        previously recorded node are dead or because it was executed in a different
        generation. Will set current_node to None and in_recording to False if successful.
        N)r  r  r  dealloc_current_path_weakrefs(clear_current_path_state_and_set_to_nonerC  'check_warn_on_unable_to_start_executingru   r  s     r@   r  z+CUDAGraphTreeManager.try_end_curr_recording  s}           ,,, ((*..099;11399;44[Ar?   c                    | j                   rJ | j                  y| j                         r| j                          y| j                  j	                         r| j                          yy)z
        Check if the current executing node can be terminated, either because all outputs of the
        previously executed node are dead or because it was executed in a different generation.
        Will set current_node to None if successful.
        N)r  r  r  r  rC  ry   s    r@   r  z+CUDAGraphTreeManager.try_end_curr_execution  s`     $$$$$((*99;11399; 4r?   c                    | j                         r| j                          d | _        y | j                  J | j                  j                         rd | _        y | j	                  |       y rB   )r  r  r  rC  r  r  s     r@   r  z(CUDAGraphTreeManager.try_end_curr_warmup 	  sb    ((*..0 $D  ,,,113 $D44[Ar?   c                H   || j                   v s| j                         sy| j                  J | j                  j                  D cg c]  }|j                  j
                  |k(  r| }}t        |      dk  ryt        j                  || j                  f      D ch c].  }|j                   |j                  j                  j
                  0 }}t        |      t        |      k(  ry| j                   j                  |       t        j                  d       yc c}w c c}w )zCWarn if we in a potential loop where we are unable to hit fast pathNrF   zUnable to hit fast path of CUDAGraphs because of pending, uninvoked backwards. Consider running with torch.no_grad() or using torch.compiler.cudagraph_mark_step_begin() before each model invocation)r  r  r  r?  r  rL   r3  r   r!  r  r2  r8  warn)ru   r  r>  existing_nodesr$  parentss         r@   r  z<CUDAGraphTreeManager.check_warn_on_unable_to_start_executing	  s    4000779  ,,, ))99
$$'';6 
 
 ~!#
 __^d6G6G5IJ
xx# HH%%((
 

 w<3~..!!+.+	
'

s   "D3Dc                   | j                   J | j                   j                  D ]  }|j                  J t        |j                        t        |j                        k(  sJ t        |j                  |j                        D ]M  \  }}|d n |       }||r|j                         nd}d| d}t        j                  j                  ||       O  t               }| j                   j                         D ]]  } |       }|s|j                         |vs |j                  |j                                t        j                  j                  |       _ y )Nz[Could not find stack trace]ziError: accessing tensor output of CUDAGraphs that has been overwritten by a subsequent run. Stack trace: z. To prevent overwriting, clone the tensor outside of torch.compile() or call torch.compiler.cudagraph_mark_step_begin() before each model invocation.)r  r?  r   r3  r  r  striprP   rQ   _set_storage_access_error_msgr1  r-  r   r2  _free_And_Remove_DeleterFn)	ru   r>  r  stack_tracetenr  deletedr   _storage_derefs	            r@   r  z2CUDAGraphTreeManager.dealloc_current_path_weakrefs/	  s\     ,,, %%55 	AD$$000t++,D4E4E0FFFF"%d&:&:D<M<M"N A;idQS; #  %%'7 $$/= 1gg  66sC@!A	A* %,,??A 	DK(]N+"6"6"8"GK002333NC		Dr?   c                ~    t        | j                  t              sJ | j                  j                          d | _        y rB   )r   r  ra  r)  ry   s    r@   r  z=CUDAGraphTreeManager.clear_current_path_state_and_set_to_noneO	  s2    $++];;;**, r?   c                   t        | j                  t              sJ | xj                  dz  c_        t        j                  d| j                         | j                  j                  }| j                  j                  }||J g }| j                  j                          t        | j                  j                               }|D cg c]	  } |        }}| j                  j                         }t        j                  j                  ||||       t        |      D ]!  }t        j                  j!                  |       # t"        j$                  j&                  rit)        | j*                  | j,                  |       |D ]B  }	 |	       }
|
J t        j                  j/                  |
      sJ |	j1                         |vrBJ  yyc c}w )z
        Checkpoint the current execution state in the caching allocator so that
        additional cudagraph recordings can be made respecting existent live storages.
        rF   zDCheckpointing cuda caching allocator state. Number of checkpoints %dN)r   r  ra  r  r   r  r  r   r'  r/  r-  r  rP   rQ   _cuda_setCheckpointPoolStater1  %_cuda_cudaCachingAllocator_raw_deleterG   rc   r.  r0  rn   r  _has_Standard_Deleterr   )ru   rZ  r   stale_storageslive_storages_wrappersr  live_storages_weak_refsr  r  wrapperstorage_ptrs              r@   r  zBCUDAGraphTreeManager.apply_checkpoint_execution_state_in_allocatorT	  s   
 $++];;;((A-(		R,,	

 !!<<"")) V%777 %' 	446!%d&7&7&J&J&L!M <R-Raac-R-R!..NNP--E>+B	

 )* 	@CHH::3?	@ ==44!!4#?#?AW 2 D%i"...xx55kBBB'')1CCCC	D	 5 .Ss   Gc                    | j                   g S | j                   j                         D cg c]	  } |        c}S c c}w rB   )r  r-  )ru   r  s     r@   .live_cudagraph_pool_storages_in_curr_executionzCCUDAGraphTreeManager.live_cudagraph_pool_storages_in_curr_execution	  s<     $I "..AACDDDDs   <Nr   )r%  r   r  r%   r   r*   r   )r   zCallable[[Tensor], bool])r   rJ   )r  r%   r   r   r   r   )r   Optional[GraphID])r  r
  r  r%   r   r   )r>  ra  r%  r   r   r*   )r   r%   )r   r)   r   r   r   Sequence[int]r   rJ  r   r   r   Tuple[torch.Tensor, ...]r   Tuple[PlaceholderInfo, ...]r   Tuple[int, ...]r   Tuple[ModelType, OutputType]r   )r   zIterator[CUDAGraphNode])r   rH  )r  rH  r   r   r   )r  r%   r   r   )r   zList[StorageWeakRefPointer])(r;   r<   r=   rM   rv   r:  r  r  r  r  r  r  r  r   r  r  r  r  r  r   rO  r  r  r  r  setterr  rO  r  r  r  r  r  r  r  r  r  r  r  r	  r>   r?   r@   r   r      s   0b=H=
7S%S/>S	S$O	
(	
7A	
		
m=^!*1)18B1	1>$!$/>$	$$)$8B$	$<13   )	
 ,  , 2 , 
'8 ; ; 8 8 " " 2C2	2 26 , , 2 2@>B*<$B 
DD@!
,D\E	$Er?   r   )rD   r2   r   r   r   )r   Generator[None, None, None])r   zContextManager[None])r   r   r   r   r   r   )rn   rK   r   ri   rM  )rn   rK   r   r   r   zOptional[CUDAGraphTreeManager])r   r)   r   r   r   r  r   r   r   r   r   r)   )r>   )r   r)   r   r   r   r  rn   rK   r   r   r   r   r   rJ  r   r  r   r  r   r  r   r  )r  Optional[StorageWeakRefWrapper]r   r   )r  r  r   z+Optional[Tuple[StorageWeakRefPointer, int]])r   rK   r  rI  r
  rK  r   r  )r  zOptional[Tensor]r   r  )rS  rI  r   r   )rS  rI  r\  r   r   rj  )rh  z	List[Any]r   r   )r   rK   rS  rI  ry  zList[StorageWeakRefWrapper]r   r   )rM   
__future__r   re   dataclassesr  ru  r   r   r  rq   rd  r8  r   collectionsr   enumr   r   typingr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   torch.fxrP   r   torch._dynamo.mutation_guardr   torch._dynamo.utilsr   r   torch._inductor.compile_fxr   r   r   r   r    r!   r"   torch._inductor.cudagraph_utilsr#   r$   r%   r&   r'   r(   r)   r*   r+   r,    torch.multiprocessing.reductionsr-   torch.storager.   torch.utilsr/   r  torch.utils.weakr0   torch._inductor.utilsr1   torch.typesr2   rK   StorageWeakRefPointerStorageDataPtrNBytesr3   backendsr^   is_builttorch._Cr6   r9   r7   _logginggetArtifactLoggerr;   r   rc  rG   	dataclassrJ   rS   contextmanagerrU   r[   ra   rg   ri   r   r   rr   r   r   rQ   _stash_obj_in_tlsr   r   r   r   r   r   r   r4   r  r  r  r  rW  r   PathLivenessr   StackTracesr  	InputList
OutputList	LevelListrQ  rS  r  rU  r^  ra  rV  r  rk  r0  r  r   r   r>   r?   r@   <module>r3     sO  "H #    	   
     #     &   : <     < ( ) * /! 	C./ 	>>!   nn&&x>  d#  $
+    = = 	; 	;&[% [%| 		 !# &y~~6     4e6S6S T   /1I1I J'&&3, 6:44.24#4-!-!-! %-! 	-!
 -! -!f (* +/*,02*, %
    ( ( . ( "BFH FHR-	"-	"0	" 44*44E4 4 4.$ S/ DJ8C=!J J\ 	
		 	P P %& o 	 	j jZV
?1 1 1  41  
	1 h	T 	d I
E I
Er?   