
    ɯwgW                        d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m
Z
mZ ddlmZmZmZmZmZmZmZmZ ddlZddlmc mZ ddlmZ ddlmZ dZ ed      Z  ed      Z! ed	      Z"e#Z$e#Z%e#Z&e#Z' ejP                  e)      Z* G d
 dejV                        Z,e
 G d d             Z- G d de.      Z/ G d de/      Z0 G d de.      Z1e
 G d d             Z2 G d d      Z3 G d d      Z4 G d d      Z5dee e!f   dee e"f   deee e!e"f      fdZ6d ejn                  d!eed"f   d#ee8ef   deeejr                  ef      fd$Z: G d% d&      Z; G d' d(e      Z< G d) d*      Z=d+ Z> e=       Z?y),a  
This module introduces CUDA Sanitizer, a tool for detecting synchronization errors between kernels ran on different streams.

It stores information on accesses to tensors to determine if they are synchronized
or not. When enabled in a python program and a possible data race is detected, a
detailed warning will be printed and the program will exit.

It can be enabled either by importing this module and calling
:func:`enable_cuda_sanitizer()` or by exporting the ``TORCH_CUDA_SANITIZER``
environment variable.
    N)	dataclassfield)AnyDictIteratorListOptionalSetTupleTypeVar)_pytree)TorchDispatchModeTKTVaTVbc                   V    e Zd Z ej                         Z ej                         Zd Zy)
AccessTypec                 .    | t         j                  u rdS dS )Nzreading fromz
writing to)r   READselfs    Z/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/torch/cuda/_sanitizer.py__str__zAccessType.__str__1   s    !%!8~JlJ    N)__name__
__module____qualname__enumautor   WRITEr    r   r   r   r   -   s"    499;DDIIKEKr   r   c                   r    e Zd ZU dZeed<   eed<   eed<   eed<   e	e   ed<   e
ed<   ej                  ed<   y	)
AccessaW  Stores information about a single access to a tensor by a kernel.

    Args:
        type: either AccessType.READ or AccessType.Write.
        seq_num: the sequential number of the kernel performing the access.
        stream: the stream id of the stream executing the kernel.
        operator: the schema of the launched kernel, which lists the
            arguments and return type.
        aliases: the arguments in the schema this access corresponds to.
        is_output: Whether the tensor was an output of the kernel.
        stack_trace: the stack summary object captured during access.
    typeseq_numstreamoperatoraliases	is_outputstack_traceN)r   r   r   __doc__r   __annotations__SeqNumStreamIdstrr   bool	tracebackStackSummaryr!   r   r   r#   r#   5   s:     OM#YO'''r   r#   c                       e Zd ZdZy)SynchronizationErrorz1Base class for errors detected by CUDA Sanitizer.N)r   r   r   r+   r!   r   r   r4   r4   M   s    ;r   r4   c                   H    e Zd ZdZdedeej                     dedefdZ	d Z
y)	UnsynchronizedAccessErrorzIStores information about two unsynchronized accesses to one data pointer.data_ptrallocation_stack_tracecurrent_accessprevious_accessc                 <    || _         || _        || _        || _        y N)r7   r8   r9   r:   )r   r7   r8   r9   r:   s        r   __init__z"UnsynchronizedAccessError.__init__T   s#     !&<#,.r   c           
      n   dt         ffd}t        j                         5 j                  t	        j
                  d| j                   d| j                  j                   d              || j                         j                  d| j                  j                   d        || j                         | j                  r<j                  dd	j                  | j                  j                                       nj                  d
       j                         cd d d        S # 1 sw Y   y xY w)Naccessc                    j                  | j                   d| j                          | j                  rJj                  ddj	                  | j                        z          | j
                  rj                  d       | j
                  rj                  d       j                  ddj	                  | j                  j                                d       y )N
z argument(s) z, z, and toz the outputz
With stack trace:
 )writer'   r$   r(   joinr)   r*   format)r?   messages    r   format_accessz8UnsynchronizedAccessError.__str__.<locals>.format_accessa   s    MMV__-R}=>~~o		&..0IIJ##MM*-m,MM'0B0B0I0I0K(L'MRPr   z                    ============================
                    CSAN detected a possible data race on tensor with data pointer z&
                    Access by stream z$ during kernel:
                    zPrevious access by stream z during kernel:
z'Tensor was allocated with stack trace:
rB   z&Trace for tensor allocation not found.)r#   ioStringIOrC   textwrapdedentr7   r9   r&   r:   r8   rD   rE   getvalue)r   rG   rF   s     @r   r   z!UnsynchronizedAccessError.__str__`   s   
	& 
	 [[] 	&gMMTTXTaTaSb c&&*&9&9&@&@%A B $--.MM,T-A-A-H-H,IIZ[ $../**>wwt::AACDEG
 FG##%1	& 	& 	&s   C?D++D4N)r   r   r   r+   DataPtrr	   r1   r2   r#   r=   r   r!   r   r   r6   r6   Q   s@    S
/
/ !))?)? @
/ 	
/
  
/%&r   r6   c                   (    e Zd ZdZdee   fdZd Zy)CUDASanitizerErrorsz4Wrapper class for errors reported by CUDA Sanitizer.errorsc                     || _         y r<   )rP   )r   rP   s     r   r=   zCUDASanitizerErrors.__init__   s	    r   c                 4    dt        | j                         dS )Nz	detected z errors)lenrP   r   s    r   r   zCUDASanitizerErrors.__str__   s    3t{{+,G44r   N)r   r   r   r+   r   r4   r=   r   r!   r   r   rO   rO      s    >t$89 5r   rO   c                   l    e Zd ZU dZeej                     ed<    ee	      Z
ee   ed<   dZee   ed<   y)
TensorInfoa  Stores information about a single tensor and recent accesses to it.

    Args:
        allocation_stack_trace: the stack summary object captured during tensor
            allocation. Can be ``None`` if the allocation wasn't caught by CSAN.
        reads: list of read accesses to the tensor that were performed since
            the last write.
        write: the last write access to the tensor.
    r8   )default_factoryreadsNrC   )r   r   r   r+   r	   r1   r2   r,   r   listrW   r   r#   rC   r!   r   r   rU   rU      s<     %Y%;%;<<5E4<5"E8F"r   rU   c                       e Zd ZddZdeddfdZdeddfdZdedeej                     ddfdZ
deddfd	Zdedefd
Zdedeej                     fdZdedee   fdZdedee   fdZdededdfdZdededdfdZy)_TensorsAccessedreturnNc                     i | _         y r<   accessesr   s    r   r=   z_TensorsAccessed.__init__   s	    35r   r7   c                 r    || j                   vr)t        j                  d|       | j                  |d        y y )NzFound tensor with pointer: %s, but no matching tensor allocation in the trace. Backfilling the trace now. Perhaps the sanitizer was enabled after some torch operations?)r^   loggerinfocreate_tensorr   r7   s     r   ensure_tensor_existsz%_TensorsAccessed.ensure_tensor_exists   s<    4==(KKQ 	 x. )r   c                 p    || j                   v r(t        j                  d|       | j                  |       y y )NzFound duplicate tensor allocation in the trace for tensor with pointer: %s. Assuming the trace for tensor deallocation wasn't caught and backfilling it now. Perhaps the sanitizer was enabled after some torch operations?)r^   r`   ra   delete_tensorrc   s     r   ensure_tensor_does_not_existz-_TensorsAccessed.ensure_tensor_does_not_exist   s:    t}}$KKQ  x( %r   r*   c                 4    t        |      | j                  |<   y r<   )rU   r^   r   r7   r*   s      r   rb   z_TensorsAccessed.create_tensor   s     #-["9hr   c                     | j                   |= y r<   r]   rc   s     r   rf   z_TensorsAccessed.delete_tensor   s    MM(#r   c                 <    | j                   |   j                  rdS dS )NTFr^   rW   rc   s     r   !were_there_reads_since_last_writez2_TensorsAccessed.were_there_reads_since_last_write   s    }}X.44t?%?r   c                 4    | j                   |   j                  S r<   )r^   r8   rc   s     r   get_allocation_stack_tracez+_TensorsAccessed.get_allocation_stack_trace   s     }}X&===r   c                 4    | j                   |   j                  S r<   )r^   rC   rc   s     r   	get_writez_TensorsAccessed.get_write       }}X&,,,r   c                 4    | j                   |   j                  S r<   rl   rc   s     r   	get_readsz_TensorsAccessed.get_reads   rr   r   r?   c                 T    | j                   |   j                  j                  |       y r<   )r^   rW   appendr   r7   r?   s      r   add_readz_TensorsAccessed.add_read   s    h%%,,V4r   c                 T    || j                   |   _        g | j                   |   _        y r<   )r^   rC   rW   rw   s      r   	set_writez_TensorsAccessed.set_write   s$    (.h%(*h%r   r[   N)r   r   r   r=   rM   rd   rg   r	   r1   r2   rb   rf   r0   rm   ro   r#   rq   r   rt   rx   rz   r!   r   r   rZ   rZ      s   6/W / /	)W 	) 	)::.6y7M7M.N:	:
$g $$ $@' @d @>>	)((	)>
-' -hv.> --' -d6l -5 5& 5T 5+' +6 +d +r   rZ   c                      e Zd ZddZdeddfdZdeddfdZdeddfdZdeddfd	Z	deddfd
Z
deddfdZdededdfdZdededdfdZdeeef   deeef   ddfdZdededdfdZdeddfdZdeddfdZddZdedededefdZy)StreamSynchronizationsr[   Nc                 X    i | _         i | _        i | _        | j                  t               y r<   )current_sync_statesrecorded_sync_stateshost_sync_statecreate_streamDEFAULT_STREAM_IDr   s    r   r=   zStreamSynchronizations.__init__   s)    KM KM!79,-r   r&   c                 p    || j                   vr(t        j                  d|       | j                  |       y y )NzFound Stream with id: %s, but no matching stream creation in the trace. Backfilling the trace now. Perhaps the sanitizer was enabled after some torch operations?)r   r`   ra   r   r   r&   s     r   _ensure_stream_existsz,StreamSynchronizations._ensure_stream_exists   s<    111KKQ 	 v& 2r   eventc                 p    || j                   vr(t        j                  d|       | j                  |       y y )NzFound Event with id: %s, but no matching event creation in the trace. Backfilling the trace now. Perhaps the sanitizer was enabled after some torch operations?)r   r`   ra   create_eventr   r   s     r   _ensure_event_existsz+StreamSynchronizations._ensure_event_exists   s<    111KKQ 	 e$ 2r   c                 p    || j                   v r(t        j                  d|       | j                  |       y y )NzFound duplicate event creation in the trace for event with id: %s. Assuming the trace for event deletion wasn't caught and backfilling it now. Perhaps the sanitizer was enabled after some torch operations?)r   r`   ra   delete_eventr   s     r   _ensure_event_does_not_existz3StreamSynchronizations._ensure_event_does_not_exist   s<    D---KKQ  e$ .r   c                     || j                   v rt        j                  d|       y d| j                  |<   | j                  j	                         | j                   |<   y )NzFound duplicate Stream creation in the trace for Stream with id: %s. PyTorch Streams are only created once, so this trace entry is ignored.r   )r   r`   ra   r   copyr   s     r   r   z$StreamSynchronizations.create_stream   sW    T---KK* 	 ,-D  (/3/C/C/H/H/JD$$V,r   c                 D    | j                  |       i | j                  |<   y r<   )r   r   r   s     r   r   z#StreamSynchronizations.create_event  s     ))%0+-!!%(r   c                 @    | j                  |       | j                  |= y r<   )r   r   r   s     r   r   z#StreamSynchronizations.delete_event  s    !!%(%%e,r   r%   c                 J    | j                  |       || j                  |   |<   y r<   )r   r   )r   r&   r%   s      r   update_seq_numz%StreamSynchronizations.update_seq_num  s%    ""6*3:  (0r   c                     | j                  |       | j                  |       | j                  |   j                         | j                  |<   y r<   )r   r   r   r   r   r   r   r&   s      r   record_statez#StreamSynchronizations.record_state  sB    !!%(""6*+/+C+CF+K+P+P+R!!%(r   stateotherc                 r    |j                         D ]$  \  }}t        |j                  |d      |      ||<   & y N)itemsmaxget)r   r   r   r&   r%   s        r   _state_wait_for_otherz,StreamSynchronizations._state_wait_for_other  s:      %{{} 	@OFG		&" 5w?E&M	@r   c                     | j                  |       | j                  |       | j                  | j                  |   | j                  |          y r<   )r   r   r   r   r   )r   r&   r   s      r   stream_wait_for_eventz,StreamSynchronizations.stream_wait_for_event#  sG    ""6*!!%(""$$V,d.G.G.N	
r   c                     | j                  |       | j                  j                         D ]  }| j                  ||        | j	                  | j
                  | j                  |          y r<   )r   r   keysr   r   r   r   r   s      r   all_streams_wait_for_eventz1StreamSynchronizations.all_streams_wait_for_event*  sd    !!%(..335 	6F&&vu5	6 	""  $";";E"B	
r   c                     | j                  |       | j                  j                         D ]!  }| j                  || j                  |          # | j                  | j                  | j                  |          y r<   )r   r   valuesr   r   r   r&   r   s      r   all_streams_wait_for_streamz2StreamSynchronizations.all_streams_wait_for_stream3  sq    ""6*--446 	PE&&ud.F.Fv.NO	P 	""  $":":6"B	
r   c                     | j                   j                         D ]  \  }}||   | j                  |<    | j                   j                         D ]  }| j	                  || j                           y r<   )r   r   r   r   r   r   s      r   sync_all_streamsz'StreamSynchronizations.sync_all_streams<  sp    !55;;= 	9MFE+0=D  (	9 --446 	DE&&ud.B.BC	Dr   current_streamother_streamc                     | j                  |       | j                  |       || j                  |   j                  |d      k  S r   )r   r   r   )r   r   r%   r   s       r   is_ordered_afterz'StreamSynchronizations.is_ordered_afterC  sD     	"">2""<0$22>BFF|UWXXXr   r{   )r   r   r   r=   r.   r   EventIdr   r   r   r   r   r-   r   r   r   r   r   r   r   r   r0   r   r!   r   r   r}   r}      sY   .'H ' '%' %d %	%' 	%d 	%
KH 
K 
K.' .d .-' -d -;X ; ;4 ;S' S8 S S
@(F*+@4869I4J@	@
H 
W 
 

 
D 

( 
t 
DY&Y17YGOY	Yr   r}   c                      e Zd ZdZddZdedee   dee   dee   d	ed
e	e
ee   f   dee   fdZdeddfdZdeddfdZdededdfdZdededdfdZdeddfdZdeddfdZdeddfdZddZdeddfdZdeddfdZy)EventHandlerzAnalyzes CSAN trace for synchronization errors.

    Stores information on each stream's synchronizations with other streams as well
    as tensor accesses to determine whether a given kernel launch might cause a
    data race.
    r[   Nc                 N    t               | _        t               | _        d| _        y )Nr   )rZ   tensors_accessedr}   syncsr%   r   s    r   r=   zEventHandler.__init__S  s     0 2+-
 r   r&   	read_only
read_writeoutputsr'   tensor_aliasesc           
      <    dt         dt        dt        t           dd f fd}g  xj                  dz  c_         j                  j                  | j                         t        j                  j                  t        j                  t        j                               d      }|j                          |D ]  }	 j                  j                  |	       t        t        j                    j                  ||||	   |	|v |      }
 ||	|
 j                  j#                  |	              j                  j%                  |	|
        |D ]  }	 j                  j                  |	       t        t        j&                   j                  ||||	   |	|v |      }
 j                  j)                  |	      r+ j                  j+                  |	      D ]  } ||	|
|        n# ||	|
 j                  j#                  |	              j                  j-                  |	|
        S )	Nr7   r9   r:   r[   c                     |y j                   j                  |j                  |j                  |j                        s7j	                  t        | j                  j                  |       ||             y y r<   )r   r   r&   r%   rv   r6   r   ro   )r7   r9   r:   
error_listr   s      r   check_conflictz:EventHandler._handle_kernel_launch.<locals>.check_conflicta  ss     &::..%%'>'>@V@V !!- --HHR&'	r      Flookup_lines)rM   r#   r	   r%   r   r   r1   r2   extract
walk_stackinspectcurrentframereverser   rd   r   r   rq   rx   r    rm   rt   rz   )r   r&   r   r   r   r'   r   r   r*   r7   r9   r:   r   s   `           @r   _handle_kernel_launchz"EventHandler._handle_kernel_launchX  s   		/5	HPQWHX		" 24


!!&$,,7,,44  !5!5!78u 5 

 	! 	EH!!66x@#x(G#N .$*?*?*I*I(*S !!**8^D	E  # 	FH!!66x@#  x(G#N $$FFxP'+'<'<'F'Fx'P NO"8^_MN nd.C.C.M.Mh.W !!++HnE%	F( r   r   c                 :    | j                   j                  |       y r<   )r   r   r   s     r   _handle_event_creationz#EventHandler._handle_event_creation      

&r   c                 :    | j                   j                  |       y r<   )r   r   r   s     r   _handle_event_deletionz#EventHandler._handle_event_deletion  r   r   c                 <    | j                   j                  ||       y r<   )r   r   r   s      r   _handle_event_recordz!EventHandler._handle_event_record  s    

v.r   c                 <    | j                   j                  ||       y r<   )r   r   r   s      r   _handle_event_waitzEventHandler._handle_event_wait  s    

((7r   r7   c                    | j                   j                  |       t        j                  j	                  t        j
                  t        j                               d      }|j                          | j                   j                  ||       y )NFr   )
r   rg   r1   r2   r   r   r   r   r   rb   ri   s      r   _handle_memory_allocationz&EventHandler._handle_memory_allocation  sr    ::8D,,44  !5!5!78u 5 

 	++	
r   c                 p    | j                   j                  |       | j                   j                  |       y r<   )r   rd   rf   rc   s     r   _handle_memory_deallocationz(EventHandler._handle_memory_deallocation  s*    228<++H5r   c                 :    | j                   j                  |       y r<   )r   r   r   s     r   _handle_stream_creationz$EventHandler._handle_stream_creation  s    

  (r   c                 8    | j                   j                          y r<   )r   r   r   s    r   _handle_device_synchronizationz+EventHandler._handle_device_synchronization  s    

##%r   c                 :    | j                   j                  |       y r<   )r   r   r   s     r   _handle_stream_synchronizationz+EventHandler._handle_stream_synchronization  s    

..v6r   c                 :    | j                   j                  |       y r<   )r   r   r   s     r   _handle_event_synchronizationz*EventHandler._handle_event_synchronization  s    

--e4r   r{   )r   r   r   r+   r=   r.   r
   rM   r/   r   intr   r4   r   r   r   r   r   r   r   r   r   r   r   r   r!   r   r   r   r   K  s;   !
HH w<H L	H
 WH H S$s)^,H 
"	#HT'G ' ''G ' '/' /8 / /8 8 8d 8
' 
d 
6G 6 6)h )4 )&7X 7$ 757 5t 5r   r   abr[   c              #   Z   K   | j                         D ]  \  }}||v s||||   f  y wr<   )r   )r   r   argvalues       r   
zip_by_keyr     s7     ggi %
U!8uaf$$%s   ++schemaargs.kwargsc              #     K   | j                   d t        |       }| j                   t        |      d  D ci c]  }|j                  | }}t        ||      E d {    t	        ||      D ]  \  }}}||f  y c c}w 7 %wr<   )	argumentsrS   namezipr   )	r   r   r   schema_argsr   schema_kwargs_argumentr   s	            r   zip_argumentsr     s      "";SY/K.4.>.>s4y{.KLsSXXs]LML;%%%(?  8U 	 M%s   3BA;	BB &Bc                       e Zd ZddZ	 	 ddededee   deddf
dZd	e	j                  d
eedf   deeef   ddfdZdeddfdZy)ArgumentHandlerr[   Nc                 l    t               | _        t               | _        i | _        t               | _        y r<   )setdataptrs_readdataptrs_writtenr   r   r   s    r   r=   zArgumentHandler.__init__  s'    +.5.1e8:%(Ur   r   is_writer   r)   c                    t        |t        j                        r|j                  r|j	                         }|r| j
                  j                  |       n| j                  j                  |       | j                  j                  |g        || j                  |   j                  |       |r| j                  j                  |       y y y y r<   )
isinstancetorchTensoris_cudar7   r   addr   r   
setdefaultrv   r   )r   r   r   r   r)   r7   s         r   _handle_argumentz ArgumentHandler._handle_argument  s     eU\\*u}}~~'H%%))(3""&&x0**8R8##H-44T:  *  0=*r   r   r   .r   c                     t        |||      D ]k  \  }}|j                  d uxr |j                  j                  }t        j                  t        j                  | j                  ||j                        |       m y )N)r   r   )	r   
alias_infor   pytree	tree_map_	functoolspartialr   r   )r   r   r   r   r   r   r   s          r   parse_inputszArgumentHandler.parse_inputs  ss      -VT6B 	OHe**$6W8;N;N;W;WH!!))H8== 		r   r   c                 p    t        j                  t        j                  | j                  dd      |       y )NT)r   r)   )r   r  r  r  r   )r   r   s     r   parse_outputszArgumentHandler.parse_outputs  s+    d33ddS	
r   r{   NF)r   r   r   r=   r   r0   r	   r/   r   r   FunctionSchemar   r   r  r  r!   r   r   r   r     s    + #++ + sm	+
 + 
+($$ CHo S#X	
 

S 
T 
r   r   c                       e Zd ZddZddZy)CUDASanitizerDispatchModeNc                    t               | _        t        j                  j	                          t        j                  | j                  j                         t        j                  | j                  j                         t        j                  | j                  j                         t        j                  | j                  j                         t        j                  | j                  j                         t        j                   | j                  j"                         t        j$                  | j                  j&                         t        j(                  | j                  j*                         t        j,                  | j                  j.                         t        j0                  | j                  j2                         y r<   )r   event_handlerr   _C_activate_gpu_trace	gpu_trace$register_callback_for_event_creationr   $register_callback_for_event_deletionr   "register_callback_for_event_recordr    register_callback_for_event_waitr   'register_callback_for_memory_allocationr   )register_callback_for_memory_deallocationr   %register_callback_for_stream_creationr   ,register_callback_for_device_synchronizationr   ,register_callback_for_stream_synchronizationr   +register_callback_for_event_synchronizationr   r   s    r   r=   z"CUDASanitizerDispatchMode.__init__  sP   )^$$&6655	
 	6655	
 	4433	
 	2211	
 	9988	
 	;;::	
 	7766	
 	>>==	
 	>>==	
 	==<<	
r   c                    |i }t               }|j                  |j                  ||        ||i |}|j                  |       | j                  j                  t        j                  j                         j                  |j                  |j                  z
  |j                  |j                  |j                  |j                        }|r-|D ]  }t        |t        j                           t#        |      |S )N)file)r   r  _schemar  r  r   r   cudar   cuda_streamr   r   r   r   printsysstderrrO   )	r   functypesr   r   argument_handlerr   rP   errors	            r   __torch_dispatch__z,CUDASanitizerDispatchMode.__torch_dispatch__2  s    >F*,%%dllD&A''&&w/##99JJ%%'33**-=-N-NN--$$LL++
  .e#**-.%f--r   r{   )r!   N)r   r   r   r=   r&  r!   r   r   r
  r
    s     
Dr   r
  c                   $    e Zd ZdZddZd Zd Zy)CUDASanitizera  Manages the lifetime of a CUDASanitizer dispatch mode object.

    The CUDASanitizer class wraps the entering/exiting functions of the dispatch mode
    context manager in the enable function/destructor, respectively. This is to
    explicitly set the lifetime of the dispatch mode object to that of the application.
    This approach was deemed more elegant than using the atexit module.
    Nc                 0    t               | _        d| _        y r  )r
  dispatchenabledr   s    r   r=   zCUDASanitizer.__init__U  s    13r   c                 F    | j                   j                          d| _        y )NT)r*  	__enter__r+  r   s    r   enablezCUDASanitizer.enableY  s    !r   c                 X    | j                   r| j                  j                  d d d        y y r<   )r+  r*  __exit__r   s    r   __del__zCUDASanitizer.__del__]  s$    <<MM""4t4 r   r{   )r   r   r   r+   r=   r.  r1  r!   r   r   r(  r(  L  s    5r   r(  c                  ,    t         j                          y)ac  Enable CUDA Sanitizer.

    The sanitizer will begin to analyze low-level CUDA calls invoked by torch functions
    for synchronization errors. All data races found will be printed to the standard
    error output along with stack traces of suspected causes. For best results, the
    sanitizer should be enabled at the very beginning of the program.
    N)cuda_sanitizerr.  r!   r   r   enable_cuda_sanitizerr4  b  s     r   )@r+   r   r  r   rH   loggingr   rJ   r1   dataclassesr   r   typingr   r   r   r   r	   r
   r   r   r   torch.cuda._gpu_tracer  
_gpu_tracer  torch.utilsr   r   torch.utils._python_dispatchr   r   r   r   r   r   rM   r.   r   r-   	getLoggerr   r`   Enumr   r#   	Exceptionr4   r6   rO   rU   rZ   r}   r   r   r  r/   Argumentr   r   r
  r(  r4  r3  r!   r   r   <module>r@     s  
    	  
   ( K K K  ) ) ) :  T]enen

				8	$K K ( ( (.<9 <4& 4 4&n5) 5 # # # 4+ 4+nnY nYb~5 ~5B%$r3w- %DSM %huRc\?R6S %	   	 (-c3h	 AEc3h	 eENNC'()	 .
 .
b: 1 :z5 5, r   