DŽg ddlZddlZddlZddlZddlZddlmZmZm Z m Z m Z m Z m Z mZmZmZddlmZddlZddlmZddlmZddlmZmZmZmZmZmZddlm Z ddl!m"Z"ed e#fZ$ed e#fZ%ejLe'Z(Gd d ejRZ*e*jVd e*jXde*jZde*j\de*j^de*j`de*jbdddiZ2e3e2Dcic]\}}|| c}}Z4GddejRZ5e5Dcic]}||jlc}Z7ejpdddGdd Z9ejpGddZ:ejpdddGdd e9Z;dede ee e;e e;ffd Zd#e edeed$ffd%Z?Gd&d'Z@Gd(d)ZAGd*d+ZBejpGd,d-ZCGd.d/ZDGd0d1ZEejpGd2d3ZFejpGd4d5ZGGd6d7ZHGd8d9ZIycc}}wcc}w):N) Anycast DefaultDictDictIteratorListOptionalSetTupleUnion)Literal)FunctionSchema)_ProfilerResult) _EventType_ExtraFields_Allocation_ExtraFields_TorchOp_ProfilerEvent_TensorMetadata RecordScope) _element_size)_utilsKey TensorKeyceZdZejZejZejZejZejZ ejZ ejZ y)CategoryN) __name__ __module__ __qualname__enumautoINPUT TEMPORARY ACTIVATIONGRADIENTAUTOGRAD_DETAIL PARAMETEROPTIMIZER_STATEg/home/mcse/projects/flask_80/flask-venv/lib/python3.12/site-packages/torch/profiler/_memory_profiler.pyrr*s] DIIKE IJtyy{HdiikO IdiikOr)r darkgreen goldenrodblack mediumpurplered mediumblue royalbluegreyceZdZejZejZejZejZy)ActionN) rrrrr PREEXISTINGCREATEINCREMENT_VERSIONDESTROYr(r)r*r4r4Bs7$))+K TYY[F! diikGr)r4TF)eq unsafe_hashfrozenc,eZdZUejed<y)rdeviceN)rrrtorchr=__annotations__r(r)r*rrLs LLr)cNeZdZUdZeed<eed<defdZdede fdZ defdZ y ) _StoragezBundle storage pointer and id. All profiling logic should use `allocation_id`, however it is useful to print storage pointers for debugging and unit tests sometimes look up values using the storage data pointer of a live Tensor.ptr allocation_idreturncNt|jdd|jdS)Nz>18 ())hexrBrCselfs r*__repr__z_Storage.__repr__\s'dhh-$Bt'9'9&:!<|||t||t||SyrN)rrA)rbrcrCr=s r*_makezTensorKey._makezs/  !')VYm0TU Ur)alloccz|j|j|j|j|jSrN)rer[rBrCr=)clsrfs r*from_allocationzTensorKey.from_allocations)yy599e.A.A5<<PPr)tc|<|j|j|j|j|jSyrN)rer[storage_data_ptrrCr=)rhrjs r* from_tensorzTensorKey.from_tensors1 =99QTT1#5#5qQ Qr)c|j|jj|jj|jj fSrN)r[r\rCr=typeindexrIs r*r`zTensorKey._as_sortables3ww 22DKK4D4DdkkFWFWWWr)N)rrrrUrVr?rArWrKrYra staticmethodr r>r=re classmethodrrirrmpropertyr r`r(r)r*rrfs G I#I6K6D6 C= c]  }   +   Q$;Q@UQQH_5(;:O XeCc3$67XXr)noderDc#4K|j}|jdtjk(r|jdjt j k(r|jdk(r|r|djdtjk(r|djdvr|djdjrdt|djdjdtr7dtj|djdjdfy|jdtjk(r|jd}|j|jJ|jK|jj D]2\}}}tj|tj|f4|jL|jj D]2\}}}tj|tj|f4yyyw)Nrztorch::autograd::AccumulateGrad)z aten::detachz aten::add_)childrentypedrTorchOpscoperBACKWARD_FUNCTIONnameinputsrOrrrmPyCallmodule optimizer parameters)rtrw typed_fields_pp_grads r*!_extract_parameters_and_gradientsrs}}H 1 +++ JJqM  ;#@#@ @ II: :  QK  a J$6$6 6 QK   > > QK  a ' ' x{((+2215 GI))(1+*;*;A*>*E*Ea*HIII A*++ +zz!} ""*l.D.D.LLL    * , 3 3 > > N 1f++A. 0E0Ef0MMM N  ! ! - , 6 6 A A N 61++A. 0E0Ef0MMM N . ,sHHc#@Kt|D] \}}| |ywrNrrtrrs r*extract_parametersrs(6t< 6 =Gsc#DKt|D]\}}| ||fywrNrrs r*extract_gradientsrs/7t< 6  V)Os  event.cg}|rW|jdtjk(r(|j|jdj|j }|rWt |SNrrv)rxrryappendrzparenttuple)rscopess r* get_scopesrsT F  ;;q>Z// / MM%++a... /   =r)ceZdZdZededeeedffdZ ededee dffdZ edefdZ e dedeee dffd Zy ) SchemaMatcheraLookup operator schema based on profiled name. When profiling we record the operator's name but not the schema. However some analysis requires that information. Fortunately we can look up registered schema from the recorded name. We do not, however, record the overload and so we must compare the profiled arguments with all overloads to determine viable matches. Note: Once https://github.com/pytorch/pytorch/issues/78871 is completed this code will be obsolete. rjrD.c 6d}|j|D]_}|xs|jDcgc]}dc}}t|jD]'\}}||xxt|jddzcc<)at |xsd|j DScc}w)aDetermine which inputs may have mutated based on function schema. Note that we don't need to resolve down to a single schema to perform this analysis. An input is mutable if it is mutable in any overload. In practice, however, it is overwhelmingly common to match a single overload. If we cannot find any valid schema then we must be conservative and assume all inputs are mutable. NFis_writec3 K|]}dywrNr().0rs r* z3SchemaMatcher.inputs_are_mutable..s 8! 8s ) match_schemas arguments enumerategetattr alias_inforr})rhrjmutableschemariargs r*inputs_are_mutablez SchemaMatcher.inputs_are_mutables)-''* IFB1A1A!BA%!BG#F$4$45 I3 gcnnj%HH  I I W8 8qxx 899 "Cs Bctd|jDdtffd tfdj|jxsdDS)Nc3K|]^}t|trtj|n4t|tr#|Dcgc]}tj|c}n|`ycc}wwrN)rOrrrmlistrrjs r*rz.SchemaMatcher.match_schemas..sb  )31o(FI ! !! $;EQ:MA6q)''*6    7s>A+A&A+rDct|jtk(xr(tfdt|jDS)Nc3\K|]#\}}j||j%ywrN) _types_matchro)robserved schema_argrhs r*rz?SchemaMatcher.match_schemas..matches..s0C(Hj  :??;Cs),)lenrallzip)rrh signatures r*matchesz,SchemaMatcher.match_schemas..matchessHv''(C N:sC,/ 6;K;K,LC@ r)c34K|]}|s |ywrNr()rsrs r*rz.SchemaMatcher.match_schemas.. sO1GAJQOs r()rr}rYlookup_schemasr|)rhrjrrs` @@r*rzSchemaMatcher.match_schemassS  XX    t  O 2 2166 : @bOOOr)c t|tjjr(|j }|duxs|j ||St|tjj ry|jtjjjr$t|txrtd|DStjjtftjjtdftjj t"ftjj$t&ftjj(t*ftjj,t.ftjj0t"t&t*t.fff}|D]\}}t||st||cS|duS)NTc3<K|]}t|tywrN)rOrrrs r*rz-SchemaMatcher._types_match..s6-. 1i(6s)rOr>_C OptionalTypegetElementTyperAnyType isSubtypeOfListType ofTensorsrr TensorTyperNoneTyperoBoolTyperYIntTyperV FloatTypefloat ComplexTypecomplex NumberType)rhr schema_typetype_mapjit_typepy_typess r*rzSchemaMatcher._types_matchsp k588#8#8 9%446Kt#Ns'7'7+'N N k588#3#3 4  " "588#4#4#>#>#@ Ah-#62:63  XX ) , XX  T + XX   % XX  s # XX   ' XX ! !7 + XX 4eW"= >K #+ 6 Hh+x0!(H55 64r)r|c| d|vryttjj|S#t$rYywxYw)Nz::)rr>r_jit_get_schemas_for_operator RuntimeError)r|s r*rzSchemaMatcher.lookup_schemas0s@ 4??EF F  s/'/ ;;N)rrrrUrrrr r rYrrrrrqrWrr(r)r*rrs :#7:E(4.RUBUz!OpTree.__init__..Is AOOr)key)experimental_event_tree _root_nodesrsorteddfs _sorted_nodesrJrs r*__init__zOpTree.__init__Gs.!99;"6$((*:S#TUr)c/fKtj|jg|i|Ed{y7wrN)r traverse_dfsrrJargskwargss r*rz OpTree.dfsKs*&&t'7'7I$I&IIIs '1/1.c|jSrN)rrIs r* sorted_nodeszOpTree.sorted_nodesNs!!!r)) rrrrrrrrrsr rr(r)r*rrFsPVV4VJh~&>J"eNC$78""r)rc^eZdZdeddfdZdeeddfdZede de efdZ d e fd Z y) SizeMapop_treerDNcbi|_|jD]X}|jdtjk(r5|j |jdD]}|j |Y|jdtjk(sz|jd}|j|jJ|jA|jjD](\}}}|j ||j |*|j|jjD]C\}}}|j ||j ||D]\}}|j |E[i} |jD]}|jdtjk(s$|jd} tj| } | sKt| j} | j!| | } | | k7sx| d| }t"j%d||jj'| y)Nrrvz vs. z(Mismatch between allocation and free: %s)_valuesrrxrry_flat_tensor_inputs_update_valuesr~rrr Allocationrriabs alloc_size setdefaultlogwarningupdate)rJrrtrjrrrrstate allocations alloc_fieldsrnew_size prior_sizedeltas r*rzSizeMap.__init__Ts-/ (( 3Dzz!} 2 2211$**Q-@+A''*+A*"3"33#zz!} #**2l6L6L6TTT&&2(4(;(;(F(F4 1f++A.++F34 ))5,8,B,B,M,M3(65++A.++F3$)3DAq //233 3(-/ (( WDzz!} 5 55#zz!} // ="<#:#:;H!,!7!7X!FJ"X-#-,eH: > $NPUV! W$ K(r)rjctj|}|||jtjk(rt dt |jxsdg|jxsdgD}|t|jz}|dk\sJ|t |jj|d||j|<yyyy)Nc32K|]}|d|dzywrrvNr(rs r*rz)SizeMap._update_values..sOAAaD1Q4KOsrvr) rrmlayoutr>stridedmaxrsizesstridesrdtyperget)rJrjrn num_bytess r*rzSizeMap._update_valuess##A& ?q}U]]1JOQWW^QYY=M1#)NOOAM!''22I> 1i[ 1> #DLL$4$4S!$ A!_-At$  s5AAAArc |j|SrN)rrJrs r* __getitem__zSizeMap.__getitem__s||C  r))rrrrrr rrrqrrrrrr(r)r*rrSse*)*)4*)XI 9IdI 4/9R!y!r)rcbeZdZUdZeeed<dZeeed<e defdZ e defdZ y) DataFlowEdgeN input_versionFmutatedrDc|jduSrN)rrIs r* is_allocationzDataFlowEdge.is_allocations!!T))r)c|jduSrN)rrIs r* is_deletionzDataFlowEdge.is_deletions||t##r)) rrrrr rVr?rrYrsrrr(r)r*rrsR#'M8C='#GXd^# *t**$T$$r)rceZdZdeddddfdZdeeeffdZe deee e e fffdZ e deee ffd Ze de ed ffd Ze de fd Zy) DataFlowNodergraph DataFlowGraphrDNc ||_||_|j|_|jj D]:\}}|j s|j r |jj|<|jj Dcic]#\}}|||jj|f%}}}td|jDsJ|d|jycc}}w)Nc3,K|] \}}||k(ywrNr(rs r*rz(DataFlowNode.__init__..s8da168sz, ) _event_graph_determine_edges_edgesitemsrrbumpoutputslookuprvalues)rJrrredgekvversionss r*rzDataFlowNode.__init__s  595J5J5L **, &IC||D$6$6   % & ?Cll>P>P>RSdaA4;;--a011SS8hoo&788VXJb :VV8Ts(C:cttj|jg}i}d|DD]}t |j t j|D]\}}t|tr?tj|}|j|tj|Ut|tsf|D]@}tj|}|j|tj|Bt!j"t$}|j'D]M\}} | |r|j(j+|nd||_d| vxst| dk(} | ||_O|D]} | j0dt2j4k(s$| j0dj6dksAtj9| j0d}||} || j. Jd|d| _|r|j(j+|nd| _|D]j} | j0dt2j4k(s$| j0dj6dkDsAd|tj9| j0d_lt;t=d|j'DS) Nc3|K|]4}|jdtjk(s$|jd6ywr)rxrryrs r*rz0DataFlowNode._determine_edges..s-S!!''!* @R@R2R1771:Ss%<<TrNrrvzDouble delete: c30K|]\}}| ||fywrNr(rr)r*s r*rz0DataFlowNode._determine_edges..sMdaq}Aq6Ms  )rrrrrr}rrrOrrrmrsetaddr collections defaultdictrr#r r&rrrxrrrridictr) rJsubtreemutable_by_keyr op_inputrr op_input_iedges mutable_setrrr(s r*r!zDataFlowNode._determine_edgess++T[[M:;JLSwS KB%( =;;B?& K!'h8#//9C"--c359==gF $/&.K '33J?&11#su=AA'JK K K '' 5 . 4 4 6 - CFI4;;+=+=c+Brc (  ;.RE+4F'4Q%,c " - LAwwqzZ222qwwqz7L7Lq7P// ;Sz{dll&>W/RUQV@WW># @CT[[%7%7%<"  L RAwwqzZ222qwwqz7L7Lq7PMQi// ;<J R FMekkmMMNNr)c |jjDcic]A\}}|js0|t|jt t |jfCc}}Scc}}wrN)r"r#rrYrrrVrrJr)r*s r*r}zDataFlowNode.inputss[  ))+  1?? QYYc1??!;< <   sAA(c|jjDcic]G\}}|jr |jr |jr||j dn|j dzIc}}Scc}}wr)r"r#rrrrr=s r*r%zDataFlowNode.outputssa ))+ 1 !)) AOO+q11D D   sA A..cVtd|jjDS)Nc3ZK|]#\}}|js|js |%ywrN)rrr0s r*rz-DataFlowNode.intermediates..s% !QQ]]A s + ++)rr"r#rIs r* intermediateszDataFlowNode.intermediatess) ++++-   r)c.|jjSrN)rrrIs r* start_timezDataFlowNode.start_times{{(((r))rrrrrrrrr!rsr rYrVr}r%rArCr(r)r*rrs Wn W_ W W1O$y,'>"?1Of YdCi(889   in-   uY^4  )C))r)rceZdZdeddfdZedeedffdZdZ edee dffdZ e dedee dffd Z d edefd Zd eddfd Zd eddfd Zy)rrrDNc||_|j||_i|_|jDcgc]}t ||c}|_|j jd|jycc}w)Nc|jSrN)rCrs r*rz(DataFlowGraph.__init__..s ALLr)r) _op_tree_extract_leaf_events _leaf_events_active_version leaf_eventsr _flow_nodessortvalidate)rJres r*rzDataFlowGraph.__init__sk  55g>?A;?;K;KLaLD1L "89 MsA<.c,t|jSrN)rrLrIs r* flow_nodeszDataFlowGraph.flow_nodes sT%%&&r)c0t}|jD]^}t|jj}||z}|r-J|jj d|j d|||z}`i}|jD]}|jjD](\}\}}|j|d} | |k(r!J| |f|jjD]*\}}|j||} || k\s J|| f|||<,y)N r) r1rQr%r#rr|r"r}r ) rJr%rt node_outputs duplicatestensor_versionsrrversionexpected prior_versions r*rNzDataFlowGraph.validates5.1eOO $Dt||1134L </J! Sdkk&6&6%7q Qzl#S S> | #G  $13OO /D%)[[%6%6%8 @!\a*..sA67*?Xw,??* @!% 2 2 4 / W / 3 3C A -/I'=1II/'.$ /  /r)c|jSrN)rIrIs r*rKzDataFlowGraph.leaf_events$s   r)cgdtdtfddtffd }|j|D]}tt dS)aPartially traverse the op tree and extract top level ops. Consider the following code: ``` with record_function("My annotation"): x.zero_() y.zero_() ``` The op tree (assuming no Autograd) will look like: TorchOp: "My annotation" TorchOp: zero_ TorchOp: fill_ TorchOp: zero_ TorchOp: fill_ The recursive structure of operator calls makes data flow unwieldy. In order to simplify analysis we would like to select the highest level ops to represent in the graph. In this case those are the `zero_` ops; the fact that `fill_` is called is an implementation detail. We also do not want to group everything under "My annotation" as this could create overly coarse bundles and lose critical semantics. To address this issue we walk over the graph and select the topmost torch ops ** which match at least one operator schema **. These form the leaves of the first pass through the op tree. (As well as any allocations or frees which do are not part of a kernel.) These events form the logical nodes in our data flow graph. rOrDc|jdtjk(xrW|jdjtj k(xs+t tj|jdSr) rxrryrzrr{rYrr)rOs r*leaf_opz3DataFlowGraph._extract_leaf_events..leaf_opKs_771:!3!33   K$A$AAA 33AGGAJ?@ r)c|s|jtjk(rj|gS|jSrN)tagrrrrw)rOrKr]s r* children_fnz7DataFlowGraph._extract_leaf_events..children_fnQs8qzQUUj&;&;;""1% :: r))r`c|jSrNrrs r*rz4DataFlowGraph._extract_leaf_events..[s qr)r)rrYrrr)rr`rrKr]s @@r*rHz"DataFlowGraph._extract_leaf_events(s^B-/  ~ $   > 5 A  VK-FGHHr)rcF|jj|d}|J|SNrrJr)rJrrWs r*r&zDataFlowGraph.lookup]s+&&11#q9"""r)ch|jj|d}|J|dz|j|<y)Nrv)rJr )rJrrYs r*r$zDataFlowGraph.bumpbs;,,00d; ((($1A$5S!r)c^|jj|dJd|j|<yrcrdrs r*deletezDataFlowGraph.deletegs1##..sA6BBB$(S!r))rrrrrrsr rrQrNrrKrqrHrrVr&r$rgr(r)r*rrs4'E,"34''/*!U>3#67!!2If2I~s7J1K2I2Ih) 6 6d6 ))))r)rceZdZUdZeeed<eje Z e e efed<eje Z e eefed<ejeZee ed<y)CategoryElementNby_iddefault_factoryby_key by_version _by_id_keyset)rrrrjr rr? dataclassesfieldr5rmrrrn TensorAndIDr1ror r(r)r*ririlsq $E8H $(9 (9(9$(OFDH$ %O.?k.?.?PT.UJ[(*+U%6K$5$5c$JM3y>Jr)riceZdZUejdZeeefe d<de de ddfdZ de de ddfd Z de d ede ddfd Zde d ede ddfd Zded edee fd Zy) CategoryDictc4tjtSrN)r3r4rir(r)r*rzCategoryDict.zs 7 7 Hr)rkrrcategoryrDNc||j|j_|j|jjj |yrN)rr[rjror2rJrrvs r* set_by_idzCategoryDict.set_by_id}s8%- SVV" SVV**..s3r)cP||j|jj|<yrN)rr[rmrxs r* set_by_keyzCategoryDict.set_by_keys+3 SVV##C(r)rWcT||j|jj||f<yrN)rr[rnrJrrWrvs r*set_by_versionzCategoryDict.set_by_versions#:B SVV''g7r)cn|j|jjj||f|yrN)rr[rnrr}s r*setdefault_by_versionz"CategoryDict.setdefault_by_versions+ SVV''22C>8Lr)c t|trt|tsy|j|j}|j xs<|j j|dxs|jj||fdSrN) rOrrrr[rjrmr rn)rJrrWelements r*r zCategoryDict.getsn c3  3 (B,,svv& MM <~~!!#t, <!!%%sGnd; r))rrrrprqrrrVrir?rrryr{r~rrr r r(r)r*rtrtws1B1B1BH2G[o- .4Y4(4t44i4844C)CcCXCRVCMM'*M6>M M  s S Xh-? r)rtceZdZdeddfdZedeeeee efdffdZ de fdZ de eeeffdZdeefd Zdd Zdd Zdd Zdd ZddZddZdZy) MemoryProfilerrDNct||_t|j|_t |j|_t |_|j|j|j|j|j|j|jyrN)rrGr_data_flow_graphr _size_maprt _categories_set_gradients_and_temporaries#_set_parameters_using_python_tracer _set_inputs_set_parameters_using_data_flow_set_activations_set_optimizer_state_set_autograd_detailrs r*rzMemoryProfile.__init__sv -dmm < /'> ++- 002  ,,.  !!# !!#r).c ^g}i}i}jjD]S}|jdtjk(s%|jd}|j }|dkD}|j }tj|} | ||| |f<pt|j} |j| jf} |rU| |vr&|j|tj| df|fd|| <|j|tj| df|f|j|tj | df| f|j#| dr.|jdtj$| df| fVj'} t)t+| j-} | j-D cgc]%\} }| df|vr|dk(rdtj$| |ff'}} }j.j0D]}|j2j5D]\} }|j6r,|| df}|j|tj| dffnV|j8rJ|j:j }|j<}|J|j|tj| |ff|j>s|| df}|j|tj | | | ff|jAfd|D|jCdtE|Scc}} w) NrrvTFr.c3VK|] \}}\}}||||fj|f"ywrN)r)rtimeactionrrWrJs r*rz)MemoryProfile.timeline..s9 ,fnsG6C>4>>#+> ? s&)c*|d|djfSr)valuers r*rz(MemoryProfile.timeline..s1Q41"4r)r)#rGrrxrrrrrrirr=rBrr4r7r6r8popr5_category_snapshotr5rkeysrrQr"r#rrrrrextendrMr)rJoutputallocation_times live_unknownrrrrrjtkeyrptr_and_devicesnapshot last_versionrWeventsrtr(s` r*timelinezMemoryProfile.timelines :<>@FH ]]&&( E{{1~!6!66${{1~ )44 *Q '' 00>#>?$dM%:;l112C&2&6&6 %CN$)\9"MM!"F$<$?\\ 11A"00G"...MM1f&>&>g"OP##(#u6AMM1fnnsL*L)c\|jj|i|tjk(SrN)rr rr$rs r* _is_gradientzMemoryProfile._is_gradients+#t##T4V48I8IIIr)c Zt}|jjD]}}|jd|jj D|jd|j D|j|jj |jjjD]$}|jd|jD&t|Dcic]$\}}||f|jj||&c}}Scc}}w)Nc30K|]\}\}}||fywrNr()rr)rr*s r*rz3MemoryProfile._category_snapshot..s'T91fq!A'Tc3$K|]}|df ywrNr(rrs r*rz3MemoryProfile._category_snapshot..s&NCQx&Nc3$K|]}|df ywrr(rs r*rz3MemoryProfile._category_snapshot..s&KCQx&Kr)r1rrQrr}r#rAr%rrr'rorr )rJall_tensor_versionsrtrrrWs r*rz MemoryProfile._category_snapshots03))44 =D  & &'T @Q@Q@S'T U  & &&N4;M;M&N N  & &t||'9'9'; < = !!))002 LA  & &&K1??&K K L !'': ; W'ND,,00g> >   s:)D'cPt t}jjD]e}t fd|j j D}|s3j|jd|jDgt|k(rS)a\Extract IDs of Tensors which depend or will depend on a gradient. Note that this weakened definition of "depends" requires us to loop over the data flow graph multiple times because it allows dependency information to flow backward through edges and removes the guarantee that nodes are topologically sorted. (Or indeed, even that a valid topological order exists.) Put another way, we have converted an acyclic data flow graph into a cyclic graph and we are attempting to partition cycles involving a gradient from the rest of the graph. c3K|]`\}\}}jj||tjtjfvs|j vr|j bywrN)rr rr$r&r[)rrrrWdepends_on_gradientrJs r*rzAMemoryProfile._any_version_depends_on_gradient.. sa)\a''++C9 ))8+=+=>?vv!44 FFsA&A)c34K|]}|jywrN)r[rs r*rzAMemoryProfile._any_version_depends_on_gradient..s.N#svv.Ns) r1rrrQrr}r#rr%)rJ start_sizertidsrs` @r* _any_version_depends_on_gradientz.MemoryProfile._any_version_depends_on_gradients),01J--88 O-1[[->->-@'..s3'...N.NN O$&':5**+r)ch|jjD]?}t|D]/\}}|jj |t j 1A|jjD]=}|jD],}|jj|t j.?y)z>Mark Tensors which are unambiguous and simple to reason about.N) rGrrrryrr$rrQrAr{r")rJrrrrtrs r*rz,MemoryProfile._set_gradients_and_temporariess ]]&&( FE.u5 F 6  **683D3DE F F))44 CD'' C  ++Ax/A/AB C Cr)c|jjD]?}t|D]/}||jj |t j 1AyrN)rGrrrryrr&)rJrrs r*rz1MemoryProfile._set_parameters_using_python_tracer0sW]]&&( FE'. F=$$..q(2D2DE F Fr)cj}ttjjD]m}|j j Dchc] \}\}}||f}}}}||jj z}tfd|Dsi|zoj}jjD]N}tjt|jvs)|t|jj z}P|D]?\}}|j|vsjj!||t"j$Aycc}}}w)aMark inputs based on which Tensors are updated using gradients. The process for differentiating between inputs and activations is more involved. Most Tensors in a training loop depend on at least one gradient: parameters depend on them through updates, and activations and optimizer state depend on them transitively through parameters. Critically, we do not need to know which Tensors are parameters to apply this method; we can simply walk the data flow graph to build the set of all values which depend on a gradient and then obtain the set of inputs from the conjugate set. There is, however, one hiccup. The first time we see a parameter is generally on the forward pass of the first step. We know from inspection of the data flow graph that v1 of that Tensor depends on a gradient (provided we profile an optimizer step), but not v0. To address this problem we weaken the definition of "depends on a gradient" to "any version of this Tensor depends on a gradient", which in turn strengthens the criteria for the input set enough to filter the activations in the forward pass of the first step.c3K|]C}jj|tjtjfvxs|vEywrN)rr rr$r&)rrproduces_gradientrJs r*rz,MemoryProfile._set_inputs..WsW%  $$a(X->->@R@R,SS*))*sA A N)rr1reversedrrQr}r#r%anycopyrr{rrr[rrrr!) rJrrtrrrWtensorsinput_candidatesrs ` @r*rzMemoryProfile._set_inputs6sO2#CCE/2eT22==> -D?C{{?P?P?RSS*;#|7W~SGS t||))+ +G! "W,! --113))44 >D,, 4;;0GG C (:(:(<$==  >- ULCvv00  66sGX^^T U!TsE( c j}t}|jDchc]\}}|tjk(s|}}}j j D]}|jjD chc] \}\}} || f} }}} tjt|jvs\tfd| Drqtfd|jjDr|j| s||jjz}|| j|z}t t!j j D]`}t fd|jjDs1 j#d|jjDb|j% |Dchc]\}}|j&} }}| j)z} |j+D]>\}}|j&| vsj,j/|tj0@ycc}}wcc} }}wcc}}w)aDeduce which Tensors are parameters. Consider the following code for the step of SGD with momentum (nesterov=False), where `d_p` is the gradient of `param` and `buf` is the momentum buffer. ``` buf.mul_(momentum).add_(d_p, alpha=1 - dampening) d_p = buf param.add_(d_p, alpha=-lr) ``` Both `param` and `buf` take a gradient and perform an in-place update. The python tracer will inspect calls to `nn.Module.forward` and `optim.Optimizer.step` to extract parameter and optimizer state respectively (including parameters), so this is generally a non-issue. However as a fallback we can also exploit several properties of parameters to distinguish them from other model state. First, they are directly used in the forward pass. (At this point we haven't established which parts of the graph correspond to the forward pass but we can deduce enough to suffice.) Some mutable state such as batch norm moving averages also contribute to the forward pass, but optimizer state does not. Second, a parameter is by definition used to compute at least one gradient and depends on at least one gradient. c3<K|]}j|ywrNrrrrJs r*rz@MemoryProfile._set_parameters_using_data_flow..sBa-D--q1Bc3<K|]}j|ywrNrrs r*rz@MemoryProfile._set_parameters_using_data_flow..sPa-D--q1Prc3HK|]}j|xs|vywrNr)rrrJused_for_gradients r*rz@MemoryProfile._set_parameters_using_data_flow..s4"!!1%?.?)??s"c30K|]\}\}}||fywrNr()rrrrWs r*rz@MemoryProfile._set_parameters_using_data_flow..s!)'8sLQS'N)rN)rr1r#rr!rrQr}rr{rrrr% intersection differencerrintersection_updater[rrrryr&) rJrcandidate_parametersrrvcandidate_fwd_tensorsrtrrrr}parameter_keysrs ` @r*rz-MemoryProfile._set_parameters_using_data_flowis*:**, 25!)!13 !XX5OA3 3 ))44 QD:>++:K:K:MNNsJQsElNFN--Z 5LLB6BBP4<<;M;M;OPP*66v>%););)==%$(9(9:O(PP$ Q/2eT22==> D++-"(()<@KK,>? ))44 TD:>++:K:K:MNNsJQsElNFNBHIQ 4 0 0 4 4a 8I I"H,)X -DE 11DKK9PP++-TA:D$$::SASx?R?RST TNIs 8D6  D=c |jjD]}|jdtjk(s$|jdj s>|jdj j }tj|Dcgc]\}}}| c}}D]G\}}tj|}||jj|tjIycc}}wr)rGrrxrr~rritchainrrmrryrr')rJrrrrrjrs r*rz"MemoryProfile._set_optimizer_states]]&&( RE{{1~!2!22u{{1~7O7O"[[^55@@ HH &KAuu&KLRDAq#//2C((2238P8PQR R'Ls C/ cdtjh}|jjD]}tj t |jvs)|jjD]W\}}|dk(s"|jj||dz |vs-|jj||tjYyr) rr%rrQrr{rrr%r#rr r)rJpriorrtrrWs r*rz"MemoryProfile._set_autograd_detailsx//0))44 D,, 4;;0GG$(LL$6$6$8LC!|t'7'7';';C1'MQV'V((>>(*B*B r)rDN)rrrrrrsr rVr4KeyAndIDrrYrrrrr rrr rrrrrrrrr(r)r*rrs $ $4 $@%c68S&@ A3 FG@@DJtJ Dhx6H)H$I !+#c(!+FC"F 1UfHDTT&Rr)rc6eZdZdZdZddZddZ d ddZy) MemoryProfileTimelinecH|j|_|j|_y)aThe minimum representation of the memory profile timeline includes the memory timeline and categories. The timeline consists of [timestamp, action, (TensorKey, version), numbytes] elements, to denote any actions (pre-existing, create, destroy, or increment_version) that occurred to a specific Tensor for a chunk of memory. The categories help map each (TensorKey, version) pair into a category.N)rr categories)rJmemory_profiles r*rzMemoryProfileTimeline.__init__s'// (44r)c> tj|}g}g fd}d}jD]G\}}\}} } |j|k7r|dk7rt|dz }|dk(s ||kr|dkDr|}t |dk(r9|j | j dgt D cgc]} dc} zn;||dk7r3|j | j dj|tjtjfvr ||| | |tjk(r||| | ||| dz| |tjk(r ||| | .updatesVc9-##C1  'x014E "Ie E * r)r.irrvUnknown action: )r>r=rrVrrrrr4r5r6r7r8 ValueError) rJ device_strr=timesrt_minrjrrrWnumbytesrrs ` @r*_coalesce_timelinez(MemoryProfileTimeline._coalesce_timelinesj)!# +37==! > /Av~WxzzV#BwDM{q5yQU5zQ Q aS/A#B!A#BBCeBi Q U2Y^^-.&,,fmm<<sGX.6333sGhY/sGaK26>>)sGhY/!#3F8!<==C! >F1661!a%Q&66e|+$C(7s $ F ?FNc|j|\}}ddl}t|d5}|j||g|dddy#1swYyxYw)zSaves the memory timeline as [times, sizes by category] as a JSON formatted file to the given path for the given device.rNw)rjsonopendump)rJpathrrrrfs r*export_memory_timelinez,MemoryProfileTimeline.export_memory_timeline"sM..z: u $_ ) IIuena ( ) ) )s AA c tj|}g}fd}jD]\}}\}} } |j|k7r|tjtj fvr$|j |t|| ||| fb|tjk(rK|j |t|| ||| f|j |t|| ||| dzf|tjk(r%|j |t|| ||| ftd|ddl } t|d5} | j|| dddy#1swYyxYw)zSaves the memory timeline as raw memory event tuples in the form of (timestamp, action, numbytes, category) as a JSON formatted file to the given path for the given device.crt|trjj||nd}t|SrN)rOrrr r)rrWrvrJs r*get_category_indexzLMemoryProfileTimeline.export_memory_timeline_raw..get_category_index5s:c9-##C1  &h/ /r)rvrrNr)r>r=rr4r5r6r_ACTION_TO_INDEXr7r8rrrr) rJrrr= raw_eventsrrjrrrWrrrs ` r*export_memory_timeline_rawz0MemoryProfileTimeline.export_memory_timeline_raw-s j)68  048==+ > /Av~WxzzV#&,,fmm<<!!(0 *38 6333!!(0! *38 !!(0 *3! < 6>>)!!(0! *38 !#3F8!<==W+ >Z  $_ % IIj! $ % % %s EE&c ddl}|jjd}| tdyddlm}ddlm}ddlm } ddl m } ddl } |j|} | j| d| j| d}} t| }| |z} | j!|d d z }t#j$|}t"j&j)|}t"j&j+|}| j-|d }|j/}t0j3D]8\}}t4|}|j7| d z |dd|f|dd|dzf|d:|j9t0Dcgc]}|dn |j:c}|j=d|j?ddjA|r|gngd|d z dd|d z ddgz}|jC|| ddd}|jE|jG|j:dtI|j:d5}||jKjMd}d |d!}tI|d"5}|jO|dddddd||j:ycc}w#1swY)xYw#1swY-xYw)#zsExports the memory timeline as an HTML file which contains the memory timeline plot embedded as a PNG file.rN matplotlibzDexport_memory_timeline_html failed because matplotlib was not found.) b64encode)remove)NamedTemporaryFilerv)axisi@P)figsizedpig@@gffffff?)coloralphaUnknownz Time (ms)z Memory (GB)z zMax memory allocated: z.2fz GiB Max memory reserved: z GiBwbz.pngF)suffixrgpng)formatrbzutf-8z} GPU Memory Timeline HTML r)(importlib.utilutil find_specprintbase64rosrtempfilermatplotlib.pyplotpyplotnumpyrarraymincumsumr>r=cudamax_memory_allocatedmax_memory_reservedfiguregca_CATEGORY_TO_COLORSr#r fill_betweenlegendr| set_xlabel set_ylabeljoin set_titleclosesavefigrreaddecodewrite)rJrrrtitle importlibmatplotlib_specrrrpltnpmtrrrstackedr=rrfigaxesrvrrtmpfiletmpencodedhtmlrs r*export_memory_timeline_htmlz1MemoryProfileTimeline.export_memory_timeline_htmlos #..22<@  " V  $/'  $ $Z 0xx1AuE  ))E)*W4j)$zz>>vF#jj<torch._Crtorch._C._autogradrtorch._C._profilerrrrrrr torch._utilsrtorch.profilerrrVrrr getLoggerrrEnumrr&r'r!r"r#r$r%rrrr4rr dataclassrrArrrrrrrrrrrrirtrr)rcs00r*rBs    & #.'!  K$% g!"tyy"   k NNG   | k& (11D'EFtq!adFTYY)//1AqwwJ/$E$?@ (((($D>.X.X?.Xb'N 'N eHY'))<<=>'NT^0C   eHY'234h~.5c9I3JjjZ " "@!@!F $ $ $[)[)|g)g)T KKK    >@@F ZZwG0s 7 I1!I7