¯wg`ddlZddlZddlZddlZddlmZmZmZmZddl Z ddl m cm Z ddl mZmZmZmZmZddlmZddlmZmZmZGddZGdd eZGd d eZGd d eZGddeZGddeZGddeZGddeZ GddeZ!GddeZ"deefdZ#defdZ$defdZ% d"de&de&d ee'fd!Z(y)#N)DictListOptionalSet) _EventType_ExtraFields_PyCall_ExtraFields_PyCCall_ExtraFields_TorchOp_ProfilerEvent)profile)index_of_first_match traverse_bfs traverse_dfsceZdZdZddedefdZedZde fdZ dZ d e e fd Z d e e fd Zde fd Zd Zde fdZde fdZde fdZde fdZde fdZy)Patternz Base class for all patterns, subclass this class and implement match() to define custom patterns. In subclass, define description and skip property. profshould_benchmarkc~||_||_d|_d|_d|_|j |j j J|j j j|_i|_ |jD]7}|jj|jgj|9y)Nz!Please specify a name for patternz(Please specify a description for pattern) rrname descriptionurlprofilerkineto_resultsexperimental_event_tree event_treetid_root setdefault start_tidappend)selfrrevents d/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/torch/profiler/_pattern_matcher.py__init__zPattern.__init__s 07 E}}(T]]-I-I-UUU--66NNP9; __ HE MM $ $U__b 9 @ @ G HcyNFr!s r#skipz Pattern.skip)sr%r"c:|jdt|}|S)Nz [Source Code Location] )rsource_code_location)r!r"msgs r#reportzPattern.report-s+ 9:Nu:U9V W  r%c#JKt|jEd{y7w)z Traverse the event tree and yield all events. Override this method in subclass to customize the traversal. N)rrr)s r#eventTreeTraversalzPattern.eventTreeTraversal3s  000 #!#eventsc|jdt|d}|jrt|dr|j |S|S|S)N: z events matched. benchmark)rlenrhasattrbenchmark_summary)r!r2default_summarys r#summaryzPattern.summary:sY!YYKr#f+6FG  4-&&v. %  r%c dtfd}t|dsJd|j|td|D}tfd|D}|jdt |d|||z d t ||z d d S) Ntime_nscLgd}|D]}|dkr |dd|cS|dz}|ddS)N)nsusmsiz.2f z sr()r<unit_lstunits r# format_timez.Pattern.benchmark_summary..format_timeFsK)H  !T>%c]!D622D  !c]"% %r%r5zPlease implement benchmark()c34K|]}|jywN)duration_time_ns).0r"s r# z,Pattern.benchmark_summary..PsGuE22Gsc3TK|]}t||jz!ywrF) input_shapesrG)rHr"shapes_factor_maps r#rIz,Pattern.benchmark_summary..Qs.  l51 2U5K5K K s%(r4z* events matched. Total Estimated Speedup: z (zX))intr7r5sumrr6round)r!r2rD original_timenew_timerLs @r#r8zPattern.benchmark_summaryEs & &t[)I+II) NN62GGG     yykCK=)((3MH4L(M'NbQVWdemWmopQqPrrt v r%ct)zt Return True if the event matches the pattern. This method should be overriden in subclass. )NotImplementedErrorr!r"s r#matchz Pattern.matchZs "!r%c|jrgSg}|jD]%}|j|s|j|'|SrF)r*r0rVr )r!matched_eventsr"s r#rXzPattern.matched_eventsasM 99I,,. -Ezz% %%e, -r%cP|jr|j}|jr|SrFparentrUs r#root_ofzPattern.root_ofjs llLLEll r%c|jr|jj}n|j|j}|j |}|d|||dzdfS)N)r[childrenrrindex)r!r"r_r`s r# siblings_ofzPattern.siblings_ofosU <<||,,H}}U__5Hu%%!)+!666r%c<|j|\}}|r|dSdS)Nrra)r!r"_ next_eventss r#next_ofzPattern.next_ofws'))%0;!,{1~6$6r%c<|j|\}}|r|dSdS)Nrc)r!r" prev_eventsrds r#prev_ofzPattern.prev_of{s'))%0 Q"-{2747r%cv|sy|jr)||s!|j}|jr ||s!|SrFrZ)r!r" predicates r# go_up_untilzPattern.go_up_untils4ll9U#3LLEll9U#3 r%NF)__name__ __module__ __qualname____doc__r boolr$propertyr*r r.r0rr:r8rVrXr\rarfrjrmr(r%r#rrs HW H HN 1 d>2  ^(< *">"^ 777^78^8r%rc:eZdZddededeffd ZdefdZxZ S) NamePatternrrrcHt|||d||_||_y)NzMatched Name Event: )superr$rr)r!rrr __class__s r#r$zNamePattern.__init__s) /01$8 r%r"cZtj|j|jduSrF)researchrrUs r#rVzNamePattern.matchs yyEJJ/t;;r%rn) rorprqr strrsr$r rV __classcell__rys@r#rvrvs)WC4 <> aten::fill_ aten::normal_aten::uniform_ aten::zero_)rxr$rrrinit_opsr!rrrys r#r$zExtraCUDACopyPattern.__init__s5 /0- nB  r%cb|jj xs|jj SrFr with_stack record_shapesr)s r#r*zExtraCUDACopyPattern.skip'99'''Ftyy/F/F+FFr%c(|jdk7ry|}|jsy|jd}|jdk7ry|jsy|jd}|jdk7ryt|}t|dkry|d |d|dk7ry|}|j}|y|j |}|y|jr5|jd}|j|j vry |jr5|j|j vS) Nzaten::toFrhzaten::_to_copyz aten::copy_rMrr^T)rr_ input_dtypesr6r[rjr)r!r"to_eventdtypess r#rVzExtraCUDACopyPattern.matchs  :: #~~r" ::) )~~r" :: &e$ v;? !9 q VAY 6  = U# =nnNN2&EzzT]]* nn zzT]]**r%r2c:|Dcic]}t|d}}|D]w}|d}tjdd|i}tjdd|i}|jdj}|jdj} | |z ||<y|Scc}w)Nrztorch.ones(size).to("cuda")sizestmtglobalsztorch.ones(size, device="cuda") )rKr5Timertimeitmean) r!r2r"rLshaperto_timerde_timerto_timede_times r#r5zExtraCUDACopyPattern.benchmarksCIJ%\%0#5JJ& 9E8D 2VTNH!6Hoob)..Goob)..G'.'8 e $ 9! KsBrn)rorprqrrr rsr$rtr*rVrr r5r~rs@r#rrsH  W    GG#+L !^ 4 !r%rc@eZdZdZddedeffd ZdZdefdZ xZ S) ForLoopIndexingPatterna This pattern identifies if we use a for loop to index a tensor that can be vectorized. example: tensor = torch.empty((100, 100)) for i in range(100): tensor[i] = i Pattern: aten::select | ... | aten::select | ... (Repeat) Algorithm: We start at node aten::select, and we check if we can find this alternating patterns. We also keep a dictionary to avoid duplicate match in the for loop. rrc`t|||d|_d|_t |_y)NzFor Loop Indexing Patternz6For loop indexing detected. Vectorization recommended.)rxr$rrsetvisitedrs r#r$zForLoopIndexingPattern.__init__s+ /0/ S!$ r%c#JKt|jEd{y7w)zN We need to use BFS traversal order to avoid duplicate match. N)rrr)s r#r0z)ForLoopIndexingPattern.eventTreeTraversals 000r1r"c |jdk7ry|j|jvryd}|j|\}}t |dkryd}t |d}|y|g|d|z}|t |dz d}t dt |t |D]M}|||||t |zr.|dz }|jj||jI|dk\S|dk\S)N aten::selectFr^ct|t|k7ryt||D] \}}|j|jk7s yy)NFT)r6zipr)list1list2op1op2s r#same_opsz.ForLoopIndexingPattern.match..same_opssG5zSZ'u- !S88sxx'  !r%c |jdk(S)Nr)res r#z.ForLoopIndexingPattern.match..sqvv?Wr%rr)ridrrar6r rangeadd) r!r" repeat_countrdnextrnext_select_idx indexing_opsis r#rVzForLoopIndexingPattern.matchs :: ' 88t|| # ""5)4 t9> /t5WX  "w&6!77 C %)+,q#d)S%67 A d1q3|3D/D&EF!    a,r!!   r!!r%rn) rorprqrrr rsr$r0r rVr~rs@r#rrs, 'W'' 1 ">"r%rcheZdZd dedeffd ZefdZdefdZ defdZ de efd Z xZ S) FP32MatMulPatternrrcPt|||d|_d|_d|_y)NzFP32 MatMul Patternz|You are currently using GPU that supports TF32. Please enable TF32 by setting 'torch.backends.cuda.matmul.allow_tf32 = True'zUhttps://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devicesrxr$rrrrs r#r$zFP32MatMulPattern.__init__+s2 /0)  [ kr%ctjjd}n.tdtjj D}|duxs#t |xs|jj S)NFc3>K|]}t|dddk\yw)NP)rN)rHarchs r#rIz)FP32MatMulPattern.skip..:sV43tABx=B.Vs) torchversionhipallcuda get_arch_listrxr*rr)r!has_tf32rys r#r*zFP32MatMulPattern.skip4s] ==   (HV5::;S;S;UVVH5 OEGLO 8O8O4OOr%r"c|jtjk7ryt|jt sJ|j dk(r|jjduryy)NFaten::mmT)tagrTorchOp isinstance extra_fieldsr rallow_tf32_cublasrUs r#rVzFP32MatMulPattern.match=sT 99 ** *%,,.BCCC :: #!!33u<r%c|jSrF)rrUs r#r.zFP32MatMulPattern.reportGsr%r2c,|Dcic]}t|d}}|D]}tj|ddtj}tj|ddtj}t j d||d}t j dd ||d }d tj jj_ |jd j} |jd j} | | z ||<|Scc}w) Nrrrdevicedtyper^torch.mm(matrixA, matrixB)matrixAmatrixBrz,torch.backends.cuda.matmul.allow_tf32 = True)rsetuprFr) rKrrandnfloat32r5rbackendsrmatmul allow_tf32rr) r!r2r"rLrrr fp32_timer tf32_timer fp32_time tf32_times r#r5zFP32MatMulPattern.benchmarkJsCIJ%\%0#5JJ& =Ekk%(6OGkk%(6OG"1$+@J#1D$+@J 5:ENN   & & 1"))"-22I"))"-22I'09'< e $ = ! #KsDrn)rorprqr rsr$rtr*r rVr.rr5r~rs@r#rr*sXkWkkPP> N !^ 4!r%rc:eZdZdZddedeffd ZdefdZxZ S)OptimizerSingleTensorPatterna{ This pattern identifies if we are using the single-tensor version of an optimizer. example: optimizer = torch.optim.SGD(model.parameters(), lr=0.1) By adding foreach=True to enable multi-tensor optimizer, we can gain speedup when the kernels are relatively small. Pattern: XXXXX: _single_tenser_ Algorithm: String match rrcbt|||d|_gd|_d|_d|_y)NzOptimizer Single Tensor Pattern)adamsgdadamwzDeteced optimizer running with single tensor implementation. Please enable multi tensor implementation by passing 'foreach=True' into optimizer.r)rxr$roptimizers_with_foreachrrrs r#r$z%OptimizerSingleTensorPattern.__init__ns9 /05 '?$ b r%r"cf|jD]"}|jjd|s"yy)N_single_tensor_TF)rrendswith)r!r" optimizers r#rVz"OptimizerSingleTensorPattern.matchxs855 Izz""_YK#@A r%rn rorprqrrr rsr$r rVr~rs@r#rr_s' W>r%rc:eZdZdZddedeffd ZdefdZxZ S)SynchronizedDataLoaderPatterna This pattern identifies if we are using num_workers=0 in DataLoader. example: torch.utils.data.DataLoader(dataset, batch_size=batch_size) Add num_workers=N to the arguments. N depends on system configuration. Pattern: dataloader.py(...): __iter__ dataloader.py(...): _get_iterator NOT dataloader.py(...): check_worker_number_rationality Algorithm: If we don't see check_worker_number_rationality call in the dataloader __iter__, It is not an asynchronous dataloader. rrcPt|||d|_d|_d|_y)NzSynchronized DataLoader PatternzDetected DataLoader running with synchronized implementation. Please enable asynchronous dataloading by setting num_workers > 0 when initializing DataLoader.zjhttps://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#enable-async-data-loading-and-augmentationrrs r#r$z&SynchronizedDataLoaderPattern.__init__s4 /05  n   : r%r"cBdtdtfd} |j||jdsy|jsy|jd}||jdsy|jsy|jd}||jd S#t$rYywxYw) Nr function_namec|jtjjddddxr|j |S)Nrutilsdataz dataloader.py) startswithospathjoinr)rrs r#is_dataloader_functionzCSynchronizedDataLoaderPattern.match..is_dataloader_functions:?? WgvG/-- . /r%F__iter__r _get_iteratorcheck_worker_number_rationality)r}rUnicodeDecodeErrorr_)r!r"rs r#rVz#SynchronizedDataLoaderPattern.matchs / /S /  JJ&ejj*=~~q!%ejj/B~~q!)%**6WXXX"  s B BBrnrrs@r#rrs)"  W    Y>Yr%rc:eZdZdZddedeffd ZdefdZxZ S)GradNotSetToNonePatterna This pattern identifies if we are not setting grad to None in zero_grad. example: optimizer.zero_grad() By setting set_to_none=True, we can gain speedup Pattern: XXXXX: _zero_grad NOT aten::zeros aten::zero_ aten::zero_ is called on each parameter in the model. We also want to make sure it is not called by aten::zeros. Algorithm: String match rrcPt|||d|_d|_d|_y)Nz,Gradient Set To Zero Instead of None PatternzfDetected gradient set to zero instead of None. Please add 'set_to_none=True' when calling zero_grad().zxhttps://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#disable-gradient-calculation-for-validation-or-inferencerrs r#r$z GradNotSetToNonePattern.__init__s5 /0B  F   H r%r"c|jjdsy|jsyt|jD]-}|jdk(s|jjdk7s-yy)Nz : zero_gradFrz aten::zerosT)rrr_rr[)r!r" sub_events r#rVzGradNotSetToNonePattern.matchsazz""=1~~%enn5 I-/$$))]:  r%rnrrs@r#rrs'$  W     > r%rcNeZdZdZddedeffd ZefdZde fdZ xZ S) &Conv2dBiasFollowedByBatchNorm2dPatternau This pattern identifies if we are enabling bias in Conv2d which is followed by BatchNorm2d. Bias doesn't do anything when followed by batchnorm. Pattern: nn.Module: Conv2d | nn.Module: BatchNorm2d ... aten::conv2d AND dtype of third argument is not null The third argument is the bias Algorithm: String match rrcPt|||d|_d|_d|_y)Nz5Enabling Bias in Conv2d Followed By BatchNorm PatternzcDetected bias enabled in Conv2d that is followed by BatchNorm2d. Please set 'bias=False' in Conv2d.zhttps://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#disable-bias-for-convolutions-directly-followed-by-a-batch-normrrs r#r$z/Conv2dBiasFollowedByBatchNorm2dPattern.__init__s2 /0K A O r%cL|jjduxs t| Sr')rrrxr*)r!rys r#r*z+Conv2dBiasFollowedByBatchNorm2dPattern.skips!yy&&%/?57<?r%r"c|jdk7rytt|dkst|dy|j|d}|sy|j |}|sy|jj dS)Nz aten::conv2dFrrMc8|jjdS)Nznn.Module: Conv2d)rrrs r#rz>Conv2dBiasFollowedByBatchNorm2dPattern.match.. sQVV../BCr%znn.Module: BatchNorm2d)rr6rrmrfrrUs r#rVz,Conv2dBiasFollowedByBatchNorm2dPattern.matchs} :: ' |E" #a '<+>q+A+I  C  U#zz$$%=>>r%rn) rorprqrrr rsr$rtr*r rVr~rs@r#r r s=  W  @@?>?r%r cXeZdZd dedeffd ZedZdefdZ de efdZ xZ S) MatMulDimInFP16PatternrrcPt|||d|_d|_d|_y)Nz3Matrix Multiplication Dimension Not Aligned PatternzUDetected matmul with dimension not aligned. Please use matmul with aligned dimension.z[https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#use-mixed-precision-and-amprrs r#r$zMatMulDimInFP16Pattern.__init__s) /0I rpr%cb|jj xs|jj SrFrr)s r#r*zMatMulDimInFP16Pattern.skiprr%r"cd}|jdvryt|syt|d}|tjtjfvr|t |dsyy)Nc,tfd|DS)Nc3BK|]}|ddD] }|zdk(yw)Nrr()rHrdimmultiples r#rIzCMatMulDimInFP16Pattern.match..mutiple_of.. s/Uu%PRPS*U3sX~*U*Us)r)shapesrs `r# mutiple_ofz0MatMulDimInFP16Pattern.match..mutiple_ofsUUU Ur%)rz aten::bmmz aten::addmmFrT)rrrbfloat16halfrK)r!r"r arg_dtypes r#rVzMatMulDimInFP16Pattern.matchsb V ::E EE" '* 4 4Z  > r%r2cd}|Dcic]}t|d}}|D]'}tj|ddtj}tj|ddtj}t j d||d }tj||dd dtj}tj||dd dtj}t j d||d } |j d j} | j d j} | | z ||<*|Scc}w) Nc^|Dcgc]}|tj||z zc}Scc}wrF)mathceil)rrrs r#closest_multiplez:MatMulDimInFP16Pattern.benchmark..closest_multiple.s)HNOuHtyy)9::O OOs"*rrrrr^rrrrr)rKrrfloat16r5rrr) r!r2r"r"rLrrrnot_aligned_dim_timeraligned_dim_timernot_aligned_dim_timealigned_dim_times r#r5z MatMulDimInFP16Pattern.benchmark-s@ PDJJ%\%0#5JJ& OEkk%(6OGkk%(6OG$-OO1$+@% !kk q1-fEMMGkk q1-fEMMG!*1$+@! $9#?#?#C#H#H 077;@@ '7:N'N e $' O(! +KsE rn) rorprqr rsr$rtr*r rVrr5r~rs@r#rrsLqWqq GG > !^ 4!r%rr"c|r|jtjk(s|jtjk(rt |j t tfsJ|j jjjdtjzsC|j jjd|j jjS|j}|ry)Nr:zNo source code location found)rrPyCallPyCCallrrrr caller file_namerrsep line_numberr[r"s r#r,r,Is  99 )) )UYY*:L:L-L""%8:N$O %%,,66AA'BFFBRS,,33==>a@R@R@Y@Y@e@e?fgg   +r%ct|jtsJtd|jjDS)Nc3HK|]}tt|ddyw)sizesr(N)tuplegetattrrHrs r#rIzinput_shapes..WsSAwq'2./Ss "rrr r4inputsr0s r#rKrKUs6 e((*> ?? ? S9K9K9R9RS SSr%ct|jtsJtd|jjDS)Nc36K|]}t|ddyw)rN)r5r6s r#rIzinput_dtypes..\sNqGT*Nsr7r0s r#rrZs6 e((*> ?? ? NE4F4F4M4MN NNr%r print_enablejson_report_dirc i}t||t||t||t||t ||t ||t ||g}t}g}dddg}|jd|D]} | j} | s|j| j| | D]} | j| } | |vs|j| |j| t| jd\} }|j| gjt!|| j"| j$| j&d|t(j*j-|d}t(j*j/|r# E%,,U3J)##J/ Z((>* +&' /1#'99Q<  '' 4.  /"C ( 0A IIk1Q / 0 #IL6("26(;< dii %& / / 0 0s&)I#III#)FTN))rFr rr{typingrrrrrtorch.utils.benchmarkrr5torch._C._profilerrrr r r torch.profilerr torch.profiler._utilsr rrrrvrrrrrrr rr,rKrrsr}rWr(r%r#r]s ,, ))#RRooj<'<T!7T!n<"W<"~2!2!j7@6YG6Yt,g,^(?W(?V3!W3!l + 8 +TT OO#%) 8'8'8'c] 8'r%