DŽg{UddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z ddlmZddlmZmZmZmZmZmZmZmZddlmZddlZddlmZddlm Z ejBe"Z#ejBdZ$ejJZ&d Z'd Z(d Z)d Z*e Gd dZ+e GddZ,e+Z-e,a.ej^ej^ej^ej^ddddej^d Z0ddddddddddddddddddddddddddddddddddddddddddd*dee1dee1dee1dee1dee1dee1dee1dee1dee1dee1dee1d ee1d!e2d"e2d#e2d$e2d%e2d&e2d'e2d(e2d)e2d*e2d+e2d,e2d-e2d.e2d/e2d0e2d1e2d2e2d3e2d4e2d5e2d6e2d7ee1d8eee3ee1e2ffd9e2d:e2d;e2de2fTd?Z4d@Z5dAZ6 dkdBZ7dCZ8dDZ9dEZ:dFdGjwejxe9ejxe:gzdHzZ=dIZ>dJZ?dKZ@dldLZAdldMZBejdNZDdOZEdPZFdQZGdRZHGdSdTejZJdUZKeKZLdVZMeZNdWZOdXZPdYZQdZZRd[ZSd\ZTdmd]ZUGd^d_ejZWejdd`ZXGdadbZYeZfdcdddddee3dfegeee3efee3e1fffdgegeee3e[ffdhe2die2f djZ\ddl]ZddlZddl^m_Z`y)nN) dataclassfield) __import__)AnyCallableDictListOptionalSetTupleUnion)WeakSet)log_trace_structured_event)CapturedTracebackz torch.__trace TORCH_LOGSTORCH_LOGS_OUTTORCH_LOGS_FORMAT TORCH_TRACEceZdZUeeZeeeefe d<ee Z e ee d<ee Z e ee d<ee Ze ee d<ee Ze ee d<eeZeeefe d<ee Ze ee d<eeZeeej(fe d <d Zd Zd eeeeffd ZdZdZdZde efdZdZdZdZy) LogRegistrydefault_factorylog_alias_to_log_qnamesartifact_log_qnameschild_log_qnamesartifact_namesvisible_artifactsartifact_descriptionsoff_by_default_artifact_namesartifact_log_formattersc||jvSNrselfnames `/home/mcse/projects/flask_80/flask-venv/lib/python3.12/site-packages/torch/_logging/_internal.py is_artifactzLogRegistry.is_artifactSt****c||jvSr")r)r%aliass r'is_logzLogRegistry.is_logVs4444r* log_qnamescHt|tr|g}||j|<yr") isinstancestrr)r%r,r.s r' register_logzLogRegistry.register_logZs# j# &$J.8$$U+r*c|jj||r|jj|||j|<|r|jj||#t j ||j|<yyr")raddrrrlogging Formatterr )r%r& descriptionvisibleoff_by_default log_formats r'register_artifact_namez"LogRegistry.register_artifact_name`s~ %   " " & &t ,+6""4(   . . 2 24 8  !181B1B:1ND ( ( . "r*c:|jj|yr")rr4)r%artifact_log_qnames r'register_artifact_logz!LogRegistry.register_artifact_logss   $$%78r*c:|jj|yr")rr4)r% log_qnames r'register_child_logzLogRegistry.register_child_logvs !!),r*returncp|jjDchc] }|D]}| c}}Scc}}wr")rvalues)r%qnamesqnames r'get_log_qnameszLogRegistry.get_log_qnameszsH66==?       s2c,t|jSr")setrr%s r'get_artifact_log_qnamesz#LogRegistry.get_artifact_log_qnamess4++,,r*c,t|jSr")rIrrJs r'get_child_log_qnamesz LogRegistry.get_child_log_qnamess4(())r*c||jvSr")r)r%artifact_qnames r'is_off_by_defaultzLogRegistry.is_off_by_defaults!C!CCCr*N) __name__ __module__ __qualname__rdictrrr1r __annotations__rIrr rrrrrr r5r6r(r-r r2r;r>rArGrKrMrPr*r'rr-s 5:$4OT#tCy.1O %*#$>S> "'s!;c#h; %S9NCH9#("<s3x<-2$,G4S>G /4C.H!3s8H=BRV`_ An artifact is a particular type of log message. Each artifact is assigned to a parent component. A component can emit many different kinds of artifacts. In general, an artifact is emitted if either its corresponding setting in the argument list below is turned on or if its parent component is set to a log level less than or equal to the log level of the artifact. Keyword args: all (:class:`Optional[int]`): The default log level for all components. Default: ``logging.WARN`` dynamo (:class:`Optional[int]`): The log level for the TorchDynamo component. Default: ``logging.WARN`` aot (:class:`Optional[int]`): The log level for the AOTAutograd component. Default: ``logging.WARN`` autograd (:class:`Optional[int]`): The log level for autograd. Default: ``logging.WARN`` inductor (:class:`Optional[int]`): The log level for the TorchInductor component. Default: ``logging.WARN`` dynamic (:class:`Optional[int]`): The log level for dynamic shapes. Default: ``logging.WARN`` distributed (:class:`Optional[int]`): Whether to log c10d communication operations and other debug info from PyTorch Distributed components. Default: ``logging.WARN`` c10d (:class:`Optional[int]`): Whether to log c10d communication operations related debug info in PyTorch Distributed components. Default: ``logging.WARN`` ddp (:class:`Optional[int]`): Whether to log debug info related to ``DistributedDataParallel``(DDP) from PyTorch Distributed components. Default: ``logging.WARN`` fsdp (:class:`Optional[int]`): Whether to log debug info related to ``FullyShardedDataParallel``(FSDP) in PyTorch Distributed components. Default: ``logging.WARN`` dtensor (:class:`Optional[int]`): Whether to log debug info related to ``DTensor``(DTensor) in PyTorch Distributed components. Default: ``logging.WARN`` onnx (:class:`Optional[int]`): The log level for the ONNX exporter component. Default: ``logging.WARN`` bytecode (:class:`bool`): Whether to emit the original and generated bytecode from TorchDynamo. Default: ``False`` aot_graphs (:class:`bool`): Whether to emit the graphs generated by AOTAutograd. Default: ``False`` aot_joint_graph (:class:`bool`): Whether to emit the joint forward-backward graph generated by AOTAutograd. Default: ``False`` ddp_graphs (:class:`bool`): Whether to emit graphs generated by DDPOptimizer. Default: ``False`` graph (:class:`bool`): Whether to emit the graph captured by TorchDynamo in tabular format. Default: ``False`` graph_code (:class:`bool`): Whether to emit the python source of the graph captured by TorchDynamo. Default: ``False`` graph_breaks (:class:`bool`): Whether to emit the graph breaks encountered by TorchDynamo. Default: ``False`` graph_sizes (:class:`bool`): Whether to emit tensor sizes of the graph captured by TorchDynamo. Default: ``False`` guards (:class:`bool`): Whether to emit the guards generated by TorchDynamo for each compiled function. Default: ``False`` recompiles (:class:`bool`): Whether to emit a guard failure reason and message every time TorchDynamo recompiles a function. Default: ``False`` recompiles_verbose (:class:`bool`): Whether to emit all guard failure reasons when TorchDynamo recompiles a function, even those that are not actually run. Default: ``False`` trace_source (:class:`bool`): Whether to emit when TorchDynamo begins tracing a new line. Default: ``False`` trace_call (:class:`bool`): Whether to emit detailed line location when TorchDynamo creates an FX node corresponding to function call. Python 3.11+ only. Default: ``False`` trace_bytecode (:class:`bool`): Whether to emit bytecode instructions and traced stack state as TorchDynamo traces bytecode. Default: ``False`` output_code (:class:`bool`): Whether to emit the TorchInductor output code on a per-graph basis. Default: ``False`` kernel_code (:class:`bool`): Whether to emit the TorchInductor output code on a per-kernel bases. Default: ``False`` schedule (:class:`bool`): Whether to emit the TorchInductor schedule. Default: ``False`` perf_hints (:class:`bool`): Whether to emit the TorchInductor perf hints. Default: ``False`` post_grad_graphs (:class:`bool`): Whether to emit the graphs generated by after post grad passes. Default: ``False`` onnx_diagnostics (:class:`bool`): Whether to emit the ONNX exporter diagnostics in logging. Default: ``False`` fusion (:class:`bool`): Whether to emit detailed Inductor fusion decisions. Default: ``False`` overlap (:class:`bool`): Whether to emit detailed Inductor compute/comm overlap decisions. Default: ``False`` sym_node (:class:`bool`): Whether to emit debug info for various SymNode opterations. Default: ``False`` export (:class:`Optional[int]`): The log level for export. Default: ``logging.WARN`` benchmarking (:class:`bool`): Whether to emit detailed Inductor benchmarking information. Default: ``False`` modules (dict): This argument provides an alternate way to specify the above log component and artifact settings, in the format of a keyword args dictionary given as a single argument. There are two cases where this is useful (1) if a new log component or artifact has been registered but a keyword argument for it has not been added to this function and (2) if the log level for an unregistered module needs to be set. This can be done by providing the fully-qualified module name as the key, with the log level as the value. Default: ``None`` cudagraph_static_inputs (:class:`bool`): Whether to emit debug info for cudagraph static input detection. Default: ``False`` Example:: >>> # xdoctest: +SKIP >>> import logging # The following changes the "dynamo" component to emit DEBUG-level # logs, and to emit "graph_code" artifacts. >>> torch._logging.set_logs(dynamo=logging.DEBUG, graph_code=True) # The following enables the logs for a different module >>> torch._logging.set_logs(modules={"unregistered.module.name": logging.DEBUG}) zQUsing TORCH_LOGS environment variable for log settings, ignoring call to set_logsNctj|jjD])\}}| tj |r:t |t std|d||sCtj|Ytj|s|tjvr|tjvrVtd|d|ddjtjjDcgc] }t!|c}tj#tj$j'|||td|t)ycc}w)Nz!Expected bool to enable artifact z , received zUnrecognized log level for log z: z, valid level values are: ,z6Unrecognized log or artifact name passed to set_logs: ) itertoolschainrc log_registryr(r0bool ValueError log_stater\r-rr5 _levelToNamejoinkeysr1rarget _init_logs)kwargsr,valkrs r' _set_logszset_logs.._set_logssG#//&,,.'--/J JE3{''.!#t,$;E7+cUS--e4$$U+u 8U8U/Ug222$9%3%H #':N:N:S:S:U)VQ#a&)V WXZ $$ 88<?*@*A*BC*DE*FG*HI*JK*L,M*N#configure_artifact_log) module_qnamer[rFrs r'getArtifactLoggerr&sL777d=123--1--@,AAc e   S0 0E   E "C%C&&u-3 Jr*+-(|z?)ctj|jrd|_tj |jr'|j tjd|_yyNFT) rrPr[ propagaterr^setLevelr5DEBUG)rs r'rr=sQ%%c&7&78 $$S%6%67 W]]# 8r*c,tjdS)Nz$((\+|-)?[\w\.]+,\s*)*(\+|-)?[\w\.]+?)recompilerVr*r'_gen_settings_regexrKs ::= >>r*cBtjt|duSr")r fullmatchr)settingss r'_validate_settingsrOs <<+-x 8 DDr*c rd d}|rtj}ntj}|rd}nd}dgttjj zt|Dcgc] }||dtj |"c}z}ddj|z}d}d |d |d |d }|Scc}w) NcJt||ksJ|d|t|z zzS)N )len)slengths r'pad_tozhelp_message..pad_toTs,1v3&3q6/***r*zAll registered namesz?Visible registered names (use TORCH_LOGS='+help' for full list)rp z z a Examples: TORCH_LOGS="+dynamo,aot" will set the log level of TorchDynamo to logging.DEBUG and AOT to logging.INFO TORCH_LOGS="-dynamo,+inductor" will set the log level of TorchDynamo to logging.ERROR and TorchInductor to logging.DEBUG TORCH_LOGS="aot_graphs" will enable the aot_graphs artifact TORCH_LOGS="+dynamo,schedule" will enable set the log level of TorchDynamo to logging.DEBUG and enable the schedule artifact TORCH_LOGS="+some.random.module,schedule" will set the log level of some.random.module to logging.DEBUG and enable the schedule artifact TORCH_LOGS_FORMAT="%(levelname)s: %(message)s" or any provided format string will set the output format Valid keys are "levelname", "message", "pathname", "levelno", "lineno", "filename" and "name". TORCH_LOGS_OUT=/tmp/output.txt will output the logs to /tmp/output.txt as well. This is useful when the output is long. z TORCH_LOGS Info z  ))rrrsortedrrrr) verboserprinted_artifactsheadingr&lines setting_infoexamplesmsgs r' help_messagerSs+(77(::(S  55::< = > . $<.<#E#Ed#K"LM    &++e,,LH0       C JM s"%B4 cdjdgttjj zttj z}d|d|d}|S)Nz, rpz Invalid log settings: z, must be a comma separated list of fully qualified module names, registered log names or registered artifact names. For more info on various settings, try TORCH_LOGS="help" Valid settings: r)rlistrrrr)rrvalid_settingsrs r'_invalid_settings_err_msgrsnYY  |3388: ; < |** + ,N j!  C Jr*c|dk(riS|dk(rttd|dk(rttdt|stt|t j dd|}|j d}d }t}|D]}||\}}|d k(rd }tj|r*|Jtj|}|j||Ttj|r|j|{t|rJt|stj!||ntj#||j||tt||S) NhelpF)rz+helpTz\s+rc|jtd}|jtd}|dtk(rtj}||fS|dtk(rtj }||fStj }||fS)Nrr)replaceINCR_VERBOSITY_CHARDECR_VERBOSITY_CHARr5rERRORINFO)r& clean_namelevels r'get_name_level_pairz0_parse_log_settings..get_name_level_pairs\\"5r: ''(;R@ 7) )MME 5  !W+ +MME5  LLE5  r*rpr)rrrrrsubsplitrXrr-rrar(r\_is_valid_module_has_registered_parentr2rA)r log_namesrrr&rr.s r'_parse_log_settingsrs]2~ 6e455 W d344 h '28<==vvfb(+Hs#I ! IB)$/ e 5=D   t $$ $$%==dCJ  U 3  % %d +  % %d + d #)$/))$5//5  u -6x@A A'B* r*c: t|y#t$rYywxYwNTF)r ImportErrorrs r'rrs$5 s  chtjjtd}| t |ayyr")rrrrrr) log_settings r'_update_log_state_from_envrs+**..d3K' 4 r*ctj|}tj}|jr(|j |vry|j}|jr(yr)r5rrrGparentr&)r@cur_logregistered_log_qnamess r'rrsO *G(779 .. <<0 0.. .. r*ctj|j}tjD] } |j |}t |cSt |S#t$rY9wxYw)z Given an absolute filepath corresponding to a Python module which was loaded via normal import mechanisms using sys.path, convert it into a relative path relative to one of the Python search paths. )pathlibPathresolvesyspath relative_tor1r)abs_pathrrel_paths r'make_module_path_relativerso||H%--/H! !++D1Hx= ! x=    sA"" A.-A.c0eZdZdddeffdZdZxZS)TorchLogsFormatterFtracer c0t|||_yr")super__init__ _is_trace)r%r  __class__s r'r zTorchLogsFormatter.__init__s r*c ttj|jdd}|3tj j |d}||j|S|j|_ |j|d|_ |j}|jr,|js |j|j|_ |jr|dddk7r|dz}||jz}|jr+|dddk7r|dz}||j!|jz}d|_|j$sEt'j(r1t'j*rdt'j,d|_d|_|j$s6t0j2j4j7x} d|d |_d d d d dd}|j |j8|j8}d|_| d|d |_t=|j>}|j"||jdtA|jBdzdd|jDd|d|jFd |j.|j: |j$rn|dk(sJ dtIjJ|jL} |jT4| djWd|jTjYdDz } | S|jYd} djW fd| DS#tN$r"tPjSd|jLwxYw)Nr[z %m%d %H:%M:%Srrz[rankz]:z []VIWEC)rrWARNINGrCRITICALz [__.i06dr:zfailing metadata: %rc3&K|] }d| yw)z NrV).0ls r' z,TorchLogsFormatter.format..OsLAtA3ZLsc3,K|] }d| yw)rNrV)rrprefixs r'rz,TorchLogsFormatter.format..Ss<xq_QUV  $!-!E!E!I!It" "-)0088**,A NN ????"&"6"6v"G ??v~HFOO#A   v~HD$$V%6%677A~~$"3"3"5$:M:M:O"' '8 ;F "]]99JJLL "(1-FN  (++F,<,N>NO "  $&*=/$;F !,V__=  !*fnn-=Qs6<ZZ[\b\j\j[kklj}}oQv~~.v/D/D.E G  >>7N7 ha 6?? ;<=~~)RWWL1E1Ed1KLLLHGGDME99tjD]+\}}tj|}|j|-t jD]C}tj|}ttj|5tfd|Et j!D]"}tj|}t#|$tjj t$d}t'|}t(jtj*t-|}|j/t1dt(j3|y)Nrc.tjSr")r5 FileHandler) log_file_namesr'z_init_logs..s++M:r*Tr)rgrrrrLOG_OUT_ENV_VARrrGr5rrrerrdrY StreamHandlerrKr TRACE_ENV_VARLazyTraceHandlerrfrrSrTrrV) rqoutr@rrr=trace_dir_namer]trace_log_handlers ` r'rrsM **..$ /C  "002%     * W^^$ %&99; 5 * U "002   *  ! !   $ :  +BBD$ 23s#$ ZZ^^M48N ~.Gw}}%&w/""#5D#AB *+r*c:eZdZdZdeefdZdZfdZxZ S)rvzRLike FileHandler, but the file is allocated lazily only upon the first log messageroot_dirct||_tjj|d|_t |_yr")r{r5Handlerr streamopen _builtin_open)r%r{s r'r zLazyTraceHandler.__init__s,!   & !r*c|j |jrA |j|j}d|_t|dr|j  t j j | |jy#|j}d|_t|dr|j wwxYw#t j j |wxYw#|jwxYw)Nclose)acquirer~flushhasattrrr5rtrelease)r%r~s r'rzLazyTraceHandler.closes   2;;+ !%&* "673"LLN %%++D1 LLN"&&* "673"LLN4%%++D1 LLNs3 CB0C C*2CC!C''C**C<c|jd}|jd}|j}ddlm}t |dr+t jdtjdntjjdstjdnxt jj|stjd |nBt j|t j stjd |n||_|jt j"|jd d }t%j&r,t%j(rdt%j*d}t-j.ddd||jd|_tjd|jj0nt2j5|y|jrt6|q|yy)NFz/logsr git_versionMAST_HPC_JOB_NAMEz>LazyTraceHandler: disabled because not fbcode or conda on mastzpytorch/trace:enablezYLazyTraceHandler: disabled because justknobs_check('pytorch/trace:enable') returned Falsez4LazyTraceHandler: disabled because %s does not existz6LazyTraceHandler: disabled because %s is not writeableT)exist_okrrank__zw+z.logdedicated_log_torch_trace_)modesuffixr!dirdeletezLazyTraceHandler: logging to %s)r~r{r torch.versionversionrrgetenvrinfor_utils_internaljustknobs_checkrexistsaccessW_OKmakedirsr.r/r0r1tempfileNamedTemporaryFiler&rfrar emit)r%rBok TRACE_LOG_DIR open_func torch_version ranksuffixrs r'rzLazyTraceHandler.emits ;; B}}$ '  .. 5M=9 "56>HHX..>>?UVHHs 6HHN%="'':HHP% %2DM}}( DMMD9 $$&4+>+>+@#((9!;J&99!7 |D   :DKK)ris r'rz#trace_structured..{s0P1A0Ps!z ]r)indentzutf-8 has_payloadr)r?rA)extra stacklevel)#callabletyperfr[r.r/r0r1rr3r4r5 compile_idrrr_logging structuredfrom_tracebackrextractsummaryr0r1rrr=r>hashlibmd5updateencode hexdigestdebugr) r&rrrrrBrDrAhs r'trace_structuredrFs" L LL L  F 1${2C1DEF   D 0j1A0BCD  $&"}t   "t':':'1   #40Kr*)FFN)Fr")a functoolsrrr=r5ros.pathrrrr dataclassesrr importlibrtypingrrrr r r r r weakrefrtorch._logging.structuredrtorch._utils_internalrtorch.utils._tracebackrrrQrrfrDEFAULT_LOG_LEVELrrsrNrurrXrrrDEFAULT_LOGGINGr9rr1rrr2rrrrrescapeVERBOSITY_REGEXrrrrr lru_cacherrrrrr6rrQrUrYr[rSr_rcrgrjrmrrtrvrrrTobjectr torch._guardstorch.distributedrrr.rVr*r'rs   ( III <4g! G  o . OO "(  ZDZD ZDz #$#$ #$L} J ll << LL||  "!"!%!!$ "" 59#&+$)W~ #~ SM~ # ~ sm ~ c] ~sm~#~ 3-~ #~ 3-~c]~ 3-~~~ !~"#~$ %~&'~()~*+~, -~./~01~23~45~67~89~:;~<=~>?~@A~BC~D E~FG~H SMI~Jd3c4i 001 2K~LM~NO~PQ~R $S~T"U~VW~B Q6PT"  hh  -.  :M0NOPQ   ?E8v   22j5 *N=**N=b&'(" 9   # ( ;,|T!w,,T!nT((44IM ?1 >J" ?1 ?1"eDcNE#s(O$CDDE ?1 XeCK&899: ?1?1?1D r*