wgddlmZddlmZmZmZmZmZddlm Z ddl m Z ddl m Z edZGd d eZdgd Zdgd Zdhd ZdidZdjdZ dk dldZdmdZdmdZdmdZdmdZdmdZdndZdmdZdodZdodZdpdZ dqdZ!dmdZ"dmdZ#dmdZ$dmdZ%dmd Z&drd!Z'dmd"Z(dmd#Z)dmd$Z*dsd%Z+dtd&Z,dud'Z-dvd(Z.dmd)Z/dmd*Z0dmd+Z1dmd,Z2dmd-Z3dmd.Z4dwd/Z5dxd0Z6dyd1Z7dzd2Z8d{d3Z9d|d4Z:d}d5Z;d~d6Zdd9Z?dd:Z@dd;ZA d dd=ZBd>ZCd?ZDd@ZEdAZFdBZGdCZHdDZIdEZJdFZK ddGZL ddHZMddIZNdJZOdKZP ddLZQddMZR ddNZSddOZTddPZUddQZVddRZWddSZXddTZY ddUZZdVZ[ ddWZ\ddXZ]dYZ^ddZZ_ dd[Z`dd\Zadd]Zbdd^Zcdd_Zddd`ZeddaZfddbZgdcZhdddZiddeZjddfZky<)) annotations)ListOptionalSequenceTupleTypeVar)ir)core)mathTceZdZfdZxZS)IncompatibleTypeErrorImplc||_||_d|jjzdz|jjz|_tt ||jy)Nzinvalid operands of type  and )type_atype_b__repr__messagesuperr__init__)selfrr __class__s ]/home/mcse/projects/flask/flask-venv/lib/python3.12/site-packages/triton/language/semantic.pyrz"IncompatibleTypeErrorImpl.__init__sX  2T[[5I5I5KKgUX\XcXcXlXlXnn  '7 E)__name__ __module__ __qualname__r __classcell__)rs@rrr sFFrrc|dvrtd|tj|j|tjS)Nrr r z+program_id axis must be 0, 1, or 2 but got ) ValueErrortltensorcreate_get_program_idint32axisbuilders r program_idr+s= 9FtfMNN 99W2248"(( CCrc|dvrtd|tj|j|tjS)Nr"z-num_programs axis must be 0, 1, or 2 but got )r#r$r%create_get_num_programsr'r(s r num_programsr. s= 9HOPP 99W44T:BHH EErc`|j}|j}|j}|j}||k(r ||kDr|S|S|tjjj k(r ||k\r|S|S|tjjj k(r ||k\r|S|St d|d|)Nzunexpected signedness r) int_bitwidthint_signednessr$dtype SIGNEDNESSUNSIGNED TypeError)a_tyb_tya_rankb_ranka_snb_sns rinteger_promote_implr<+s   F   F   D   D t|t0D0 $$-- -'t1T1 $$-- -'t1T1 ,TF%v> ??rc@|js|jrtjS|js|jrtjS|j s|j r"|rtjStj S|js|jrR|rtjS|jr |jrtjStjS|jr|jstd|d||rL|j|jk7r3td|jzdz|jzdzt||S)Nunexpected type rzCannot use /, #, or % with x because they have different signedness;this is unlikely to result in a useful answer. Cast them to the same signedness.)is_fp64r$float64is_fp32float32is_fp16float16is_bf16bfloat16is_intr5r1rr<)r6r7 div_or_mods rcomputation_type_implrJ;s7 ||~zz ||~zz ||~ :: ::  ||~ ::  <<>dlln;; zz ;;= *4&dV<==d))T-@-@@5 G'QTXTaTaTcckkl l d ++rc|jrL|s t|||jr||k7r t|||jr t||yyN)is_ptrr is_floating)rr allow_ptr_as rcheck_ptr_type_implrPcs[ }}+FF; ; ==?& 0+FF; ;    +FF; ; rcHt|||\}}|jj}|jj}t|||t||||rG|j s7|j s't |||} t || |}t || |}||fSrL)broadcast_impl_valuetypescalarrPrMrJcast) lhsrhsr* allow_lhs_ptr allow_rhs_ptrarithmetic_checkrI lhs_sca_ty rhs_sca_ty ret_sca_tys rbinary_op_type_checking_implr^os$Cg6HCJJ J > J > 1 1 3J) r^rSrTrMr5r$r% create_addptrhandlerN create_faddrH create_addinputotherr*input_scalar_tyother_scalar_tys raddrisO/ugtTRLE5jj''Ojj''OO$:$:$<677(>(>(@eu**++**++yy..u||U\\JEJJWW  $ $ &yy,,U\\5<<H%**UU    !yy++ELL%,,GTT &&78 99rc zt|||dd\}}|jj}|jrNt j |j |jt||j|jS|jrDt j |j|j|j|jS|jrDt j |j|j|j|jStd|)NTFr>)r^rSrTrMr$r%r`raminusrN create_fsubrH create_subr5rerfr* scalar_tys rsubrps/ugtUSLE5 !!Iyy..u||U5'=R=Y=YZ\a\f\fggyy,,U\\5<<H%**UU    yy++ELL%,,GTT &yk2 33rct|||\}}|jj}|jrDt j |j |j|j|jS|jrDt j |j|j|j|jStd|Nr>) r^rSrTrNr$r% create_fmulrarH create_mulr5rns rmulrus/ugFLE5 !!Iyy,,U\\5<<H%**UU    yy++ELL%,,GTT &yk2 33rc 6t|||dddd\}}|jj}|jj}|jr|j rt |||}n|j r|jrt |||}n|j rG|j r7t |t j|}t |t j|}nc|jrE|jr5|j|jkDrt |||}nt |||}ntd|t j|j|j|j|jSNFTr>) r^rSrTrNrHrUr$rCfp_mantissa_widthr5r% create_fdivrards rtruedivrzsH/ugueUY[_`LE5jj''Ojj''O""$)?)?)AUOW5    !o&A&A&CUOW5    !o&<&<&>UBJJ0UBJJ0  $ $ &?+F+F+H  , ,/P/P P9E9E*?*;<== 99W((u||Dejj QQrc Zt|||dddd\}}|jj}|jj}|jr|jrt ||}t |||}t |||}|j rDtj|j|j|j|jStj|j|j|j|jStd|rw) r^rSrTrHr<rU is_int_signedr$r% create_sdivra create_udivr5)rerfr*rgrhret_tys rfloordivrs/ugueUY[_`LE5jj''Ojj''OO$:$:$<%oGUFG,UFG,    !99W00u||LejjY Y99W00u||LejjY Y &&78 99rc d|jj}|jj}|jr|js tdt |||dddd\}}|j |j |j }tj||jS)Nz4both operands of fdiv must have floating scalar typeFT) rSrTrNr5r^ryrar$r%)rerf ieee_roundingr*rgrhrets rfdivrsjj''Ojj''O  & & (0K0K0MNOO/ugueUZ\`aLE5   ellELL 9C 99S%** %%rc t|||dddd\}}|jj}|jj}|jr;t |t t jt||d|||||}|S|jr|j|jk7r3td|jzdz|jzdz|jrDtj|j!|j"|j"|jStj|j%|j"|j"|jStd|)NFT_builderz Cannot mod z by r?r>)r^rSrTrNrprur floorrrHr1r5rr|r$r% create_sremra create_urem)rerfr*rorhrs rmodrsR/ugueUY[_`LE5 !!Ijj''O%TZZUE5'(JU\]_dfmnpwx      # #'E'E EMI,>,>,@@6IOLdLdLffjoop p  " " $99W00u||LejjY Y99W00u||LejjY Y &yk2 33rcft|||\}}|j}|jr|tjj k(rDtj |j|j|j|jS|tjjk(rDtj |j|j|j|jStd||jrDtj |j|j|j|jS|jrDtj |j!|j|j|jSt#d|NzUnexpected propagate_nan Unexpected dtype )r^r2rNr$ PropagateNanALLr%create_minimumfrarSNONEcreate_minnumfr#r| create_minsiis_int_unsigned create_minuir5xy propagate_nanr*r2s rminimumr3 '1g 6DAq GGE  BOO// /99W44QXXqxxH!&&Q Q boo22 299W33AHHahhGP P8HI I    yy--ahhA166JJ    yy--ahhA166JJ+E7344rcft|||\}}|j}|jr|tjj k(rDtj |j|j|j|jS|tjjk(rDtj |j|j|j|jStd||jrDtj |j|j|j|jS|jrDtj |j!|j|j|jSt#d|r)r^r2rNr$rrr%create_maximumfrarSrcreate_maxnumfr#r| create_maxsir create_maxuir5rs rmaximumrrrcXt|||\}}t|||\}}t|||\}}|j}|jrPtj|j |j |j |j ||jStd|d)Nrz(. Only floating point clamp is supported) r^r2rNr$r% create_clampfrarSr5)rminmaxrr*r2s rclampr%s+Cg>HC )!S' :FAs )!S' :FAs GGE yy..qxxSZZQ^_abagaghh+E72Z[\\rc@t|||ddd\}}|jj}|jj}|jr|js t ||t ||}||k7r t |||}||k7r t |||}||fS)NF)r^rSrTrHrr<rU)rerfr* input_sca_ty other_sca_tyr]s rbitwise_op_type_checking_implr6s/ugueUZ[LE5::$$L::$$L    (;(;(=' lCC%lLAJ\!UJ0\!UJ0 %<rct|||\}}tj|j|j|j|j SrL)rr$r% create_andrarSrerfr*s rand_rE>0wGLE5 99W'' ellCUZZ PPrct|||\}}tj|j|j|j|j SrL)rr$r% create_orrarSrs ror_rJs>0wGLE5 99W&&u||U\\BEJJ OOrct|||\}}tj|j|j|j|j SrL)rr$r% create_xorrarSrs rxor_rOrrc|jjs t|tjd|}|jjs t|tjd|}t |||SNint1)rSis_int1bitcastr$r2rrs r logical_andrTs_ ::   rxx/9 ::   rxx/9 ug &&rc|jjs t|tjd|}|jjs t|tjd|}t |||Sr)rSrrr$r2rrs r logical_orr\s_ ::   rxx/9 ::   rxx/9 ueW %%rc|jjs t|tjd|}t ||Sr)rSrrr$r2invert)rer*s rnot_rds6 ::   rxx/9 % !!rct|||\}}tj|j|j|j|j SrL)rr$r% create_lshrrarSrs rlshrrj>0wGLE5 99W((u||Dejj QQrct|||\}}tj|j|j|j|j SrL)rr$r% create_ashrrarSrs rashrrorrct|||\}}tj|j|j|j|j SrL)rr$r% create_shlrarSrs rshlrtrrc|SrL)res rplusr~s Lrc|jj}|jrtd|j zdzt j |j|j||}t|||S)Nz$wrong type argument to unary minus ()) rSrTrMr#rr$r%get_null_valueto_irrp)rer*r_0s rrkrksr::$$L?,BWBWBYY\__`` 7)),*<*?? GE ]]288U +F 99W..uc:F CCrct|tjr.|jjdk(sJdt |||}nj| t d|dk(r!|j|j|}n!t|d|j}||}tj||}t|||S)Nr zonly accepts size-1 tensorz2dtype must be specified when value is not a tensorrget_) rr$r%numelvaluerUr#rrgetattrnamesplat)rr r2r* get_value_fns rfullrs%#{{  A%C'CC%UE7+ =QR R A:**5;;w+?@E"7d5::,,?@L 'E %' w ''rc|jjrJdt|dk(r|Stj|j |}tj |j|j||S)NzCannot splat a block tensorr) rSrlenr$rr2r% create_splatra)r rr*rs rr r sdzz""$C&CC $ 5zQ ]]5;; .F 99W))%,,> GGrcd}|D]}||z} |jj|k7r tdtj|jj |}tj |j|j|||S)Nr z:reshape() cannot change total number of elements in tensor) rSr r#r$rrTr%create_reshapera)re dst_shape can_reorderr*r srs rreshaper%s| E    zz5 UVV ]]5::,,i 8F 99W++ELL)[QSY ZZrc|jDcgc]}tj|}}|j|d|jj st |||Stj|jj|}tj|j|j||Scc}w)Nr )rr*) rr$_constexpr_to_valueinsertrSrr rrTr%create_expand_dimsra)rer)r*rrrs r expand_dimsr/s49KK@q''*@I@ T1 ::   U)W== ]]5::,,i 8F 99W// dCV LLAsCcL|sJdt|jdk(sJtj|jj |jd|jdzg}tj |j|j|j|S)Nz;current implementation of `cat` always may reorder elementsr r) rrr$rrSrTr% create_catra)rVrWrr*ret_types rcatr!:s| UUU; syy>Q  }}SXX__syy|ciil/J.KLH 99W'' CJJ? JJrct|||\}}|jgk(}|rt|d|}t|d|}t|jdtj rt j d}nd}|j|gz}t j |jj|}t j|j|j|j|}|rt|dgd|}|S)Nrr Frr*) rRrrrr$ constexprrrSrTr% create_joinrar)abr* was_rank_1two new_shaper rs rjoinr,As 1g .DAqBJ 1g & 1g &!''"+r||,ll1o3%I}}QVV]]I6H ))G''!((;X FCcA3E7C Jrct|jdkDsJtj|jddk(sJ|jdd}tj|j j |}|j|j\}}tj||tj||fS)Nrr#r ) rrr$rrrSrT create_splitrar%)r'r*r+r outLHSoutRHSs rsplitr1Zs L1    " "1772; /1 45 4 I}}QVV]]I6H))!((3NFF &(# &(# rct|jt|k7r tdtd|Dt t t|k7rtd|t j|jj|Dcgc]}|j|c}}t j|j|j||Scc}w)Nz5permute dims must have the same length as input shapec3FK|]}tj|ywrL)r$r).0ds r zpermute..js 6Ab$$Q' 6s!z?permute dims must be a permutation of 0, 1, ..., n-1, but were ) rrr#sortedlistrr$rrSrTr% create_transra)redimsr*r5r s rpermuter;gs 5;;3t9$PQQ 6 66$uSY?O:PPZ[_Z`abb}}UZZ..0NAQ0NOH 99W))%,,=x HH1OsC c |jjsPtj|j|}tj|j |j ||S|jj}t|t|k7rtd|d|||k(r|St|D]0\}}|||k7s|dk7std||d|d|d|d| tj|jj|}tj|j|j ||S)Nz!Cannot broadcast, rank mismatch: , r z3Cannot broadcast, the expanded size of the tensor (z ) must match the existing size (z) at non-singleton dimension : ) rSrr$rr%rraget_block_shapesrr# enumeraterTcreate_broadcast)rerr*r src_shapeiitems rbroadcast_impl_shaperEqs; ::   uzz51yy--ellEBFKK ++-I 9~U#tdt!| zdzt!| zdzt!| z||k7rPtj|j|} tj |j#|j|| }||k7rPtj|j|} tj |j#|j|| }||fS)Nrr z?Cannot make_shape_compatible: incompatible dimensions at index r>r)rSrr$rrTrr%rrar?rrrr@appendr#strrA) rVrWr*lhs_tyrhs_ty lhs_shape rhs_shape_ ret_shaperCleftrightrs rrRrRsQ XXF XXF!2v}}fll;ii,,SZZ9P9P9RSU[\V 8OS__ 6??#4v}}fll;ii,,SZZ9P9P9RSU[\N 8OK  v0++- ++- y>C N *3y>3y>: 6ii : :3::q I " fmmaS9_ MO"335  6 ^c)n ,3y>3y>: 6ii : :3::q I " fmmaS9_ MO"335  6 9~Y///  + aGAtaLEqy  '1*%4-  & "-/21v"68<"=?B4y"IKR"SUXY^U_"`aa a  !]]6==)? ? 99W++ELL&,,w:OPRX YYrNc|j}t|tjr |j}t|tjr |j}|j r8tj |j|jj}||k(r|S|j}|j}t|}d}|jrf|jrV|j|jkr=|tjj}nH|tjjk7r+d}n(|&tdt!|zdzt!|z|j#s|j#r|j$j&sJd|j)s|j)r<|j*j-d Jd|j*d||||S|j/r|js"|jr|j/s|r@tj0|j3|j4|j7|||S|j9r|j;r |j=r6|j;s&t?t?|tj@|||S|jxr+|jxr|j|jkD}|r?tj0|jC|j4|j7||S|jxr+|jxr|j|jk} | r?tj0|jE|j4|j7||S|jGr |jGr|jH|jHk7s|jJ|jJk7r|jMxr|jO } |jOrW|jPj7|} tj0|jS| |jP} tU|| |Stj0|jW|j4|j7|| |S|jYr|jGr|jOrW|jPj7|} tj0|jS| |jP} tU|| |S|jMr?tj0|j[|j4|j7||Stj0|j]|j4|j7||S|jGr|jYr|jOs|jMs?tj0|j_|j4|j7||Stj0|ja|j4|j7||S|jcr|jGr|jH} | d k(r?tj0|je|j4|j7||S| d k(rWtUt?|tjf|tj0|jid tjf|S|jGrO|jcr?tj0|jk|j4|j7||S|jcrO|jcr?tj0|jm|j4|j7||SJd |d |)NFTz]fp_downcast_rounding should be set only for truncating fp conversions. Source scalar type is z and destination type is z4fp8e4nv data type is not supported on CUDA arch < 89convert_custom_typesz0target doesn't provide conversion for this type.r@r rz cannot cast z to )7rSrr$r%r rrrTr?rXrNrZr rTrUr#rH is_fp8e4nvoptions allow_fp8e4nv is_fp8e4b15 codegen_fnsgetis_fp8r%create_fp_to_fprarrDrBrFrUrCcreate_fp_trunc create_fp_extrHr0r1r|is_boolr2rrcreate_int_castis_standard_floatingcreate_fp_to_sicreate_fp_to_uicreate_ui_to_fpcreate_si_to_fprMcreate_ptr_to_intint64 get_int64create_int_to_ptrr[)rer\r*fp_downcast_roundingr]r^r_use_custom_rounding truncate_fpext_fp sign_extendtyrbitwidths rrUrUs ZZF&",,'& 5399 v}}ejj.I.I.KL  JJ11EFJ$:$:%  ' '**G*G G  '@P@P@U@U)= !R%5%5%:%: :RVW>W[e[t[t>t ..0M9K9K9M5M    ""7+B711"5u{{CBUB0 099W44U\\6<->-@    ""7+B711"5u{{CBUB0 0  % % '99W44U\\6<>@    z'?'?'A99W44U\\6<99W66u||V\\RYEZ[]cd d q=T%7;RYYwGXGXYZG[]_]e]e=fhop pz002yy225<<gAVWY_``z002yy// fll7>STV\]]4LtF8445rctjj}|rQ|dk(rtjj}|S|dk(rtjj}|St d|d|S)Nz.ca.cgCache modifier  not supported)r CACHE_MODIFIERrCACGr#cache_modifiercaches r_str_to_load_cache_modifierrQst    " "E U "%%((E L u $%%((E L~.>nMN N Lrcdtjj}|r|dk(rtjj}|S|dk(rtjj}|S|dk(rtjj }|S|dk(rtjj }|Std|d|S)Nz.wbrz.csz.wtrr)r rrWBrCSWTr#rs r_str_to_store_cache_modifierr]s    " "E U "%%((E Lu $%%((E L u $%%((E L u $%%((E L~.>nMN N Lrctjj}|rQ|dk(rtjj}|S|dk(rtjj}|St d|d|S)N evict_last evict_firstzEviction policy r)r EVICTION_POLICYNORMAL EVICT_LAST EVICT_FIRSTr#)eviction_policyevictions r_str_to_eviction_policyrmsu!!((H l *))44H O  -))55H O//@OP P Orcd}|rQ|dk(rtjj}|S|dk(rtjj}|St d|d|S)NzeronanzPadding option r)r PADDING_OPTIONPAD_ZEROPAD_NANr#)padding_optionpaddings r_str_to_padding_optionryshG V #''00G N u $''//G N~.>nMN N Nrcdtjj}|r|dk(rtjj}|S|dk(rtjj}|S|dk(rtjj}|S|dk(rtjj }|St d|d|S)Nacquirereleaseacq_relrelaxedMemory semantic r)r MEM_SEMANTICACQUIRE_RELEASEACQUIRERELEASERELAXEDr#) sem_optionsems r _str_to_semrs // ) )C  "//))C J9 $//))C J 9 $//11C J 9 $//))C J/ |>JK K Jrc"tjj}|rr|dk(rtjj}|S|dk(rtjj}|S|dk(rtjj}|St d|d|S)Ngpuctasysrr)r MEM_SYNC_SCOPEGPUCTASYSTEMr#) scope_optionscopes r _str_to_scopers    ! !E 5 %%))E L U "%%))E L U "%%,,E L/ ~^LM M Lrc~|rt|ds|g}|Dcgc]*}t|tjr |jn|,}}|D]+}t|t rd|cxkrt |kr(JJt |dkDsJt |t t|k(sJdt|Sycc}w)N__iter__rz'Duplicate dimension in `boundary_check`r) hasattrrr$r%r rrsetr7)boundary_check block_shapeelemdims r_canonicalize_boundary_checkrs~z2,-N]klUY 4(F$**DPll! HCc3'A,Gs;7G,G GG,G GG H>"Q&&&>"c#n*=&>>i@ii>n%% ms/B:c || td|jjj} | tjk7sJd| j r(|t jjk(r td|jj} t|| j}tj|j|j|||||| S)NK`mask` and `other` arguments cannot be specified for loading block pointers3`tl.int1` should be rewrited in `tl.make_block_ptr`z@Padding option `nan` is not supported for integer block pointers)r#rS element_tyr$rrHr rrrr?r%create_tensor_pointer_loadra) ptrmaskrfrrrr is_volatiler*elt_tyr\s r_load_block_pointerrs 5,fgg XX + +F RWW SSS  }}7b&7&7&?&??[\\XX F2.&BYBYB[\N 99**3::~wPUW_almou wwrc |jjjs'td|jj d| | td|s|r td|jj sN|r%|jj r td|r%|jj r td|jj rN|%t ||jj|}|%t ||jj|}|jj} | j} | tjk(r=tj} tj| | j} t|| |}| t|| |}|jj r1|jj} tj| | } n| } |2tj |j#|j$|||| Stj |j'|j$|j$|r |j$nd|||| S)NUnsupported ptr type z in `tl.load`z)`other` cannot be provided without `mask`z`padding_option` or `boundary_check` argument is not supported for loading a tensor ofpointers or loading a scalar. Because the compiler does not know the boundary; please use block pointers (defined by `make_block_ptr`) insteadEMask argument cannot be block type if pointer argument is not a blockzFOther argument cannot be block type if pointer argument is not a block)rSrTrMr#rrrEr?rr$rint8 pointer_type address_spacerUrr% create_loadracreate_masked_load) rrrfrrrrrr*ptr_tyrrr\s r _load_legacyrs$ 88?? ! ! #01B1B1D0E]STT |)DEE.TU U 88    DII&&(de e UZZ((*ef f xx  'chh.G.G.I7SD  (0I0I0KWUEXX__F   F)=)=>3( UFG, xx))+vu- |yy,,SZZ+VX^__yy  & &szz4;;PU [_afhp'2 45;= =rc  t|} t|} t|} |jj r7|jj j rt||||| | | || St||||| | | || SrL) rrrrSrMrrrr) rrrfrrrrrr*rrrs rloadrs ( 7E&7H$^4G xxSXX0099;"3e^WeU]_jlsttCunguhXcelmmrc t||d}|j|j||j|t |t |}t j||SNF require_i64)_convert_to_ir_valuescreate_descriptor_loadrarrrr$r%)desc_ptroffsetsrrrSr*rs rdescriptor_loadrsW#GW%HG&&xGAT'B>'R'>'O QA 99Q rct||d}tj|j|j|j|tj Sr)rr$r%create_descriptor_storeravoid)rr rr*s rdescriptor_storers@#GW%HG 99W44X__ellT[\^`^e^e ffrc | td|jjj}|jj s t |||}|jj sJd||jjk(s&Jd|d|jjd|jjj|jjk(s@Jd|jjjd|jjd|jjj}|t jk7sJdt||}t|||}t j|j|j|j|||t jS) Nrz-Value argument must be block type or a scalarz Block shape(z) and value shape(z ) mismatchzBlock element type(z) and value element type(r)r#rSrr?rrEr$rrrUr%create_tensor_pointer_storerar) rvalrrrrr*rrs r_store_block_pointerrs fgg((%%668K 88   "3 W= 88   O OO  #((33 ] k]"4SXX5N5N5P4QQ[\]  88   ) )SXX-@-@ @qDWX[X`X`XkXkXvXvWwxQRURZRZReReQffpCqq @ XX + +F RWW SSS 2.+NN sFG $C 99W88SZZQ_afhpqWW rc |jjjs'td|jj d|r td|jj sL|jj r td|r%|jj r td|jj rLt ||jj|}|%t ||jj|}|jj}|j}|tjk(r=tj}tj||j}t|||}t|||}|sJtj|j!|j"|j"||tj$S|jjj's tdtj|j)|j"|j"|j"||tj$S)Nrz in `tl.store`z`boundary_check` argument is not supported for storing a tensor of pointers or storing a scalar. Because the compiler does not know the boundary; please use block pointers (defined by `make_block_ptr`) insteadzFValue argument cannot be block type if pointer argument is not a blockrz"Mask must have boolean scalar type)rSrTrMr#rrrEr?rr$rrrrrUr% create_storerarrocreate_masked_store) rrrrrrr*rrs r _store_legacyr;s 88?? ! ! #01B1B1D0E^TUUAB B 88    88   ef f DII&&(de e xx"3(A(A(CWM  'chh.G.G.I7SD XX__F   F)=)=>3( sFG $C yy--cjj#**eXVXZX_X_`` 99   # # %=>> 99W00SZZV[]efhjhoho pprc t|}t|}|jjs$|jjjr t d|jj r5|jjjrt|||||||St|||||||S)N"Cannot store to a constant pointer) rrrSis_constrTr#rMrrrr) rrrrrrr*rrs rstorergs ) 8E&7H xxchhoo668=>> xxSXX0099;#CdNE8U\]]S#t^UHgVVrc Bt|}t|}|jjj}|j dvr t dtj|j|j|j|j|||jS)N)rrdz9atomic_cas only supports elements with width {16, 32, 64}) rrrSrTrrZr#r$r%create_atomic_casra)rcmprrrr*rs r atomic_casr}s| c C % E++J$$L8TUU 99W..szz3::szzSVX]^`c`h`h iirc|jjjs&td|jj z|jj s$|jj j r td|jjj }|tjur|dk7rtd|zdz|tjtjtjtjfvrtd|zdzt|z|jjrN|%t||jj!|}|%t||jj!|}t#||jjj |}|s|j%d}tj}|jjrf|j'||jj!}tj(tj|jj!}tj*||}|||fS)Nz)Pointer argument of store instruction is rriatomic_z does not support fp16z does not support T)rSrTrMr#rrrr$rErrint16rGrHrrEr?rUget_int1rrr%)rrropr*rmask_irmask_tys ratom_red_typechecking_implrs 88?? ! ! #DsxxGXGXGZZ[[ xxchh11::<=>>++JRZZB%KR*BBCCbggrww"++>>R*>>ZPQQ xx  'chh.G.G.I7SD ?&sCHH,E,E,GQC sCHHOO.. 8C ""4('' 88   **7CHH4M4M4OPGmmBGGSXX-F-F-HIGyy'* T>rc t|||d|\}}}t|}t|}|jj}|j r|j rjtj|jtjj|j|j|j|||jStj|jtjj|j|j|j|||jS|tjtj hvrt#d|t%gd||}|tjk(rtj&ntj(}t+|||} t+|tj,|d|} |tjk(rtj.ntj0} t+|| |} t+|tj,| d|} t3|||}t5|||}tj|jtjj| j| jt7|||j||| j}tj|jtjj8| j| jt7|||j||| j}t;||||}t+|||S)Nrz#atomic_max not supported for dtype r )rrrrSrTrHr|r$r%create_atomic_rmwr ATOMIC_OPMAXraUMAXrCrAr5rr'rwrruint32uint64rrrUMINwhererrrrrr*sca_tyri_typei_vali_ptrui_typeui_valui_ptrposnegpos_retneg_retrs r atomic_maxr/S$wONCd c C % E XX__F }}    !99))",,*:*:CJJ TXT_T_adfklnqnvnvx x99))",,*;*;SZZUYU`U`beglmorowowy y bjj"**--=fXFGG C )D2::-RXX288F C )E C3W =E!RZZ/biiRYYG S'7 +F S"//'15w ?F T7 +C Cw 'Cii!!",,"2"2ELL%,,"&tS'":"A"A3 OPUPZPZ\Gii!!",,"3"3V]]FMM"&tS'":"A"A3 OPVP[P[]G Wgw /C 3 ((rc t|||d|\}}}t|}t|}|jj}|j r|j rjtj|jtjj|j|j|j|||jStj|jtjj|j|j|j|||jS|tjtj hvrt#d|t%gd||}|tjk(rtj&ntj(}t+|||} t+|tj,|d|} |tjk(rtj.ntj0} t+|| |} t+|tj,| d|} t3|||}t5|||}tj|jtjj| j| jt7|||j||| j}tj|jtjj8| j| jt7|||j||| j}t;||||}t+|||S)Nrz#atomic_min not supported for dtype rr )rrrrSrTrHr|r$r%rr rMINrar rCrAr5rr'rwrrrr rrrrr r s r atomic_minrrrc t|||d|\}}}t|}t|}|jj}|j rt jjnt jj}tj|j||j|j|j|||jS)Nri)rrrrSrTrNr rFADDADDr$r%rra)rrrrrr*r rs r atomic_addr s/S$wONCd c C % E XX__F$002   8H8HB 99W..r3::szz4;;X[]bcehemem nnrc (t|||d|\}}}t|}t|}tj|j t jj|j|j|j|||jS)Nand) rrrr$r%rr rANDrarSrrrrrr*s r atomic_andr%x/S$wONCd c C % E 99W..r||/?/?SZZY]YdYdfikpqXX rc (t|||d|\}}}t|}t|}tj|j t jj|j|j|j|||jS)Nor) rrrr$r%rr rORrarSr$s r atomic_orr*sv/S$gNNCd c C % E 99W..r|| CJJX\XcXcehjopXX rc (t|||d|\}}}t|}t|}tj|j t jj|j|j|j|||jS)Nxor) rrrr$r%rr rXORrarSr$s r atomic_xorr.r&rc (t|||d|\}}}t|}t|}tj|j t jj|j|j|j|||jS)Nxchg) rrrr$r%rr rXCHGrarSr$s r atomic_xchgr2sx/S$PNCd c C % E 99W..r||/@/@#**cjjZ^ZeZegjlqrXX rc|j|jjvs!Jd|jjd||j}|dk(rd}t t j |S)Nzinput_precision must be one of z. Got TF32X3TF32x3)lowerrfallowed_dot_input_precisionsupperr r INPUT_PRECISION)input_precisionr*s r_str_to_dot_input_precisionr;sx  "goo&R&R Rp )'//*V*V)WW]^m]nop R%++-O("" 2%% 77rc  d}|jjr|jjsJ||j|j|j|jj s|jj r6t |t j|}t |t j|}||jj}t||}t|j}t|j} || cxk(rdk(s1n|| cxk(rdk(s$nJd|jd|jd|jdj|jdjk(sVJd |jd |jd |jdjd |jdjd |jdjd k\r8|jdjd k\r|jdjd k\s"Jd|jd|jd|jjjrs|jjt jk(sJd|jdjdk\sJd|j!d} t j"} n|j%r t'd|jjj)s$|jjj%r"|j+d} t j,} n4|j/r|j1dn|j+d} |} |jjd} |jjd} |dk(r|jjdnd}t j2| |r|| | gn| | g}||j5| |r|| | gn| | g}n|j6}|j|k(sJ|M|jj9r1|jj9r|jj:}nd}t j<|j?|j6|j6||||S)Nc||js\|js|jrJd|jr|jry||k(sJd|d|dy|js|jr@||k(sJd|d|d|j s|j s Jd|dyy|js|jr-|j rgd }nd d g}d }|||d |||dy|js:|js*|js|js Jd||js:|js*|js|js Jd|||k(sJd|d|dy)Nz1Dot op does not support fp8e4nv on CUDA arch < 90z First input (z) and second input (z) must have the same dtype!z0Both operands must be same type. First operand (z) and second operand (rz:Both operands must be either int8 or uint8. Operand type ()fp8e4nvfp8e5fp8e4b15r>r?c ztfd|Ds&dj|}td|d|ddy)Nc3FK|]}td|yw)is_N)r )r4 dtype_namer2s rr6zLdot..assert_dtypes_valid.._validate_dtype..=s%d AwuJ<.@ACds!r=zOnly supports z. z (r)anyr,AssertionError)r2 allowed_types operand_namesupported_typess` r_validate_dtypez9dot..assert_dtypes_valid.._validate_dtype<sMdVcdd*.))M*B,~o=NbQ]P^^`af`ggh-ijjerz First operandzSecond operandzUnsupported dtype ) rgrerkrHis_int8is_uint8allow_fp8e4b15rDrFrBr) lhs_dtype rhs_dtyperfrGrJs rassert_dtypes_validz dot..assert_dtypes_valid*s1$$ ++-i6J6J7 CB C!i&6&6&8 ) A]9+EYZcYdd,A A)!Y%5%5%7 I-P1abkalmCDMCNNO0PP- ((*i.@.@/]OPY{Z[\]/*!!#y'7'7'9))$DM%.$8Mk  =/J =:JK ((*i.?.?.AYEVEVEX\e\m\m]4' {34 ((*i.?.?.AYEVEVEX\e\m\m]4' {34 I-EykI]^g]hiD0EE-rr z+Both inputs must be either 2D or 3D; (lhs: z vs rhs: rr#zFirst input shape (z) and second input shape z= are not compatible for matmul (second index of first shape (z0) must be equal to first index of second shape (rz0All non-batch values in both first input shape (z) and second input shape (z) must be >= 16!zonly int8 supported!r rzsmall blocks not supported!rzhout_dtype=bfloat16 is unsupported. Please use out_dtype=float32/float16 and cast with `.to(tl.bfloat16)`) rSrr2rfrhrUr$rEdefault_dot_input_precisionr;rrr rTrHr get_int32r'rFr#rBget_fp32rCrDget_fp16rrrarkmax_num_imprecise_acc_defaultr% create_dot)rVrWaccr:max_num_imprecise_acc out_dtyper*rPlhs_rankrhs_rankr ret_scalar_tyMNBr acc_handles rdotrc'sE@ 88   388#4#4#66 6 399goo> yy#))"7"7"93 G,3 G,!//EE1/7KO399~H399~H x $1 $H(A(AREpqtqzqzp{|EFIFOFOEPPQDRR A 99R=  #)) # E q( 3LSYYKXUVYV_V_`bVcViViUjjZ[^[d[deg[h[n[nZoopqq  99R=  " $2)<)<)B IIbM  2 %| :399+E_`c`i`i_jjz{| & xxxx"'')A+AA)yy|!!R'F)FF'   q !     vx x  "chhoo&=&=&?   a  $-$5$5$7W  a W=M=Ma=P!  rA rA%]qA ]]=q1a)q!f EF {))"1q!Qi1a&I ZZ xx6!!!$ 99   #))"2"2"4$+OO$Q$Q !$% ! 99W'' CJJ O]rs rct|tj|}|jj r0t |||\}}t |||\}}t |||\}}t |||dd\}}|jj st |||\}}|j}tj|j|j|j|j|S)NT) rUr$rrSrrRr^r% create_selectra) conditionrrr*rMrs rr r sY1I~~ +Iq'B 1#Aq'21+Iq'B 1 '1gtT BDAq >> " " $+Iq'B 1 VVF 99W**9+;+;QXXqxxPRX YYrcd|rtj||}n|}tj||SrL)r$rr%)rrorNres_tys r wrap_tensorris-y)4 99Q rc  |tfdDd}djj t }||ks Jd|dt Dcgc] \}}||k7s |c}} t fdDsJdj Dcgc]}|jc}||jt fdttDScc}}wcc}w)Nc3fK|](}t||jjgd*yw)Tr$N)rr r )r4tr*s rr6zreduction..s*fZ[wq177==/tWUUfs.1rz&reduction axis must be < inputs rank (rc3PK|]}|jjk(ywrL)rSr)r4rlrs rr6zreduction..s5qvv||u$5s#&z-all reduction inputs must have the same shapec3K|]7}tj||jj9ywrLri get_resultrSrT)r4rCinputs reduce_oprNs rr6zreduction..s4t\]Y11!4fQinn6K6KYWt=A) tuplerSrrr@all create_reduceraverifyr) rqr)region_builder_fnr*rankrCrrlrrrNrs ` ` @@@r reductionrzs |f_eff 1INN E u:D $;H@aHH;(/=tq!19=I 5f5 5f7ff 5%%&@Aqxx&@$GIi   tafgjkqgrast tt>'As C;.C; Dcdjjt}| |cxkr|ksnJd|d|d|dkr||z }D]"}|jjk(rJd|jDcgc]}|jc}|||j t fdttDScc}w)Nrz scan axis z must be < inputs rank (rz(all scan inputs must have the same shapec3K|]7}tj||jj9ywrLro)r4rCrqscan_oprs rr6z#associative_scan..s4nVWW//2F1INN4I4I5Qnrs)rSrr create_scanrarwrtr) rqr)rxreverser*ryrlr}rs ` @@rassociative_scanrs 1INN E u:D 5D 4 S:dV3KD6QR!SS  ax   Qvv||u$P&PP$Q!!V"<188"R>R>T)UV Hrct|jt|k7r td|jj dt j ||jj|S)NzDShape of input to max_contiguous does not match the length of valuesz tt.contiguityrrr#rarr rrrs rmax_contiguousrsS 177|s6{"_``HHor||FAHHr#r )endswithr startswithrar$r% create_printr)prefixargshexr*argnew_argss r device_printrs ??3 D#  ??4 Tt# 6{Qv005v&*+s +H+ 99W))&#x@"'' JJ,sB&c \|j}|jsPtj|jd}tj |j |jd|}tj |j|j||||tjS)N)r ) rSrr$rrTr%rra create_assertr)condmsg file_name func_namelinenor*cond_tys r device_assertrs|iiG    --6yy--dkk5A7K 99W**4;;Y SYZ\^\c\c ddrct|trtj|}t|tjr|rGd|jcxkrdksnJd|jd|j |jSd|jcxkrdksnJd|jd|j |jSt|tjr|jjdk(sJd |jjsJd |jtjk7rE|rC|j|j|j|jjS|jtj k7r |sJd |jSJd t#|) Nllz@Block pointers only support 64 bit `shape/strides`, got a value z which is out of the rangezFBlock pointers only support 32 bit `offsets/block_shape`, got a value r z*Expected a scalar in shape/strides/offsetsz8Expected an integer scalar type in shape/strides/offsetszzBlock pointers only support 32 bit `offsets/block_shape`, add a `.to(tl.int32)` or use regular indexing for 64 bit supportz3Unsupported element type in shape/strides/offsets: )rrr$r%r rxrTr%r r2rHrwrpra get_int64_tyr|r'rS)r*rrs r_convert_elem_to_ir_valuer s$||D!$ % TZZ/%/ F4#zzl*D2F F/$$TZZ0 0TZZ/%/ F4#zzl*D2F F/$$TZZ0 0 D")) $zz1$R&RR$zz  "^$^^" :: !k**4;;8L8L8NPTPZPZPhPhPjk k ZZ288 #K SS S5{{TGT |TT5rcvt|dr|Dcgc]}t|||c}St|||gScc}w)Nr)rr)r* list_likerrs rrr#s?y*%R[\$)'4E\\ %gy+ F GG]s6c t||}t||}t||d}|jjr$|jjj r t d|jjt jk(rCt|t jt j|jj|}tdsgDcgc]*}t|t jr |jn|,c}t!dDsJdt|ds|g}|Dcgc]*}t|t jr |jn|,}}t#|t%t't)|k(sJdt!fd||||fDsJd |j+|j,||||}t j.|t jt j0|jjScc}wcc}w) NFrzMExpected `base` to be a pointer type (but not a block pointer type or others)rc3`K|]&}t|txrd|cxkxrdknc(yw)rrN)rr)r4rs rr6z!make_block_ptr..<s)XDz$$?4)?%)??Xs,.zGExpected a list of constant integers (`int32_t` range) in `block_shape`z.Fsdis;3y>1ds!$zBExpected shape/strides/offsets/block_shape to have the same length)rrSrMrrr#r$rrUrrrrrr%r rur7r8rrcreate_make_block_ptrrar%r) baserstridesrrorderr*rras ` rmake_block_ptrr)s "'5 1E#GW5G#GW%HG 99   !5!5!>!>!@hii yyrww&D"//"''4993J3JKWU ; +"m VabdD",,!?4::TIbK XKX XRQR X 5* %PU V:dBLL9TZZt C VE V %=Ds5z!23 3s5ss 3 dE7T[]bCcd dMLM d  * *4;;wQ\^c dF 99VR__R]]499;O;OQ\-]^ __%c Ws ./H= /Ict||d}tj|j|j||j Sr)rr$r%create_advancerarS)rrr*s radvancerPs8#GW%HG 99W++DKKA499 MMr)r)rr* ir.builderreturn tl.tensor)r6tl.dtyper7rrr)r6rr7rrIrrr)rrrrrOrrNone)FFTF)rVrrWrr*rrTuple[tl.tensor, tl.tensor])rerrfrr*rrr) rerrfrrrr*rrr)rrrrrtl.PropagateNanr*r) rrrrrrrrr*r)rerrfrr*rrr)rerr*r)rerrr)rerr*rrr)rerr*rrr)rrrz tl.block_type)rrrrr*rrr)r List[int]r2rr*rrr)r rrrr*rrr) rerrrrrr*rrr)rerr)rr*rrr) rVrrWrrrr*rrr)r'rr(rr*rrr)r'rr*rrr)rerr:z Tuple[int]r*rrr)rerrrr*rrr)rVrrWrr*rrr)rW Optional[str])rerr\rr*rrrrL) rerr\rr*rrzrrr)rrrOptional[tl.tensor]rfrrrrrHrrHrrHrrr*rrr) rrrrHrrHr*rrr)rrr rr*rrr)rrrrrrrrHrrHr*rrr)rrrrrrrrHrrHr*rrr) rrrrrrrrHr*rrz&Tuple[tl.tensor, tl.tensor, tl.tensor])rrrrrrrrHrrHr*rrr)rVrrWrrYrr:rrZrr[rr*rrr) rfrrrrrr*rrr)rqSequence[tl.tensor]r)rr*rrTuple[tl.tensor, ...]) rqrr)rrrr*rrr)rerrrr*rrr)rrrrrr)r*rrr) rrHrzList[tl.tensor]rrr*rrr) rrrrHrrHrrr*rrr)T)rrr*rrr)l __future__rtypingrrrrr _C.libtritonr r r$r r Exceptionrr+r.r<rJrPr^rirprurzrrrrrrrrrrrrrrrrrrkrrrrrrrrrrr rrr!r,r1r;rErRrXrrUrrrrrrrrrrrrrrrrrrrr r%r*r.r2r;rcr rirzrrrrrrrrrrrrrrrrs";; CLF FD F @ ,P <ejGK,16Q":0 4 4R4 :&465$5$ ]" +5 :U Q P Q '&" R R Q#$) 4 4 4 4 4 4"D"(.H[MK2 IL$2tpZ(04o5,o58Ao5n       w,7=t n n.1 nDG nVZ n n!* n ',5g :)qXWW)3W8AW,j(27]6$)N$)No#(18XX&0X5>X@ Z& u.o(o-Bo2n   8 KeU0H $`NNr