rhv UddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZmZddlmZddlmZmZmZmZmZmZmZmZmZddlmZm Z ddl!Z!ddl"Z"ddl#Z"ddl$m%Z%dd l&m'Z(dd l)m*Z*dd l+m,Z,dd l-m.Z/dd l0m1Z1m2Z2m3Z3ddl4m5Z5m6Z6ddl7m8Z8m9Z9ddl:m;Z;ddlZ>ddl?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKddlLmMZMmNZNmOZOmPZPmQZQmRZRercddlSmTZTmUZUmVZVddl#mWZWddlXmYZYddlZm[Z[m\Z\m]Z]m^Z^ddl_m`Z`ddlambZbmcZcmdZdddlemfZfe dZgeeecgebfZheiefZjeeke!jfZmekZne"jjeqdZrejeqZtddZuejGd d!ZwGd"d#ej$ZxGd$d%eZyeEd&'Gd(d)eyZzGd*d+Z{ejGd,d-Z|ejGd.d/Z}ejGd0d1Z~ejGd2d3ZejGd4d5Zeeze|e}ee~fZiZd6ed7<Gd8d9ZiZd:ed;<iZdZGd?d@eZ ddAZ ddBZddCZ d d dDZd dEZejd dFZ d dGZ ddHZddIZe"j&e"j(e"j*e"j(ie"j,e"j.e"j0e"j2e"j4e"j6e"j8e"j:e"j<e"j>e"j@f Dcic]}||c}ZdJedK< ddLZ ddNZGdOdPZGdQdRe/Z.GdSdTZe jLdUe jNVZddWZGdXdYe=eeNeZejGdZd[Zedid\ee%jZd]d^_d`ee%jZdadbdcddeee%jZdfdgdhddiee%jZdjdkdlddmee%jZdndodpddqee%jZdrdsdqtduee%jZdvdwdxddyee%jZdzd{d|dy}d~ee%jZdd~_dee%jZdd_dee%jZdd_dee%jZdd_dee%jZdddddee%jZdddtdee%jZdddddee%jZdd_dee%jZdd_dee%jZdddddee%jZdddddee%jZdd_dee%jZdd_dee%jZdd_dee%jZdd_dee%jZdd_dee%jZdd_dee%jZdd_dee%jZdd_dee%jZdd_dee%jZdd_dee%jZdd¬_dee%jZdĄdŬ_dee%jZdDŽdȬ_dee%jZdʄdˬ_dee%jZd̈́dά_dee%jZdЄdѬ_dee%jZdӄdԬ_dee%jZdքd׬_dee%jZdلdڬ_dee%jZd܄dݬ_dee%jZd߄d_Zded<ddZGddeAZGddeDZGddeZejGddZGddZeZGddZGddZe dekZe dMeeZeree"jvePeeeedfffZGddeeefZGddZGddeeeZejGddZejddZGddZGdde>Zycc}w() annotationsN)ABCabstractmethod)autoEnum)chain) AnyCallablecastClassVarGeneric NamedTupleOptional TYPE_CHECKINGUnion)SelfTypeVar)ELEMENTWISE_TYPE_PROMOTION_KIND)_pytree) OrderedSet)int_oo) PythonPrinter)free_symbol_is_typesymbol_is_typeSymT) bound_sympy ValueRanges)configmetrics)DtypePropagationOpsHandler)BasicMathOpsMixinDefaultHandler) boolean_opsDeferredLineBasegenerate_assertget_current_backendIndentedBuffer ir_dataclass ScopedDict sympy_dotsympy_index_symbol sympy_subs triton_typeunique)ops OpsHandlerOpsValue ReductionType StoreModeV)IteratorMutableMappingSequence) GraphModule)CustomGraphModulePass)Buffer ChoiceCaller FixedLayoutIRNodeLoopBody)BaseScheduling Scheduler SchedulerNodePythonWrapperCodegen_Tschedulecxtjtjrtj d|yy)NzData type propagation: %s) schedule_log isEnabledForloggingDEBUGdebug)msgs q/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/torch/_inductor/codegen/common.pydata_type_loggerrQWs*  /6<0cZeZdZUdZded<ded<d dZed dZd dZed d Z y )FileBackedGraphModulez Output of FX wrapper codegen. Exposes the same methods as ModuleType, but these map back to a GraphModule instead of Python source. r9gmzCallable[..., Any] compiled_fnc&tjddd|_tjtj |jj |j5}|j|jdddy#1swYyxYw)Nzw+z.pyF)modesuffixdelete) tempfileNamedTemporaryFileatexitregisterosremovenamewritevalue)selffs rP __post_init__z#FileBackedGraphModule.__post_init__fsi!33eE    4==#5#56 ]] a GGDJJ    s "BBc.|jjSN)r[rards rP__file__zFileBackedGraphModule.__file__ps}}!!!rRc |j|Srh)rVrdargss rPcallzFileBackedGraphModule.calltst&&rRc.|jjSrh)rUcoderis rPrczFileBackedGraphModule.valuewsww||rRNreturnNonerrstr)rm list[Any]rrr ) __name__ __module__ __qualname____doc____annotations__rfpropertyrjrnrcrRrPrTrT\sF O## ""'rRrTc<eZdZdZdZdZeddZeddZy) WorkspaceZeroModerrDrc||k(s|tjk(r|S|tjk(r|Std|d|d)NzWorkspaceZeroMode.combine(, ))r UNINITIALIZEDNotImplementedErrorabs rPcombinezWorkspaceZeroMode.combinesK 6Q+999H !// /H!$>qe2aU!"LMMrRcF|rtjStjSrh)r ZERO_ON_CALLr) zero_fills rP from_boolzWorkspaceZeroMode.from_bools $11 1 ...rRN)rrrrrrr)rboolrrr) rwrxryrrZERO_PER_GRAPH staticmethodrrr}rRrPrr|s9MLNNN//rRrc4eZdZdZeddZeddZy) CodegenSymbolzP An IR object possibly corresponding to a variable in the wrapper code. cyrhr}ris rPget_namezCodegenSymbol.get_name rRcyrhr}ris rP get_examplezCodegenSymbol.get_examplerrRNrtrrz!Union[torch.Tensor, sympy.Symbol])rwrxryrzrrrr}rRrPrrs/    rRrT)frozenceZdZUdZded<ded<ded<ded <d Zded <ejZd ed <e dddZ e ddZ e d dZ e d dZ d!dZeZd"dZd#dZd$dZed$dZeZeZeZd%dZd&dZd&dZd'dZd(dZy)) WorkspaceArga2A temporary buffer used for a single kernel, then discarded. Not registered as a traditional buffer since there are no users, so it would be dead code eliminated. Args: nbytes: The size of the buffer in bytes. zero_fill: Whether the buffer should be initialized to zero. sympy.Exprcountr zero_mode torch.devicedeviceru outer_namews_ptr inner_name torch.dtypedtypecP|ttjjSrh)nextr5graph workspace_id)prefixs rP unique_namezWorkspaceArg.unique_names!$qww334566rRc|j|jk(xr4|j|jk(xr|j|jk(Srh)rrrrs rPcan_joinzWorkspaceArg.can_joins@ LLALL ( XQWW-? XAHHPQPXPXDX rRct|j|jztj|j|j|j |j |j|jSN)rrrrrr) rrrrrrrrrrs rPjoinzWorkspaceArg.joinsS''AGG#'// Q[[I''88||||   rRc|j|jk(r2|j|jk(r|j|jk(sJtt j |j |j tj|j|j|j|j|j|jSr) rrrrsympyMaxrrrrrrs rPmaximumzWorkspaceArg.maximums GGqww 188qxx#7ALLALL|jjSrh) get_layoutrris rPrzWorkspaceArg.get_examples ,,..rRcfddlm}||j|j|jgdgS)Nr)r=rD)rrsizestride)irr=rrr)rdr=s rPrzWorkspaceArg.get_layouts.$;;****3   rRc"|jSrh)rris rPlayoutzWorkspaceArg.layouts  rRc6tjjSrh)rSZeroris rP get_offsetzWorkspaceArg.get_offsetsww||rRc|jgSrh)rris rPget_sizezWorkspaceArg.get_sizes |rRc8tjjgSrh)rrOneris rP get_stridezWorkspaceArg.get_strides }rRc|jSrh)rris rPrzWorkspaceArg.get_names rRcgSrhr}ris rPget_inputs_that_alias_outputz)WorkspaceArg.get_inputs_that_alias_outputs rRN) workspace_)rrurrru)rrrrrrr)rrrrrrr)rrr)rrrr)rrr=)rrr)rrlist[sympy.Expr]rt)rr list[str])rwrxryrzr{rtorchuint8rrrrrrrget_device_or_errorrrrr|rget_output_specmaybe_get_output_specmaybe_get_layoutrrrrrr}rRrPrrs    OJE;$77        %/ !!!O&!rRrceZdZddZddZy)TritonScratchWorkspacec ||_||_yrh)r_generate_dtype_str)rdrgenerate_dtype_strs rP__init__zTritonScratchWorkspace.__init__s #5 rRc"|jSrh)rris rPrz)TritonScratchWorkspace.generate_dtype_strs''))rRN)rintrCallable[..., str]rt)rwrxryrrr}rRrPrrs 6*rRrcpeZdZUded<ded<ded<ej j Zded<dZd ed <y) TensorArgrurabufferrrroffsetN Optional[str]alias_of) rwrxryr{rrrrrr}rRrPrr s. I K FJ%"Hm"rRrc4eZdZUded<ded<eddZy)SizeArgrurarexprcyrhr}ris rPrzSizeArg.alias_ofsrRNrrr)rwrxryr{r|rr}rRrPrrs I  rRrceZdZUded<y) ConstexprArgruraNrwrxryr{r}rRrPrrs IrRrc6eZdZUded<ded<ded<ded<y) TMADescriptorArgruraapi_typezOptional[list[sympy.Expr]] block_shapeOptional[torch.dtype]rNrr}rRrPrr$s IM++  rRrc0eZdZUded<ded<dZded<y) DeviceCodegenSchedulingConstructor schedulingWrapperConstructorwrapper_codegenNOptional[WrapperConstructor]cpp_wrapper_codegen)rwrxryr{rr}rRrPrr,s%%''8<5t|||t|<|t|<yrh)rrr))rdevice_schedulingdevice_wrapper_codegendevice_cpp_wrapper_codegendevice_custom_passs rPregister_backend_for_devicer/s),13MOF%7&!rRceZdZeZeZeZeZeZeZ eZ eZ eZ eZ y)BackendFeatureN)rwrxryrFOREACH BUCKETIZEINPLACE_BUFFERSMASKED_SCATTER_WITH_INDEXSCANSORTTUPLE_REDUCTIONPREFER_STORE_LOOP_ORDERTRITON_TEMPLATESREDUCE_TO_SINGLE_ELEMENTr}rRrPr1r1sLfGIfO $ 6D 6DfO"fv#vrRr1c:| tStt|tjr |j }n7t|t sJt ||}tj|}t|}|sJ|d}|j|Srh) rinit_backend_registration isinstancerrtyperuget_scheduling_for_deviceget_backend_features)r device_typescheduling_ctorrs rPrArAs~|&%,,'kk &#&4V 4& k*/ r1rA)rfeatures rPhas_backend_featurerFs% g~ .. . *62 22rRc<|tvrt|jSdSrh)rrrs rPr@r@s 17?1J?6 " - -TPTTrRcZ|tvr#t|}|r |jS|jSyrh)rrr)r cpp_wrapperwrapper_codegen_objs rPget_wrapper_codegen_for_devicerKsA -z+init_backend_registration..s?|F,>,>? KrRcuda)r[rZc6tj|Srh)r cuda_backend)r cuda_backendss rPr_z+init_backend_registration..sA}V-@-@A*MrRxpumps privateuseoner)_get_custom_mod_func SchedulingrFCppWrapperCodegen)rYrOcpp_wrapper_cpurPcpp_wrapper_cpu_array_refrQcpp_wrapper_gpurRcpp_wrapper_mpsrScuda_combined_schedulingrTrZrUrerVr[rWwrapperrFr@r/r aot_inductorallow_stack_allocationr_C_get_privateuse1_backend_name torch.utils.backend_registrationrg RuntimeError)rOrPrQrRrSrTrUrVrWrFprivate_backendrgr+rrr^rcs @@rPr=r=sp".@..@($(- '/ && $  K ""99 "  !(0-&  $  M   !'/#     !'/#     hh<<>O?* %o 6 >I  4\ B 23IJO"67J"K  _9L+#%#' :M_  ? +    s,D77 EEcLddlm}g|t||j|S)Nr)FlexibleLayout)rrxr+contiguous_strides)index index_varssizesrxs rPindex_prevent_reorderingr}!s, $ UU TIj.*K*KE*RS TTrRc|t|<yrh)r()rdevice_op_overridess rPregister_device_op_overridesr,s(;V$rRct|tsJt|tsddlm}m}ddlm}ddl m}t|S)NrD)cpu_device_op_overridesmps_device_op_overrides)r) r>rur?r(rrr`rrd)rrrrxpu_op_overridess rPget_device_op_overridesr2s2 fc "0DL0 " #F-@ #F ++rRzdict[torch.dtype, torch.dtype]DTYPE_TO_COMPUTATION_DTYPEcr|tvrtjS|dvrd|vr|dS|dS|dvrtjS|dvrtjS|dk(rd|vr|dS|dS|dk(rd|vr|dS|dS|d vr$|d}t j j|S|d k(rd|vr|dS|d Sy ) zK Given op name and a list of input dtypes, deduce the output dtype )to_dtype index_exprr)randrandn) get_index randint64 load_seed reductionrDconstant)loadstorestore_reductionto_dtype_bitcastN)r$rrfloatint64r5rr)op_namermkwargsbuf_names rPdeduce_output_dtype_by_namerSs +-zz  #*V"3vgAbA  {{   {{ K ")V"3vg@a@ J ")V"3vgAbA   7ww  ** & &")V"3vgAbA rRCSEVariableTypect}tjjr'|dk(r"|j d|dt |dytjj r|dk(rddlm}m }t||sJt||tjk(r|jrd|d }n.d |d |d }n$d |d}|jrd|d}d|d||d}|j d|dyyy)Nr[ztl.static_assert(z .dtype == rrYrD)CppCSEVariable DTYPE_TO_CPPzIsVecMaskType::valuezstd::is_same_v || std::is_same_vz decltype(z typename z ::value_typezstd::is_same_vzstatic_assert(z);)r'r test_configsruntime_triton_dtype_assert writeliner.static_cpp_dtype_assert cpp_utilsrrr>r?rris_vec)rvarrbackendrr is_same_dt c_var_types rP check_dtyper}s"#G 667h;N,SEK*R89!:J 4rRc`eZdZd dZd dZd dZd dZd dZddZe ddZ e ddZ y )DataTypePropagationc||_d|jji|_|jj D]\}}|j|j|< yNroot)body root_blockrgraphs subblocksitems)rdrkvs rPrzDataTypePropagation.__init__sU DOO))B  NN((* %DAqWWDKKN %rRc|j}|Dcgc]9}t|tjjs(|j dk7s8|;}}t |dk(rytd|D}|sytjtj|Dcgc])}|jtjj+c}Scc}wcc}w)N placeholderrc3K|]K}tj|jvxr)|jtjjduMywrh)OptimizationContextkeymetar).0ns rP zBDataTypePropagation.deduce_node_dtype_by_inputs..sS)  # #qvv - B*../55TA B) sAA)all_input_nodesr>rfxNodeoplenall functoolsreduce promote_typesrrrr)rdnodeinputsr input_nodesall_input_nodes_propagateds rPdeduce_node_dtype_by_inputsz/DataTypePropagation.deduce_node_dtype_by_inputss%% Auxx}}!=!$$-BWA   { q %() !) & " *   rrrr?deduce_node_dtyperu startswithrrrr)rdrnode_arg output_dtypes rPrz%DataTypePropagation.deduce_node_dtypes 77m # ;;( "s499~': ;;(** *yy|Hh 6 FX F6))(3 3$++s+>T$++->>+ ;; ! !"3 455d; ;8 ++ L    //55rRcn|jsJd}|jD]}tj|jvr|jtj}n t}|j ||_||jtj<|j dk(s|j }|S)Nr)nodesrrrrrr)rdr graph_dtyperopt_ctxs rPrz#DataTypePropagation.propagate_graphs{{{-1 KK ,D"&&$))3))$7$;$;<-/ 2248GM18DII)-- .{{h&%mm  ,rRc>|j|jdSr)rrris rP propagatezDataTypePropagation.propagates##DKK$788rRc.||jSrh)r)clsrs rPpropagate_loopbodyz&DataTypePropagation.propagate_loopbodys4y""$$rRcddlm}ddlm}t ||sJt |t |j |sJt |j tj|j S)Nrr?)rC) loop_bodyr@ schedulerrCr>r?_bodyrr)rrr@rCs rPpropagate_scheduler_nodez,DataTypePropagation.propagate_scheduler_nodesX(-$ .:T :.$**h/Adjj1AA/"55djjAArRN)rr@rrrs)r torch.fx.Noderrr)rrrrr)rztorch.fx.Graphrrr)rrr)rr@rrr)rrCrrr) rwrxryrrrrrr classmethodrrr}rRrPrrsJ% * 6:$9%%BBrRrc6eZdZddd dfdZxZS)rT)simplifypc|r]t|tjrCttj dr)tj j j|}t|%|S)Nsizevars) r>rExprhasattrr5rrrsuperdoprint)rdrrr __class__s rPrzPythonPrinter.doprintsK  44*9U77##,,T2Dwt$$rR)rrrrrrrrru)rwrxryr __classcell__rs@rPrrs248D%%-1%=A% %%rRrcTeZdZdZeddZeddZeddZeddZeddZ eddZ eddZ edd Z edd Z edd Zedd Zedd ZeddZeddZeddZeddZeddZeddZy)OpDecompositionsz! Decomposes inductor ops c|Srhr})rcs rPidentityzOpDecompositions.identity s  rRcrtjtjdtj|SNrD)r0truedivrrint32xs rP reciprocalzOpDecompositions.reciprocals"{{3<<5;;7;;rRc.tj||Srh)r0mulrs rPsquarezOpDecompositions.squareswwq!}rRctjtjdtjtj |Sr)r0subrrfloat32erfrs rPerfczOpDecompositions.erfcs*wws||Au}}5swwqzBBrRctjtjtj|tj|Srh)r0rexprr rs rPerfcxzOpDecompositions.erfcxs,wwswwszz!}-sxx{;;rRctjtj|tjdtj Sr)r0r rrrr rs rPexpm1zOpDecompositions.expm1s*wwswwqz3<<5==#ABBrRc tjtj|tjdt jdz t j S)NrD r0rlogrmathrr rs rPlog10zOpDecompositions.log10"s7wwswwqz3<<DHHRL0@%--#PQQrRc tjtj|tjdt jdz t j S)NrDrrrs rPlog2zOpDecompositions.log2&s6wwswwqz3<<DHHQK#OPPrRc tjtj|tjt j dt jS)Nr)r0rrrrrrr rs rPexp2zOpDecompositions.exp2*s3wwswwq#,,txx{EMM"JKLLrRc tjtj|tjdtj Sr)r0raddrrrrs rPlog1pzOpDecompositions.log1p.s+wwswwq#,,q%++">?@@rRc tjdtj}tj|tj |tj tj|Sr)r0rrrrrrneg)rones rPsigmoidzOpDecompositions.sigmoid2sCll1ekk*{{3SWWSWWQZ-@ ABBrRcrtj|tjdtjSNr)r0rrrrrs rPreluzOpDecompositions.relu7s"{{1cll1ekk:;;rRcVtjtj|||Srh)r0rrryzs rPfmazOpDecompositions.fma;swwswwq!}a((rRcTtjtj||Srh)r0rfloorrrs rP floor_to_intzOpDecompositions.floor_to_int@||CIIaL%00rRcTtjtj||Srh)r0rceilr.s rP ceil_to_intzOpDecompositions.ceil_to_intDs||CHHQK//rRcTtjtj||Srh)r0rtruncr.s rP trunc_to_intzOpDecompositions.trunc_to_intHr0rRc tj||}tjtj|tjdt j tjtj|tj|}tj|tj|||Sr%) r0modand_nerrrsignbitwherer)rrrconds rP remainderzOpDecompositions.remainderLsy GGAqMxx FF1cll1ekk2 3 FF3;;q>3;;q> 2 yyswwq!}a00rRcTtjtj||Srh)r0rroundr.s rP round_to_intzOpDecompositions.round_to_intUr0rRN)rcOpVarTrrrCrrCrrrC)rrCr)rCr*rCrrrC)rrCrrrrrCrrCrrCrrrC)rwrxryrzrrrrr rrrrrrr#r&r+r/r3r6r?rBr}rRrPrrs}<<CC<<CCRRQQMMAACC<<))1100111111rRrz[a-z0-9_.]+|\([^)]*\)|)flagsc|ddk7st|dkryd}t|ddD]3\}}|dk(r|dz }n |dk(r|dz}|dk(s!|t|dz k7s3y|dk(sJy)Nr(rFrDrT)r enumerate)stringrichars rP_all_in_parensrM]s ayC3v;? EVABZ(4 3; QJE S[ QJE A:!s6{Q.  A:: rRcHeZdZed dZed!dZed"dZed#dZed$dZed$dZ ed$dZ ed$dZ ed$d Z ed%d Z ed&d Z d' d(d Z d)dZd*dZ d+ d,dZd-dZ d.dZ d/dZ d0dZ d1 d2dZd3dZdej4d dd d4dZd5dZd6dZed7dZed8dZ ed9dZ!y): OpOverridescrt|ts tj|s t |r|Sd|dSNrHr)r> CSEVariable_RE_PAREN_NOT_NEEDED fullmatchrM)rJs rPparenzOpOverrides.parenms9 v{ +#--f5f%M6(!}rRct|Srh)repr)rcrs rPrzOpOverrides.constantxs E{rRc2dtj|S)N~rOrUrs rP bitwise_notzOpOverrides.bitwise_not|s;$$Q'())rRc2tj|dS)Nz == 0rZ)rs rP logical_notzOpOverrides.logical_nots##A&'u--rRc\tj|dtj|S)Nz & rZrr)s rP bitwise_andzOpOverrides.bitwise_and+##A&'s;+<+> rZr_s rPbitwise_right_shiftzOpOverrides.bitwise_right_shiftrhrRc.tj||Srh)r0rrs rP int_truedivzOpOverrides.int_truedivs {{1a  rRcTtj|tj|Srh)r0rrInteger)rars rPrzOpOverrides.load_seedsxxemmF344rRTc*tt|Srh)r,ru)rdrrcheckwrap_negs rPindirect_indexingzOpOverrides.indirect_indexings"#c(++rRcDtt|jd)Nz,: check_bounds should be handled by CSEProxyrr?rwrdrrloweruppers rP check_boundszOpOverrides.check_boundss'"Dz""##O P  rRcDtt|jd)Nz$: load should be handled by CSEProxyrtrdrarzs rPrzOpOverrides.loads%!Dz""##G H  rRNcDtt|jd)Nz%: store should be handled by CSEProxyrtrdrarzrcrXs rPrzOpOverrides.stores'"Dz""##H I  rRcDtt|jd)Nz/: store_reduction should be handled by CSEProxyrtrdrarzrcs rPrzOpOverrides.store_reductions%!Dz""##R S  rRcDtt|jd)Nz): reduction should be handled by CSEProxyrtrdr src_dtypereduction_typercs rPrzOpOverrides.reductions'"Dz""##L M  rRcDtt|jd)Nz$: scan should be handled by CSEProxyrtrddtypes combine_fnvaluess rPscanzOpOverrides.scans'"Dz""##G H  rRcDtt|jd)Nz$: sort should be handled by CSEProxyrtrdrrstable descendings rPsortzOpOverrides.sorts'"Dz""##G H  rRcDtt|jd)Nz): bucketize should be handled by CSEProxyrtrdr boundariesboundary_indicesindexing_dtyperightsortersorter_indicess rP bucketizezOpOverrides.bucketizes'"Dz""##L M  rRcDtt|jd)Nz2: halide_clamp only implemented for Halide backendrt)rdrcrrps rP halide_clampzOpOverrides.halide_clamps%!Dz""##U V  rRrD) constraintsris_purepackcDtt|jd)Nz<: inline_asm_elementwise only implemented for Triton backendrt)rdasmrrrrrs rPinline_asm_elementwisez"OpOverrides.inline_asm_elementwises'"Dz""##_ `  rRcDtt|jd)Nz.: ops.output should not appear at codegen timeAssertionErrorr?rwrls rPrzOpOverrides.outputs%Dz""##Q R  rRcDtt|jd)Nz3: ops.placeholder should not appear at codegen timerrdrzs rPrzOpOverrides.placeholder s%Dz""##V W  rRc0dfd }|_d|_|S)NcJtt|jd)Nz does not implement ops.rt)rdrmrras rP unimplementedz1OpOverrides._unimplemented..unimplementeds*%:&&''?vF rRT)rdrOrmr rr rrrC)rwis_unimplemented)rars` rP_unimplementedzOpOverrides._unimplementeds   "& )- &rRcpt||d}tt|d}| xs||k(xs t|ddS)NrF)getattrr1)rrafn default_fns rP_is_unimplementedzOpOverrides._is_unimplementeds? S$ %Zt4 vSz)SWR9KU-SSrRcP|dvsJ|tjD]\}}t||}|/|j|s&t |||j |C||j vsJd|d|j||_t ||t|y)N)r[rYcppvecrZrezmultiple definitions of z on ) pointwise_overrides_datarrrsetattrr__dict__rwr)rrfuncnamedataimpls rP_initialize_pointwise_overridesz+OpOverrides._initialize_pointwise_overrides"sEEMvME6<<> ;NHd4(D|((2C3+=+=h+GHs||3.xjS\\NK3!) X|D'9: ;rR)rJrCrrrC)rczUnion[bool, float, int]rrrrrCrD)rrCrrrC)rrCr)rCrrrCrE)rarurrCrrrCTT) rrCrUnion[sympy.Expr, int]rprrqrrr sympy.Symbol rrrrrvrrwrrrrs)rarurzrrrrCrh) rarurzrrcrCrXr4rrrs)rarurzrrcrCrrrs) rrrrrr3rc!Union[OpVarT, tuple[OpVarT, ...]]rrr)rtuple[torch.dtype, ...]rzFCallable[[tuple[OpVarT, ...], tuple[OpVarT, ...]], tuple[OpVarT, ...]]rtuple[OpVarT, ...]rrr) rrrrrrrrrrrNN)rrCr.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr]rrCrrrrr Optional[tuple[str, sympy.Expr]]rzOptional[OpVarT]rrrC)rcrCrrrprrrrC)rrCrrurrrrrrrrrrrC)rmrCrrrs)rzrrrrC)rarurrzCallable[..., OpVarT]rarurrr)rrurrrs)"rwrxryrrUrr[r]r`rcrergrjrlrrrrxrrrrrrrrrr rrrrrrrr}rRrPrOrOls'**..BBBBBBCCCC!! 55 , ,%, ,  ,  ,  &0 9= FJ   NR  * 39 AJ        &  1  +    '     #       '  #        $48+/    C  !  $    1  )      &*"]]      #             TT  ; ;rRrOc|eZdZUded<ded<dZded<dZded<ejZd ed <dZ ded <dZ ded <y) OverridesDatarurarrYNzOptional[Callable[..., str]]r[rrtype_promotion_kindrZre) rwrxryr{r[rrDEFAULTrrZrer}rRrPrr3sQ I +/F (/+/F (/'//8,0F (/(,C %,rRrairy_aicd|dS)Nzairy_ai_forward(rr}rs rPr_r_Gs(1-rRspecial_airy_ai)rrYra bessel_j0cd|dS)Nzbessel_j0_forward(rr}rs rPr_r_L*1#Q/rRcd|dS)Nz libdevice.j0(rr}rs rPr_r_M=1-rRspecial_bessel_j0)rrYr[ra bessel_j1cd|dS)Nzbessel_j1_forward(rr}rs rPr_r_RrrRcd|dS)Nz libdevice.j1(rr}rs rPr_r_SrrRspecial_bessel_j1 bessel_y0cd|dS)Nzbessel_y0_forward(rr}rs rPr_r_XrrRcd|dS)Nz libdevice.y0(rr}rs rPr_r_YrrRspecial_bessel_y0 bessel_y1cd|dS)Nzbessel_y1_forward(rr}rs rPr_r_^rrRcd|dS)Nz libdevice.y1(rr}rs rPr_r__rrRspecial_bessel_y1digammacd|dS)Nz calc_digamma(rr}rs rPr_r_ds aS*rRc |dS)Nz .digamma()r}rs rPr_r_es A3j)rR)rrYrrarcd|dS)Nz calc_erfcx(rr}rs rPr_r_l A3a(rRcd|dS)Nzlibdevice.erfcx(rr}rs rPr_r_ms+A3a0rR special_erfcxr+cd|d|d|dS)Nz std::fma(rrr}r(s rPr_r_rsis"QCr!A6rRcd|d|d|dS)Nzfmadd(rrr}r(s rPr_r_sss"QCr!A6rRcd|d|d|dS)Nzlibdevice.fma(rrr}r(s rPr_r_tss"QCr!A>rR)rrYrr[raigammacd|d|dSNz calc_igamma(rrr}r_s rPr_r_z<s"QCq1rRigammaccd|d|dSNz calc_igammac(rrr}r_s rPr_r_=2aS2rRgammainccd|d|dSrr}r_s rPr_r_rrRspecial_gammainc gammaincccd|d|dSrr}r_s rPr_r_rrRspecial_gammaincci0cd|dS)Nzcalc_i0(rr}rs rPr_r_1orRcd|dSNzlibdevice.cyl_bessel_i0(rr}rs rPr_r_3A3a8rRc |dS)Nz.i0()r}rs rPr_r_s A3erR)rrYr[rrai0ecd|dS)Nz calc_i0e(rr}rs rPr_r_ !A&rRc |dS)Nz.i0e()r}rs rPr_r_s A3frR special_i0ei1cd|dS)Nzcalc_i1(rr}rs rPr_r_rrRcd|dSNzlibdevice.cyl_bessel_i1(rr}rs rPr_r_rrR special_i1i1ecd|dS)Nz calc_i1e(rr}rs rPr_r_rrR special_i1elog_ndtrcd|dS)Nzcalc_log_ndtr(rr}rs rPr_r_sqc+rRspecial_log_ndtrmodified_bessel_i0cd|dS)Nzmodified_bessel_i0_forward(rr}rs rPr_r_3A3a8rRcd|dSrr}rs rPr_r_rrRspecial_modified_bessel_i0modified_bessel_i1cd|dS)Nzmodified_bessel_i1_forward(rr}rs rPr_r_r rRcd|dSrr}rs rPr_r_rrRspecial_modified_bessel_i1modified_bessel_k0cd|dS)Nzmodified_bessel_k0_forward(rr}rs rPr_r_r rRspecial_modified_bessel_k0modified_bessel_k1cd|dS)Nzmodified_bessel_k1_forward(rr}rs rPr_r_r rRspecial_modified_bessel_k1ndtrcd|dS)Nz calc_ndtr(rr}rs rPr_r_s 1#Q'rR special_ndtrndtricd|dS)Nz calc_ndtri(rr}rs rPr_r_rrR special_ndtri polygammac *|d|d|d|d|d|d S)Nz == 0 ? calc_digamma(z) : (z == 1 ? trigamma(z) : calc_polygamma(rz))r}r_s rPr_r_s8S%aSaS0A!DWXYWZZ\]^\__a brRscaled_modified_bessel_k0cd|dS)Nz"scaled_modified_bessel_k0_forward(rr}rs rPr_r_:1#Q?rR!special_scaled_modified_bessel_k0scaled_modified_bessel_k1cd|dS)Nz"scaled_modified_bessel_k1_forward(rr}rs rPr_r_r#rR!special_scaled_modified_bessel_k1spherical_bessel_j0cd|dS)Nzspherical_bessel_j0_forward(rr}rs rPr_r_s4QCq9rRspecial_spherical_bessel_j0zetacd|d|dS)Nzzeta(rrr}r_s rPr_r_s52aS*rR special_zetachebyshev_polynomial_tcd|d|dS)Nzchebyshev_polynomial_t_forward(rrr}r_s rPr_r_:1#Rs!DrRspecial_chebyshev_polynomial_tchebyshev_polynomial_ucd|d|dS)Nzchebyshev_polynomial_u_forward(rrr}r_s rPr_r_r0rRspecial_chebyshev_polynomial_uchebyshev_polynomial_vcd|d|dS)Nzchebyshev_polynomial_v_forward(rrr}r_s rPr_r_r0rRspecial_chebyshev_polynomial_vchebyshev_polynomial_wcd|d|dS)Nzchebyshev_polynomial_w_forward(rrr}r_s rPr_r_r0rRspecial_chebyshev_polynomial_wlegendre_polynomial_pcd|d|dS)Nzlegendre_polynomial_p_forward(rrr}r_s rPr_r_9!BqcCrRspecial_legendre_polynomial_pshifted_chebyshev_polynomial_tcd|d|dS)Nz'shifted_chebyshev_polynomial_t_forward(rrr}r_s rPr_r_B1#Rs!LrR&special_shifted_chebyshev_polynomial_tshifted_chebyshev_polynomial_ucd|d|dS)Nz'shifted_chebyshev_polynomial_u_forward(rrr}r_s rPr_r_rArR&special_shifted_chebyshev_polynomial_ushifted_chebyshev_polynomial_vcd|d|dS)Nz'shifted_chebyshev_polynomial_v_forward(rrr}r_s rPr_r_ rArR&special_shifted_chebyshev_polynomial_vshifted_chebyshev_polynomial_wcd|d|dS)Nz'shifted_chebyshev_polynomial_w_forward(rrr}r_s rPr_r_rArR&special_shifted_chebyshev_polynomial_whermite_polynomial_hcd|d|dS)Nzhermite_polynomial_h_forward(rrr}r_s rPr_r_s82aSBrRspecial_hermite_polynomial_hhermite_polynomial_hecd|d|dS)Nzhermite_polynomial_he_forward(rrr}r_s rPr_r_r=rRspecial_hermite_polynomial_helaguerre_polynomial_lcd|d|dS)Nzlaguerre_polynomial_l_forward(rrr}r_s rPr_r_!r=rRspecial_laguerre_polynomial_lzdict[str, OverridesData]rctfdtjjtjjtjj tjj fDS)Nc3&K|]}|v ywrhr})rrras rPrz$is_buffer_removed..(s   )anyr5rremoved_bufferskernelinplaced_to_removeras`rPis_buffer_removedr]'sU  GG # # HH $ $ GG & & HH ' '   rRc4eZdZdZdfd ZddZddZxZS) DeferredLinezHA line that can be 'unwritten' by adding name to V.graph.removed_bufferscVt||||_t|trJyrh)rrrar>r%)rdraliners rPrzDeferredLine.__init__6s+  d$45555rRcFt|js |jSyrh)r]rararis rP__call__zDeferredLine.__call__;s +99 rRc.t|j|Srh)r_ra)rdras rP _new_linezDeferredLine._new_line@sDIIt,,rR)rarurarur)rarurrr_)rwrxryrzrrcrerrs@rPr_r_3sR6  -rRr_ceZdZdddZy) BracesBuffercHtjdfd }|S)Nc3KtD](}jdxjdz c_*t D](}xjdzc_jd*dt D](}jdxjdz c_*tD](}xjdzc_jd*yw)N{rD})ranger_indent)_rrds rPctxz BracesBuffer.indent..ctxFs6] "s# !  "F7^ $ ! s# $ F7^ "s# !  "6] $ ! s# $sC C#)rrIterator[None]) contextlibcontextmanager)rdrros`` rPindentzBracesBuffer.indentEs$  " " $ # $u rRN)rD)rrrrz'contextlib.AbstractContextManager[None])rwrxryrsr}rRrPrgrgDsrRrgc"eZdZUded<ded<y)InplacedBufferrurr other_namesNrr}rRrPruruYs OrRruc.eZdZUded<dZded<ddZy) ArgNameruraFr is_constexprcB|j|jrdSdS)Nz : tl.constexprr)raryris rP full_namezArgName.full_nameds*))$2C2C.LMMLMMrRNrt)rwrxryr{ryr{r}rRrPrxrx^s IL$NrRrxceZdZddZy) RemovedArgcy)NREMOVEDr}ris rP__str__zRemovedArg.__str__isrRNrt)rwrxryrr}rRrPr}r}hsrRr}ceZdZe ddZddZddZeddZddZddZ ddZ ddZ dd Z dd Z dd Zd d Zd!d Zd"dZd#dZ d$ d%dZ d&dZd'dZd(dZd)dZy)* KernelArgsc~|j|t}t|tr|t |x||<}|S|Srh)getrr>r}r)rodictraresult new_results rP_lookupzKernelArgs._lookupqsD */4)A fj )*0#e*'> >E$K*  rRcJi|_i|_i|_i|_g|_yrh) input_buffersoutput_buffersinplace_buffersrworkspace_argsris rPrzKernelArgs.__init__}s)-/ACMO/1 24rRc djdjtt|j|j |j |jgS)NzKernelArgs({})r)formatrmaprWrrrrris rP__repr__zKernelArgs.__repr__sS&& II**++,,    rRc"t|tSrh)r>r}r\s rP_buffer_is_marked_removedz$KernelArgs._buffer_is_marked_removeds$ ++rRc:tjjr4tjjjj ||}|tjj vsJ|||j vrtt|j |S||jvr'tt|j|jS|jdr|jd|j|S|jd|j|S)Nseedin_ptr)r5rrmutation_real_namerrYrr rurrurrrrrs rPinputzKernelArgs.inputs 77  77$$77;;D$GD1772228D82 4&& &T0067 7 4'' '(<(r}rvappendrrr/ru)rd input_name output_namebufval alive_buffersrYinplace_buffer_idxs rP make_inplacezKernelArgs.make_inplacesk 22 2 GG % % ) )+ 6$"6"66C C6 -- -&&z2C!#z2 22 OO " "; /03D  - //668!#z2M //668c:.O "%VM%:!;c/>R!R  /01[)C03D  ,03D  -! s E3E8cJt|tj|tjj tj }t|jD]\}}tj||r?|j}tj|||j|<|j|fcS|j|jk7r|j|jk7rJ||jj||jdfS)a Allocate or extend a workspace buffer of nbytes bytes. This function manages the allocation of a workspace buffer. It either creates a new WorkspaceArg or extends an existing one. Note: - Calling this function will in-place mutate the args by adding or updating a WorkspaceArg. - The codegen for generating the Python argdefs and call_defs will check this field and allocate the buffer accordingly. - A new argument "ws_ptr" will be present in the generated code. Args: nbytes (sympy.Expr): The number of bytes to allocate. zero_fill (bool): Whether to initialize the buffer to zero. Returns: Tuple[str, int]: A tuple containing: - "ws_ptr": A string identifier for the workspace pointer. - offset: An integer representing the byte offset in the workspace. )rrrrr)rrrr5rget_current_device_or_throwrrIrrrrrrr)rdnbytesrargrK existing_argrs rPr%zKernelArgs.workspaces.'11)<77668#//1   ))<)<= OA|$$\37%++)5):):<)M##A&#..66''3>>9 ++s~~=  >   ""3'~~q  rRc tjj}t|tj t jdd|jd|j|}|jD]*}|j|jk(s||k(r#J||f|jj||jS)a Lazily allocate a graph-wide semaphores buffer with at least min_size. This is a single buffer shared by all kernels and zero initialized once at graph start. Each kernel must leave the buffer zeroed on exit. Warning: multiple calls to this function will return the same buffer. Args: min_size: the number of int32 semaphores required Returns: name of the semaphores buffer sem_ptr semaphores_rn)rrrrrr) r5rrrrrruint32r?rzrrr)rdmin_sizecurrent_devicerrs rP semaphoreszKernelArgs.semaphoress<<>'66,, $^%8%8$9>;O;O:PQ!  !// @L&&#..8l*?S,,??* @ ""3'~~rRcft|tsJt||ftj|}||j vr|j |S|j j vr0tfd|j j D|j |<S)Nc3FK|]}|jsdyw)rDN)r)rrras rPrz)KernelArgs.seed_offset..sU1!,,tBTQUs!!)r>rr?rrnrrsum)rdrarcs ` rP seed_offsetzKernelArgs.seed_offset s%%;U U';;% e$ DMM !==' ' 4=='') )&U (<(<(>UUVW  $ e rRct|tjsJt||f|jdk(rd|j |<y|j d|j |S)Nrks)r>rSymbolr?rarrrs rPrzKernelArgs.sizesX$ -AT D/AA- 99 "(DMM$ ||D$--66rRct|jj|jj|jjSrh)rrkeysrrris rP call_nameszKernelArgs.call_names!sA    # # %t':':'?'?'A4==CUCUCW  rRc|jj|d}|t|ts |jS|j j|d}|t|ts|S|j j|dS)z; Returns inner name of a given outer name. N)rrr>r}rrr)rdrainplacedrs rParg_namezKernelArgs.arg_name&s}''++D$7   8Z(H&& &))--dD9  ":k:+N !!%%dD11rRc|Srhr})rdrrs rP wrap_ptr_argzKernelArgs.wrap_ptr_arg2s rRct|Srh)ru)rdrs rP wrap_size_argzKernelArgs.wrap_size_arg5s 4yrRNcddlm}|ddlm}|}g}g}g}t|jj D]}t |tr|jd}|j} tjj|} || } |j| d| |j|j|| |j| d|jj!D]\}} ||jvrtjj|} || } |jd| d| |j|j|| |jd| d|j"j!D]\}} ||jvst | tr%tjj|} || } |j| d| |j|j|| |j| d|j$j!D]\}} |jd|d| |j|j'||jd|tjj(sltjj(j+||j,rJd |||fS) NrD) INDEX_TYPE)rrz* *zconst  zWorkspace not supported on CPU )rrrr/rrr>r}rvrr5rrrrrrrrr wrapper_codeensure_size_computedr) rddtype_to_cpp_typerr call_argsarg_defs arg_typesrouterinnerr cpp_dtype maybe_inners rP cpp_argdefszKernelArgs.cpp_argdefs8s *  $ / ,   t33::<= .H(J/((,E''EGG%%e,E)%0I OOykE73 4   T..ue< =    {!_ - .!..446 4LE5,,,GG%%e,E)%0I OOfYKr%9 :   T..ue< =   vi[2 3 4#'"5"5";";"= . E;,,, ; 0SGG%%e,E)%0I OOykK=9 :   T..ue< =    {!_ - .!MM//1 ALE5 OOfZL%9 :   T//6 7   vj\2 3ww##$$99%@  A &&I(II&I--rRc g}g}g}g}t|jjD]}t|tr|j t |j|j |jd|j tjj|jd|j t|j|jdtjj|jdt|jj|j jD]\}}||jvst|tr%|j t ||j ||j tjj||j t||tjj||j"jD]\}}|j t ||j ||j t%||j t'||tjj(stjj(j+||j,D]m}|j t |j|j |j.|j ||j |j0o||||fS)Nr)rarr)r/rrr>r}rrxrrvr5rrrrrrrrr?rrrrrr) rdrrrprecompile_argsrrrrs rPpython_argdefszKernelArgs.python_argdefshs#%! ! /1t33::<= H(J/ OOGH$7$78 9   X11"5 6   QWW..x/C/CB/GH I  " "!,,#//3''++H,@,@,DE  "    $ $ &(;(;(A(A(C  LE5,,, 5*0M OOGEN +   U #   QWW..u5 6  " " ''++E2  !MM//1 ALE5 OOGEN +   U #   T%[ )  " "75%#8 9ww##$$99%@  A&& (C OOGCNN3 4   S^^ ,  " "3 '   SYY '  ( OY>>rRc#Kt|jjD]}t|tr|j D]}|t jjvs|t jjvr<||jvr|j||jf||jvsvtt|j||jfywrh)r/rrr>r}rvr5rr[rZrrrr ru)rdrothers rPaliaseszKernelArgs.aliasesst33::<= UH(J/!-- UQWW777 ; ;;D...,,U3X5H5HHHD///sD$7$7$>?ATATTT U Us B9C,<0C,ct|jj|ttxr.t|j j|ttSrh)r>rrrr}rrs rP is_removedzKernelArgs.is_removedsK    # #D' 2J N--11$@*M NrRclt}t|jjD]1}t |t r|j |jd3|jjD]5\}}||jvst |t r%|j |7|S)Nr) rr/rrr>r}rrvrr)rd live_outsrrrs rPlive_output_bufferszKernelArgs.live_output_bufferss%/\ t33::<= 4H(J/ MM(..r2 3 4!//557 !LE5,,, 5*0M MM%  !rR)rrurz6Union[dict[_T, Union[str, RemovedArg]], dict[_T, str]]rarGrrrurqrt)rar rrrr')rrurrurrrs)rrrrrrztuple[str, int])rrrrru)rarurcrrrru)rarrrru)rrz Iterator[str])rarurrr)rrurrrrru)r SymbolLikerrrurh)rz Optional[dict[torch.dtype, str]]rrz&tuple[list[str], list[str], list[str]])rrz?tuple[list[ArgName], list[str], list[KernelArgType], list[Any]])rrzIterator[tuple[str, str]]r)rrzOrderedSet[str])rwrxryrrrrrrrrr%rrrrrrrrrrrrr}rRrPrrps  E    5  ,, @B48'!R8 7 2EI..!A.. /..`/? H/?b UN rRrcZeZdZdZ d d fd Zd dZd dZd dZd dZd dZ xZ S)rRaDA CSEVariable is just a name for an expression but it is useful to be able to annotate them on a backend dependent basis. To do so, the backends can simply overload `Kernel.create_cse_var` The "CSEVariable.update_on_args" method gives you a hook for annotations See example of TritonCSEVariable in triton.py ct|t|tsJt |||_||_d|_||_yr) rrr>rr?rabounds use_countr)rdrarrrs rPrzCSEVariable.__init__sE &+.<V <.   rRc|jSrhr\ris rPrzCSEVariable.__str__s yyrRc,t|jSrh)hashraris rP__hash__zCSEVariable.__hash__sDIIrRcXt|txr|j|jk(Srh)r>rRra)rdrs rP__eq__zCSEVariable.__eq__s!%-I%** 2IIrRcyrhr})rdrarmrs rPupdate_on_argszCSEVariable.update_on_argss rRcN|jjd|jdSrQ)rrwraris rPrzCSEVariable.__repr__s$..))*!DII=::rRrh)rarurValueRanges[Any]rrrt)rrr)robjectrrr)rarurmr rr rrrs) rwrxryrzrrrrrrrrs@rPrRrRsH(,  ! % J ;rRrR AugmentedKeyT)default)boundr.c>eZdZdZ d ddZddZddZddZddZddZ dd Z dd Z dd Z e jd d dd  ddZe jdf ddZe jdf ddZy)CSEz Common subexpression eliminationNc||_||_i|_||_|xsi|_|xsi|_|xst j|_t|_ |xsi|_ yrh) rrY_cache name_prefix store_cachereduction_cache itertoolsriter_buffer_idsrinvalidated_stores varname_map)rdrrYr iter_buffersrrrs rPrz CSE.__init__sm  FH &ALARPR  !r 6B5VY__EV3=<7B7HbrRc6g|jjD]2\}}||vs |j|=|jj|4|r9|jjDcic] \}}||vs ||c}}|_yi|_ycc}}wrh)rrrrr)rd keep_varsratmprrs rP invalidatezCSE.invalidates44++1134 2ID#)#$$T*''++D1 2 ,0KK,=,=,?RDAq1 >1a4RDKDKSs 1 B>Bc t||j|j|j|j|j |j |jS)N)rrYrrrrr)r?rrYrrrrrris rPclonez CSE.clonesPtDz;;;;((--(((( 00  rRc|j}t|j|_t|j|_t|j|_|S)zNReturn a copy of using ScopedDict so changes to *_cache aren't visible in self)rr*rrr)rdnew_cses rP scoped_copyzCSE.scoped_copysH**,#DKK0",T-A-A"B()9)9:rRc"tt|S)z@Override this method to augment cache key with backend specifics)r rrd cache_keys rP augment_keyzCSE.augment_key$sM9--rRc@||j|j|<yrhrr)rdrrs rPputzCSE.put(s36 D$$Y/0rRc<|j||jvSrh)rrrs rPcontainsz CSE.contains+s *dkk99rRcX|jj|j|dSrh)rrrrs rPtry_getz CSE.try_get.s"{{t// :DAArRc>|j|j|Srhr rs rPrzCSE.get1s{{4++I677rRT)rrb assignmentrc t|tr |j}|s|sJt|trE|jj ||_|xj dz c_tt|St|tr|j}n1t|tr |j}nt|tsJ|}|j|}|s|j||}|j!|||rt"j$j&r+t"j$j&j)|dt|trP|r |j+|j,|d|j/||j+|j0|St|trM|sJ|j+|j3|j,|d|j|j0|S|r |j,|d||j0} n||j0} |j+| |rPt4j6j8st4j6j:r|t=dk7r t?||||S|jj ||_|xj dz c_|S)NrDT) only_oncez =z = rY) r>r2rcrRrtightenrr rr(getvaluer%rarurnewvarr r5rZ current_nodecodegen_originating_inforrsplicerYrerrrrr'r) rdrrrrbrrrrras rPgeneratez CSE.generate4s\ dH %::D "" dK (++--f5DK NNa N. . n - I . / IdC( ((Ill9%++fe,C HHY $88((HH))BB$CdN3!((DKK=R)@AMM$'$$T[[1: 9 &67%%:$$$++se3tyyk$++'WX4 -""&++se3tfT[[MJ"& }5$$T*#"//KK%22JJ!-/1U:#FC7  ++F3CJ MMQ M rRc|jt|j}tjj |||}||j |<|Srh)rrrr5rZcreate_cse_varr)rdrrvar_namers rPrz CSE.newvar|sR &&'T-A-A(B'CDhh%%h>%(" rRctj|jvfdtjj ||}||j<|S)NcdS)Nzduplicate name: r}r\srPr_zCSE.namedvar..s4DTF2KrR)r _check_valuerr5rZr)rdrarrrs ` rPnamedvarz CSE.namedvarsS  (( (*K hh%%dFE:!$ rR)rrrNNNN)rrurYrurrurzOptional[itertools.count[int]]rz.Optional[MutableMapping[str, CSEVariableType]]rz|jj|||yrh)r(__exit__)rdexc_typeexc_valexc_tbs rPr-zCodeGen.__exit__s   7F;rRrqr"r.r r/r r0r rrrs)rwrxryrr+r-rrs@rPr%r%s1 &  %D D sAAAA AAc#K||}|dux}r t}|j}|j}|j}|j}||_||_||_|j |_ d||_||_||_||_|r |rJdyy#||_||_||_||_|r |rJdwwxYww)Nz$unexpected store inside swap_buffers)r(r8r9r:r=r) rdlbcbsbdisallow_storesr8r9r:r=s rP swap_bufferszKernel.swap_bufferss :B Dj (? (!B ,,hh   ??$ F DJ"DL DKDHEEEv2 DJ"DL DKDHEEEv2sA/C 2B6)C *C  C ctrhrrzs rPrz Kernel.loadrrRc|j} |j|_|j||||_S#||_wxYw)z+A load the depends on an index we have read)r8r9r)rdrarzrIs rP indirect_loadzKernel.indirect_loads8  DJ99T5)DJDJs "8 Actrhrr~s rPrzKernel.store_reductionrrRctrhrr|s rPrz Kernel.store "!rRctrhrrs rPrzKernel.reduction "!rRctrhrrs rPrz Kernel.scan s "!rRctrhrrs rPrz Kernel.sortrXrRctrhrris rP var_rangeszKernel.var_ranges rrRct)z3 See [Note: Inductor bucketize op] rrs rPrzKernel.bucketize#s "!rRctrhrris rPassert_functionzKernel.assert_function2s!!rRc vt|tr t|}t|tsJt||t|tsJ|t|tsJ|r|rd|d|d|d|d }|d|d|}n|r |d|}|}n |sJ|d|}|}|r d|d|d}|jd|d|dS) NrHz <= z) & (z < rz) | ~(z, "index out of bounds: z"))r>rRrur?r_)rdrrvrwmaskr> cond_prints rPindirect_assertzKernel.indirect_assert6s c; 'c(C#s#.T#Y.#} 5# 666} 5# 666 UugT#eC5E7! ?  %%a&:&:4&@A rRcH|jt| |||yrh)remove_kernel_local_buffersrr-)rdr.r/r0rs rPr-zKernel.__exit__es  ((* 7F3rRctjjsytfd|jD}t|jD]N}||j vs||j jvs+j||s>j|PD]}||j jvrw|j j|}t|trEtfd|jD}|r|j||j j||j#|y)z Any buffers that are both created and have a last use in the same kernel can be removed. Note that V.graph.scheduler can be None when codegening triton template kernels. Nc3tK|]/}|jvrj|j1ywrh) name_to_bufdefining_op_name)rrrs rPrz5Kernel.remove_kernel_local_buffers..ts;& i+++  ! !# & 7 7 9& s58c3&K|]}|v ywrhr})rrnames_to_removes rPrz5Kernel.remove_kernel_local_buffers..sKaQ/1KrW)r5rrrr?r>rmr$can_buffer_be_removed_through_fusionrrr>r}rrvremove_inplace_bufferr[ remove_buffer)rdfused_node_namesrarr`rsrs @@rPrmz"Kernel.remove_kernel_local_buffersis%GG%%  %& ..&   ,6<++ *DD222 7 77BB* ##D) *$ )Dtyy000ii//5c:.K3??KK..t4''++D1""4( )rRctjd|t|jj|<|j j |y)Nzremove_buffer(%r))rrNrrmrrYrrs rPrvzKernel.remove_buffers; %t,)0   &   &rRctjd|t|jj|<|j j |y)Nzremoving_inplace_buffer(%r))rrNrrmrrYrrs rPruzKernel.remove_inplace_buffers9 /6*1 !!$'   &rRc t|ttfr|Dcgc]}|j|c}Stj j j|}t|jd}|Dcic]W}t|tjtjtjfr||jj!|Y}}t#||Scc}wcc}w)Nc|jSrhr\)ss rPr_z(Kernel.rename_indexing..s !&&rR)r)r>listtuplerename_indexingr5rrrsorted free_symbolsrr UNBACKED_INTSIZEPRECOMPUTED_SIZErmrr-)rdrzrsorted_symbols replacementss rPrzKernel.rename_indexings edE] +5:;D((+; ;  ))%0 2 28HI$  %%II)) tyy~~a    %..<  s C%;AC*ct|i|Srh)rR)rdrmrs rPrzKernel.create_cse_varsD+F++rRcZ|y|jj|jS)zC Returns arg name of a given input or output node. N)rmrr)rdrs rPrzKernel.arg_names' <yy!!$--/22rR)NT)rmzOptional[KernelArgs]rFrrrrs)rrCrrrpr)rLr(rMOptional[IndentedBuffer]rNrrrrprarurzrrrrRrarurzrrcrRrrrsrh rarurzrrcrRrXr4rrrs rrrrrr3rc+Union[CSEVariable, tuple[CSEVariable, ...]]rrrrrrzUCallable[[tuple[CSEVariable, ...], tuple[CSEVariable, ...]], tuple[CSEVariable, ...]]rtuple[CSEVariable, ...]rrr rrrrrrrrrrr)rrzdict[sympy.Symbol, sympy.Expr]rrRrrrrRrrrrrrrzOptional[CSEVariable]rrrRrt) rzUnion[CSEVariable, str]rvrrwrraz!Optional[Union[CSEVariable, str]]rrrur)rzrrrrur"r1rqrarurrrs)rzz;Union[list[sympy.Expr], tuple[sympy.Expr, ...], sympy.Expr]rrr)rmr rr rrrR)rr>rrr)#rwrxryr4r{rYr5rrqrrrJrPrrSrrrrrr\rr|r_rcrxrfr+r-rmrvrurrrrrs@rPr3r3sM3FC9=I6=PT /( /HL /  /D&&(,'+ F F %F % F  FF8""SW"" *"3>"FO" " """& " ; " 5 ""'" " ( " !""'"(" "  " ! ""4804 " "C "& " $ "  "1 ". "  """37 V $VV V 0 V V<""&0"9="FJ" " "4%)N'' /P/ /.,3rRr3c8eZdZUdZded<dZded<dZded <y) rrz ClassVar[str]rNrrrruops_name)rwrxryrr{rrr}rRrPrrs!"C"#'E 'HcrRrcb ddl}|j|jS#t$rYywxYw)Nr) undefined)jinja2 EnvironmentStrictUndefined ImportError)rs rP jinja2_envrs?!!,,"   s " ..ceZdZdZe d d dZed dZe d dZd dZ ddZ ddZ y)KernelTemplatezg Base class for defining kernel templates. Children classes: TritonTemplate, CUDATemplate c|jd}t|dkDr|ddDcgc] }d|z|z|zc}|dddj|Scc}w)NTrDrr) splitlinesrr)source num_indentsindents_spacinglinesras rPindent_except_firstz"KernelTemplate.indent_except_firstsd!!$' u:>INqrAE&4<E!"Iwwu~sAct}|ytj|jd<ddlm} |j |S#|$r}Gdd|}|||d}~wwxYw)Nrr)TemplateSyntaxErrorc(eZdZdfd ZddZxZS)IKernelTemplate._template_from_string..DetailedTemplateSyntaxErrorct||j|j|j|j ||_yrh)rrmessagelinenorafilenameoriginal_error)rdrrs rPrzRKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__init__s>G$&..&--&++&//  +9D'rRcFd|jd}|d|jdz }t|jdr|jjj d}|dz }t d|jdz }tt||jdz}t||D]s}||jdz k(rN||dzd ||dz }t|jd s=|d d |jjdz zzd zz }c||dzd||dz }u|S)NzError in template at line  zError message: rz Context: rrrDz: --> columnz rz^ z: ) rrrrrsplitmaxminrrlr)rd error_inforstartendrKs rPrzQKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__str__sA#=dkk]"!MJODLL>"DDJt22H= $ 3 3 : : @ @ F"l2 #At{{Q 7!#e*dkkAo>!&uc!2 KA DKK!O3 *QveAhZr.J J #*4+>+>#I$.(/*-1D1D1K1Ka1O*P)Q*/)0%&J !+QveAhZr.J J  K&%rR)rrrrrsrt)rwrxryrrrrs@rPDetailedTemplateSyntaxErrorrs 9&rRr)rrrfiltersrr from_string)renvrers rP_template_from_stringz$KernelTemplate._template_from_stringsjl ;-;-O-O )*.# 8??6* *"! 8 &.A &>.a0a 7C! 8sAA!AA!ctjjt|tt fr.|Dcic]!}|j |j#c}n |j |jidfd }|Scc}w)Nc>j|}||S|Srh)r)rar_get_dtype_reallookups rPrz1KernelTemplate._fake_get_dtype..get_dtype s'ZZ%F! "4( (rR)rarurrr)r5rrr>r}r~r) fake_outsrrrrs @@rP_fake_get_dtypezKernelTemplate._fake_get_dtype sr''++ i$ /AJK#cllncmmo5KF((*I,?,?,ABF ) Ls&B c||_yrhr\rs rPrzKernelTemplate.__init__% s  rRc  |j|jdi|y#t$rQ}tj d|t |tj tjk|cYd}~Sd}~wwxYw)a% Maybe generates a new ChoiceCaller and appends it into existing choices. Returns None if success, otherwise returns the error. choices: A list of ChoiceCallers. kwargs: Additional kwargs to be passed to self.generate() to generate a new ChoiceCaller. Nz3Cannot Append Choice: %s. KernelTemplate type is %s) stack_infor}) rrrrinfor?getEffectiveLevelrLINFO)rdchoicesrrs rPmaybe_append_choicez"KernelTemplate.maybe_append_choice( sn  NN=4==262 3"  HHET 002W\\A   H s!$ A>AA93A>9A>c t)zM Generates a ChoiceCaller instance from the given arguments. r)rdrs rPrzKernelTemplate.generate? s "!rRN))rrurrrrrrru)rrurrr )rzUnion[list[Buffer], Buffer]rrzCallable[[str], torch.dtype]r)rrvrr rrzOptional[NotImplementedError])rr rrr<) rwrxryrzrrrrrrrr}rRrPrrs >?"%8; *8*8X. %" ,/ &."rRrc"eZdZdZdfd ZddZddZ d ddZ ddZddZ ddZ d ddZ dd Z dd Z dd Z dd Z d dd ZxZS)rjcbt|ddlm}||_||_||_y)NrValueRangeAnalysis)rrrr vr_analysisrZparent_handler)rdrZrrrs rPrzCSEProxy.__init__J s+ /-/ ,rRc |jgi t|ji}t}t d dk(rdk(r |j nodk(rRdk(rMt jjjjtjdj ndvrt|}|i dvr Jd d  fd }tj||S) Nmaskedr[rY)r[rYre)r[rYrctttfr n} dz dk(r#t|tr|j||_t j jjt j j|}|jtjjstjjr)|Jtt j j|||S)NrDrY)rr)r>r}r~rRrr5rZr=rr9rrrrrr) r var_dtypecsevarrmrrrrar output_idxs rPdo_csez!CSEProxy._default..do_csek s lT5M:Z(!  !OJ%Jq+$>177?#XX\\**  " +F  ! !$f 5##??&&>> ,,,AHH,,fi@MrR)rrurrrR)_bound_variablerrr!r'rr5 interpreterrrrrrpytreetree_map) rdrarmrrc dtype_handlerdtype_oprrrrrs ``` @@@@rP_defaultzCSEProxy._defaultR s %%%d>#''e 0 0}d3H#T4V4L ' '+ ++   >vu--rRcz ddlm}ddlm}ddlm}t tj|rtjSt tj|rtjStjj j|k(r|jjt |jjts$Jt!|jj|jjj# tjSt$j&rjt)||r^t+ fddDrtjS|rJd d}t-t/||}t1|j2||StjS) z If the variable comes from an FX node, we forward the bound we have already computed Else, if the variable when codegen'ing another op, we try to compute its bounds rr)TritonTemplateKernelrD)CUDATemplateKernelc3:K|]}|jvywrh)r)rr|fx_nodes rPrz+CSEProxy._bound_variable.. sV11&Vs) set_indirectrrct|tr |jSt|tjr t |S|Srh)r>rRrrrrrs rP arg_to_boundz.CSEProxy._bound_variable..arg_to_bound s2a-88O5::.&q>)HrR)rr rrr )rrselect_algorithmrcuda.cuda_kernelrr>r5rZrr#rrrrBdictr?rrcompute_all_boundsrrXr}rrr) rdrarmrrrrr arg_boundsrs @rPrzCSEProxy._bound_variable sP 0;8 ahh 4 5&&( ( ahh 2 3&&( (--,, >>T !dkk&@&@&Ldkk88$?  **B ?;;--11';;N;N;PQ Q  & &73Et+LV0UVV"**,,  : c,56J274++T2J? ?""$$rRc$t|trtj|}t|tjsJt ||f|j jdkr|rtj|tj|tj}|j jdk\r0tj|d}tj|||}n|}t!j"}|j t!j"k7rt|tj$r|j t!t& dz}t!|j|z|j|z}|j jdk\r"|j t!dt&z} || z}|j(j*j-|j(j.||}|j0j3|||} t5|ro|j jdk\ } t|tj$ xs|j j|k } |j(j7| || | | S)Nrr)r)r>rrrnrr?rrvr0rrrlongrwltr<rr#NumberrrZr=rr9rrrr&rx) rdrrrprqstmr new_bounds neg_boundspos sympy_var assert_lower assert_uppers rPrrzCSEProxy.indirect_indexing s dC ==&D$ +?d4j$-??+ ::  a ggc3>>$ #CD::##q(QB))BS1C%,,.Jzz[0022z$ 7U!ZZ+vgr*BB ($$t+Z-=-=-D ::##q(**{1f'==C!+c!1J++//**4;;+>+>J*WC''99#tUK 5 ! # 0 0A 56L)$ ==   4'BL KK $ $YlL QrRc>|jj||||Srh)rZrxrus rPrxzCSEProxy.check_bounds s{{''dE5AArRc||jjjvr)tjjj |t |tjr|jj||S|jjj}||vr||S|jj||}|jdk(r|jxjdz c_ |Sr)rZr=rr5r>rrrTMPrSrrrr;)rdrarzrouts rPrz CSEProxy.load s 4;;??55 5 HH & & * *4 0 udhh /;;,,T59 9kkoo11 ; t$ $kktU+ ==A  KK A %  rRcl||jjj|<|jjr{|tj j vr^|jjj|}|jD]%}||jjj|<'yyyrh) rZr=rrr5rname_to_buffer get_output get_mutations)rdrarcr other_names rP_update_store_cachezCSEProxy._update_store_cache s,1 ##D) ;; # #0F0F(F++**55d;C!//1 @ :? ++J7 @)G #rRc|jjj|||j|||tj j vr |jj||||yy)N)rX)rZr?rrr5rrYrr|s rPrzCSEProxy.store sc &&**40 <  $ $T5 1 qww.. . KK  dE5t  < /rRc|jjj||j|||tj j vr|jj|||Syrh)rZr?rrr5rrYrr~s rPrzCSEProxy.store_reduction sZ &&**40   u- qww.. .;;..tUEB B /rRc||jxjdz c_|jj||||Sr)rZr<rrs rPrzCSEProxy.reduction s4 !!Q&!{{$$UI~uMMrRc<|jj|||Srh)rZrrs rPrz CSEProxy.scan! s{{ F;;rRc>|jj||||Srh)rZrrs rPrz CSEProxy.sort, s{{ CCrRc D|jj|||||||S)a [Note: Inductor bucketize op] Inputs: ------- values: the values to be bucketized. boundaries: a tuple containing (a) the name of the boundaries tensor (which must be sorted, unless the sorting tensor is present), (b) the length of the tensor in the last dimension (i.e. the length of one set of boundaries), (c) the number of elements in the underlying storage (i.e. the length of the flattened tensor, ignoring striding), and (d) the stride of the tensor in the last dimension. boundary_indices: indices into a flattened version of the boundaries tensor, of the same size and shape as "values". Each index points to the first element in the set of boundaries to be used for the corresponding value. indexing_dtype: the dtype to use when indexing into the boundaries tensor. This must be int64 or int32. This additionally specifies the dtype of the return value. right: see "Details" below. sorter: an optional tuple containing (a) the name of an optional sorting tensor, used to access unsorted boundaries without reordering the boundaries tensor, and (b) the stride of the tensor in the last dimension. The values in the sorting tensor are used as indices into the *last* dimension of the boundaries tensor, with all other indices matching. The size of the sorting and boundaries tensors must be equivalent. sorter_indices: must be present if the sorting array is present; see "boundary_indices" for the equivalent definition for the boundaries tensor. Output: ------- The buckets each value belongs in, within a given set of boundaries. 0 indicates a position before the first boundary, and len(boundaries_set) represents a position after the last boundary. Details: -------- Given a value and a set of boundaries, calculate the bucket that each value belongs to. This works differently in 1-D and N-D cases. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [0, 4, 4, 8], right=True return = [[ 0, 1, 1, 1], [1, 3, 3, 4]]. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [[0, 4], [4, 8]], right=True return = [[ 0, 1, 1, 1], [0, 1, 1, 2]] Note that in the N-D boundaries case, the shape of "values" and "boundaries" must match in every dimension _except_ the last. When right == False, bucket i refers to range (boundaries[i], boundaries[i+1]]. When right == True, bucket i refers to range [boundaries[i], boundaries[i+1]). Boundaries must be non-decreasing, or a sorter must be provided which would re-index offsets in a non-decreasing order (e.g. the second output of torch.sort(offsets)). Otherwise, the result is undefined. )rZrrs rPrzCSEProxy.bucketize5 s1L{{$$         rR)rZz Kernel[Any]rzOpsHandler[Any])rarurmztuple[Any, ...]rzdict[str, Any]rrr )rarurmr rr rrrr) rrRrrrprrqrrrrrr)rarurcrRrrrsrhrrrrrrr)rwrxryrarrrrrrxrrrrrrrrrrs@rPrjrjG s D-8.t+%b / /%/ /  /  /bBB&0B9=BFJB B "@SW== *=3>=FO= =CNNN& N ; N 5 N <' <  <( < ! <D'D(D D  D ! D 4804N N CN & N $ N  N 1N .N  N rRrj)rOrurrrsr) rrur+rr,rr-rr.Optional[CustomGraphModulePass]rrrs)rUnion[torch.device, str, None]rrzOrderedSet[BackendFeature])rr rEr1rrr)rrurrzOptional[SchedulingConstructor])F)rrurIrrrr)rrurrr rq)rzSequence[sympy.Expr]r{r r|r rrr)rrurrrrrs)rrurrr)rrurmr rr rrr)rr(rrrrrrrs)rJrurrrr}r)rrr ) __future__rr]rq dataclassesenumrrrLrrr_rer[abcrrrrrtypingr r r r r rrrrtyping_extensionsrrrrtorch.fxtorch._prims_commonr torch.utilsrrtorch.utils._ordered_setrtorch.utils._sympy.numbersrtorch.utils._sympy.printersr_PythonPrintertorch.utils._sympy.symbolrrrtorch.utils._sympy.value_rangesrrrrr dtype_propagationr! ops_handlerr"r#utilsr$r%r&r'r(r)r*r+r,r-r.r/ virtualizedr0r1r2r3r4r5collections.abcr6r7r8r9custom_graph_passr:rr;r<r=r>rr@rrArBrCrorFrGrr?rrurrrC_logginggetArtifactLoggerrwrJ getLoggerrrQ dataclassrTrrrrrrrrr KernelArgTyperr{rr(r)r/r1rArFr@rKrMcacher=r}rrbfloat16rfloat16rr float64int8int16rrruint16ruint64rrrrrcompile IGNORECASErSrMrOrr INT_TO_FLOATrr]r_rgrurxr}rrrRrrr~rReductionCacheKeyrr%r3rrrrjrs0rPr4s "    #   , ?)/-GOOD:;    QPBB$9>>$DD- B$hy&9%:N%JK23sELL()JF~~//*E g!=  >/ /(  C  Ta=aaH** ###   !!! === lIw8H,VW ,.).5"5"p:<6;DFAF8@D:> 7  7, 7/ 7!= 7 8 7  7 &T &3 *33$3 *35C3 3U &+   " ! V KK\U U$U U U; ;&7; ; , NNEKK MM5;;> JJ MM MM JJ KK KK KK KK LL LL LL    u >:,' ' '' 'T: :!0:9D: :2aBaBH%N%S1S1l"rzz";2==Q D;#%5z#D;N  - - - 6:`6 ;HH -  `6 ;HH /-  `6;HH /- `6$;HH /- %`60;HH /- 1`6< ;HH *)   =`6L ;HH (0   M`6X ;HH 66>   Y`6h ;HH 1  i`6r ;HH 2  s`6|;HH 1 }`6F;HH 2 G`6P;HH %8$  Q`6^ ;HH &%   _`6j;HH %8  k`6v ;HH &  w`6@;HH + A`6L%;HH 88 ) M`6X%;HH 88 ) Y`6d%;HH 8 )e`6n%;HH 8 )o`6z ;HH '  {`6D ;HH (  E`6N;HH c  O`6^,;HH ? 0_`6h,;HH ? 0i`6t&;HH 9 *u`6~ ;HH *  `6H);HH D -I`6R);HH D -S`6\);HH D -]`6f);HH D -g`6p(;HH C ,q`6z$1;HH L 5${`6D$1;HH L 5$E`6N$1;HH L 5$O`6X$1;HH L 5$Y`6b';HH B +c`6l(;HH C ,m`6v(;HH C ,w`62`F -#-">*Z  NNN ,JJZ !;!;H5 +; T  k5c!1223 5e'/=0 1eP < <\3Wgo.\3~  p"p"f| ~| O8s7 e