
    rh:                   	   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z" dd	l#m$Z$m%Z% ddl&Z&ddl'Z&ddl(Z&ddl)m*c m+Z, dd
l-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8 ddl9m:Z: ddl;m<Z<m=Z= ddl>m?Z? ddl@mAZA ddlBmCZD ddlEmFZFmGZG ddlHmIZI ddlJmKZKmLZL ddlMmNZN ddlOmCZC ddlPmQZQ ddlRmSZS  e	j                  eU      ZVe&j                  j                  ZXe&j                  j                  ZYeZZe"eZe&j                  j                  f   Z\ G d de       Z] G d d e       Z^ G d! d"e       Z_ e!d#      Z`e"e&j                  j                  j                  ecf   Zd G d$ d%      Ze ee       Zf	 d	 	 	 	 	 	 	 dd&Zg G d' d(      Zh G d) d*ei      Zje"ehejf   Zkdd+Zl G d, d-      Zm G d. d/e      Zn G d0 d1en      Zo G d2 d3en      Zp G d4 d5en      Zq G d6 d7en      Zr G d8 d9en      Zseted:f   Zu G d; d<es      Zv G d= d>ev      Zw G d? d@ev      Zx G dA dBev      Zy G dC dDes      Zz G dE dFez      Z{ G dG dHez      Z| G dI dJez      Z} G dK dLen      Z~ G dM dNen      Z G dO dPen      Z G dQ dR      Z G dS dTe       Zej                   G dU dV             Zej                   G dW dXe             Zej                   G dY dZe             Zej                   G d[ d\e             Zdd]Zdd^Z	 d	 	 	 	 	 	 	 	 	 dd`Zeddadd_f	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddbZ eA       Zdcedd<   	 	 	 	 	 	 	 	 	 	 	 	 ddeZ ee      j                   dfz  dgz  Zg Zdhedi<   eddad_f	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddjZ eDj(                  d_k      	 	 d	 	 	 	 	 	 	 	 	 	 	 ddl       Z	 	 d	 	 	 	 	 	 	 	 	 	 	 ddmZefd_dn	 	 	 	 	 	 	 	 	 ddoZefd_dn	 	 	 	 	 	 	 	 	 ddpZddqZ ej4                  dr      ZddsZddtZdduZddvZddwZddxZ G dy dz      Zdd{Z	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 dd|Z e&jJ                         d}dd~	 	 	 	 	 	 	 	 	 dd       Z e&jN                         dd       ZddZddZddZddZddZ eA       Zdced<   	 d	 	 	 	 	 	 	 ddZddZddZy)a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)
Collection	GeneratorIterableMappingSequence)Path)AnyCallableNoReturnOptionalProtocolTypeVarUnion)SelfTypeIs)enable_python_dispatcher)counters)is_integer_dtype)unset_fake_temporarily)make_fxstatically_known_true)	_get_attr)immutable_dictimmutable_list)GraphTransformObserver)
OrderedSet   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_typec                       e Zd ZU ded<   ddZy)SearchFnstr__name__c                     y N selfargskwargss      r/var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/torch/_inductor/pattern_matcher.py__call__zSearchFn.__call__]           Nr7   r   r8   r   returnr   )r1   
__module____qualname____annotations__r:   r4   r<   r9   r/   r/   Z   s
    M=r<   r/   c                      e Zd ZddZy)	ReplaceFnc                     y r3   r4   r5   s      r9   r:   zReplaceFn.__call__a   r;   r<   Nr=   r1   r?   r@   r:   r4   r<   r9   rC   rC   `   s    =r<   rC   c                  $    e Zd Z	 	 	 	 	 	 	 	 ddZy)TraceFnc                     y r3   r4   )r6   fnr7   r8   s       r9   r:   zTraceFn.__call__e   s    "r<   N)rI   zUnion[SearchFn, ReplaceFn]r7   r   r8   r   r>   torch.fx.GraphModulerE   r4   r<   r9   rG   rG   d   s'    #,#58#DG#	#r<   rG   Tc                      e Zd ZddZy)Multiplec                2    dt               vs| t        u sJ y y )NMULTIPLE)globalsrO   r6   s    r9   __init__zMultiple.__init__q   s    *dh.>>>.>*r<   Nr>   None)r1   r?   r@   rR   r4   r<   r9   rM   rM   p   s    ?r<   rM   c                   ddl m}m} t        j                  j
                  ry| j                  dg       j                         }|j                   ||||j                               | j                  d |j                  j                         D               || d<   y | j                  d |j                  j                         D               y )Nr   )
NodeSourceNodeSourceAction	from_nodec              3  v   K   | ]1  \  }}|t         j                  j                  j                  v r||f 3 y wr3   torchfxproxy_COPY_META_FIELDS.0kvs      r9   	<genexpr>z!_transfer_meta.<locals>.<genexpr>   6      
1EHHNN444 F
   79c              3  v   K   | ]1  \  }}|t         j                  j                  j                  v r||f 3 y wr3   rZ   r_   s      r9   rc   z!_transfer_meta.<locals>.<genexpr>   rd   re   )torch.fx.tracebackrV   rW   r$   traceenabledgetcopyappendREPLACEupdatemetaitems)new_metaold_node	pass_namerV   rW   new_from_nodes         r9   _transfer_metaru   z   s     @
 || ["5::<Z)=M=U=UVW 
 ++-
 	

 !. 
 ++-
 	
r<   c                       e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   	 	 d	 	 	 	 	 	 	 	 	 d fdZedd       ZddZddZ	ddZ
d dZd!dZd"dZ	 	 	 	 	 	 d#dZ	 	 d$	 	 	 	 	 	 	 	 	 d%dZ xZS )&Matcha  
    Represents a successfully matched pattern.

    The `Match` object is returned to represent a successfully matched
    pattern. Included in the Match are the pattern that was matched, the graph
    nodes matched, and any args that were used during the matching.

    The args and kwargs are specific to the type of pattern that was matched and
    provide hints about what was matched.
    PatternExprpattern	list[Any]r7   dict[str, Any]r8   list[torch.fx.Node]nodesz'dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxzOptional[torch.fx.GraphModule]replacement_graphc                    t         |           || _        t        |xs g       | _        |xs i | _        g | _        i | _        || _        d | _	        y r3   )
superrR   ry   listr7   r8   r}   r~   r   r   )r6   r   ry   r7   r8   	__class__s        r9   rR   zMatch.__init__   sO     	$	l
!%r<   c                .    | j                   j                  S r3   )r   graphrQ   s    r9   r   zMatch.graph   s    xx~~r<   c                2   | j                   rwt        | j                   j                               t        |j                   j                               z  D ]-  }| j                   |   |j                   |   k7  s#t        d|       | j                  j                  |j                         | j                  j                  |j                         | j                   j                  |j                          | j                  j                  |j                         y )Nzkwarg mismatch: {})	r8   r!   keysFailedMatchr7   extendr}   rn   r~   )r6   otherkeys      r9   r   zMatch.extend   s    ;;!$++"2"2"45
5<<CTCTCV8WW A;;s#u||C'88%&:C@@A 			$

%++&5<<(EMM*r<   c                f    | j                   rt        | j                         g| _         | S g | _         | S r3   )r7   tuplerQ   s    r9   bundlezMatch.bundle   s.    *.))U499%&	 :<	r<   c                <    d| j                    d| j                   dS )NzMatch(..., , )r7   r8   rQ   s    r9   __repr__zMatch.__repr__   s    TYYKr$++a88r<   c                    | j                   }t        | j                        D ]-  }|j                  r|j                  r|j                  |       / y r3   )r   reversedr}   _erasedusers
erase_node)r6   r   ns      r9   erase_nodeszMatch.erase_nodes   s>    

$**% 	$A99QWW  #	$r<   c                    | j                   j                  D cg c]  }|| j                   j                  |   nd ! c}S c c}w r3   )r   outputspattern_to_node)r6   ps     r9   output_nodeszMatch.output_nodes   sF     XX%%
 -.MTXX%%a(tC
 	
 
s   $A c                B    t        d | j                         D              S )Nc              3  &   K   | ]	  }|s|  y wr3   r4   )r`   r   s     r9   rc   z$Match.output_node.<locals>.<genexpr>   s     8!aA8s   )nextr   rQ   s    r9   output_nodezMatch.output_node   s    8t002888r<   c                \    t         j                  | | j                  j                  ||       y r3   )ReplacementPatternEntryreplace_with_graphr   r   )r6   r   r7   s      r9   r   zMatch.replace_with_graph   s$     	 22$((.."3T	
r<   c                   ddl m}m} t        |j                  |      r|j                  |j                  nt        j                         }dd}|5  |t        j                  t        |      } || j                        rzi | j                  d   j                  d   \  }	}
i |
}
t        | j                        | j                  }}dfd}t        j                   j"                  j%                  |||f|	|
f       t        j&                  j)                  |fd      } |||      }t        j&                  j)                  |d	       } |||      }t+        |j,                  j                        t+        |j,                  j                        k(  sJ t/        |j,                  j                  |j,                  j                        D ]0  \  }}d|j                  v s|j                  d   |j                  d<   2 n*t        j&                  j)                  |d
       } |||      }t+        | j                        dk(  r@|j,                  j                  D ]'  }t1        |j                  | j                  d   d       ) t2        j5                  | | j6                  j,                  ||       ddd       y# 1 sw Y   yxY w)a	  Replace with a graph generated by tracing the replacement_fn.

        Args:
            run_functional_passes (bool). If we should run passes that
                assume functional IR (like DCE, remove_noop_ops), on the
                replacement graph.

        r   )NullHandlerVNc                H   t        |       dk7  ry| d   }d|j                  vry|j                  t        t        j
                  j                  j                  t        j
                  j                  j                  t        j
                  j                  j                  g      v S )Nr+   Fr   eager_input_vals)
lenro   targetr!   r[   opshigher_order triton_kernel_wrapper_functionalauto_functionalizedauto_functionalized_v2)r}   nodes     r9   !should_propagate_eager_input_valszCMatch.replace_by_example.<locals>.should_propagate_eager_input_vals   s~    5zQ8D!2;;*II**KKII**>>II**AA#  r<   )run_functional_passesr   c                Z    t        | t        j                  j                        r|| <   y y r3   )
isinstancer[   r\   Node)r   valnode_to_vals     r9   recordz(Match.replace_by_example.<locals>.record   s$    !$6,/D) 7r<   c                    |    S r3   r4   )argr   s    r9   <lambda>z*Match.replace_by_example.<locals>.<lambda>(  s    +cBR r<   c                     | j                   d   S Nr   ro   r   s    r9   r   z*Match.replace_by_example.<locals>.<lambda>.      #((5/ r<   c                     | j                   d   S r   r   r   s    r9   r   z*Match.replace_by_example.<locals>.<lambda>A  r   r<   r+   replace_by_examplerq   rr   rs   )r}   r|   r>   bool)r   torch.fx.Noder   r   r>   rT   )torch._inductor.virtualizedr   r   r   	fake_mode
contextlibnullcontext	functoolspartialfwd_onlyr}   ro   r   r7   r8   r[   utils_pytreetree_mapr\   map_argr   r   zipru   r   r   r   )r6   replacement_fnr7   trace_fnr   r   r   contextr   	fake_argsfake_kwargs
match_argsmatch_kwargsr   example_valsgraph_with_eager_valsreplacementrr   new_noder   r   s                       @r9   r   zMatch.replace_by_example   sz    	? q{{K8Q[[=P KK'') 		  D	$,,4I 1< !)-A););<N)O&	;-o+0+;T[[L
0 ##,,Z6K8P  %xx//6RS )1(N%  %xx//6QR&'<lK 066<<=%%++B    +.)//55{7H7H7N7N+ &Hh *X]]:<DMM.=&89	  %xx//6QR&~|D4::!#$**00 A"!"!%A"6 $66	D	 D	 D	s   FJ4CJ44J=)NN)
r   r   ry   rx   r7   zOptional[Sequence[Any]]r8   zOptional[dict[str, Any]]r>   rT   )r>   torch.fx.Graph)r   rw   r>   rT   )r>   rw   r>   r0   rS   )r>   zlist[Optional[torch.fx.Node]])r>   r   )r   r   r7   Sequence[Any]r>   rT   NT)
r   rC   r7   r   r   zOptional[TraceFn]r   r   r>   rT   )r1   r?   r@   __doc__rA   rR   propertyr   r   r   r   r   r   r   r   r   __classcell__r   s   @r9   rw   rw      s   	 
O44	55 )-+/&& & &	&
 )& 
&&  +
9$
9
!/
7D
	
 '+&*i!i i $	i
  $i 
ir<   rw   c                  4    e Zd ZU dZded<   ddZd	dZd
dZy)r   z
    Represents a unsuccessful match.

    The `FailedMatch` object is returned to represent a failure to match a
    pattern.
    r0   format_stringc                f    || _         t        |      dkD  rt        d|       || _        || _        y )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r   r   RuntimeErrorr7   r8   )r6   r   r7   r8   s       r9   rR   zFailedMatch.__init__]  sB    * }#hivhwx  	r<   c                b     | j                   j                  | j                  i | j                  S r3   )r   formatr7   r8   rQ   s    r9   __str__zFailedMatch.__str__h  s(    (t!!(($))Ct{{CCr<   c                     yNFr4   rQ   s    r9   __bool__zFailedMatch.__bool__k      r<   N)r   r0   r7   r   r8   r   r>   rT   r   r>   r   )r1   r?   r@   r   rA   rR   r   r   r4   r<   r9   r   r   S  s     	Dr<   r   c                    t        |       S )z|
    TypeIs cannot act on `self`. Thus this function exists to let mypy
    recognize FailedMatch.__bool__ as a TypeIs.
    )r   )ms    r9   is_matchr   r  s    
 7Nr<   c                  d    e Zd ZU dZded<   ded<   ded<   ded	<   	 d	 	 	 	 	 	 	 ddZddZddZy
)r   zC
    Internal state needed while running PatternExpr._match().
    list[Optional[PatternExpr]]r   z*dict[PatternExpr, Optional[torch.fx.Node]]r   r   r   zlist[NodeOrConstant]exclusive_node_setNc               V    || _         |i n
t        |      | _        || _        g | _        y r3   )r   dictr   r   r   )r6   r   r   r   s       r9   rR   zMatchContext.__init__  s.     %4%<r$BW
"$r<   c                    || j                   v r)| j                   |   |k(  rt        | |      S t        d      S |j                  ||       }|| j                   vsJ |r|nd| j                   |<   |S )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   rw   r   _match)r6   ry   r   r   s       r9   matchzMatchContext.match  sy    d***##G,4T7++"#=>>NN4&d2222201tW%r<   c                    | j                   j                         D ci c]  \  }}|j                         r||| c}}S c c}}w r3   )r   rp   has_multiple_users)r6   ry   r   s      r9   filter_multi_user_patternsz'MatchContext.filter_multi_user_patterns  sL     "&!5!5!;!;!=
))+0@ TM
 	
 
s   Ar3   )r   r   r   z*Optional[dict[PatternExpr, torch.fx.Node]]r   r   r>   rT   )ry   rx   r   NodeOrConstantr>   MatchResult)r>   z dict[PatternExpr, torch.fx.Node])r1   r?   r@   r   rA   rR   r  r  r4   r<   r9   r   r   z  s^     )(??,,
 GK
%,
% D
%
 
% 

%

r<   r   c                  V    e Zd ZdZed	d       Zd
dZddZddZ	 	 	 	 	 	 ddZ	ddZ
y)rx   z+
    Base class for types of patterns.
    c                     y r3   r4   r6   r   r   s      r9   r   zPatternExpr._match  s    MPr<   c                    	 t        | g|j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wNr   )r   r   r  r   r6   r   es      r9   r  zPatternExpr.match  s<    	djj9??dKK 	H	s   '* 	?:??c                     yr   r4   rQ   s    r9   r  zPatternExpr.has_multiple_users  r   r<   c                4    | j                   j                  dz   S )Nz())r   r1   rQ   s    r9   r   zPatternExpr.__repr__  s    ~~&&--r<   c              #  L   K   | |j                   v r|j                   |     y y wr3   )r   r6   r   searcheds      r9   find_anchor_nodeszPatternExpr.find_anchor_nodes  s,      3&&&%%d++ 's   "$c                .    t        || j                        S )z
        Compare two `PatternExpr`s and return true if they are the
        same. Note this is NOT matching a pattern - it is comparing the pattern
        structures (for debugging).
        )r   r   )r6   r   s     r9   
pattern_eqzPatternExpr.pattern_eq  s     %00r<   Nr   r   r   r   r>   r  r   r   r>   r  r   r   r   r   r  OrderedSet[torch.fx.Node]r>   z.Generator[Optional[torch.fx.Node], None, None]r   r   r>   r   )r1   r?   r@   r   r   r   r  r  r   r  r  r4   r<   r9   rx   rx     sH     P P.,,+D,	7,1r<   rx   c                      e Zd ZdZddZy)Argzn
    Capture an arg which will become an input to the handler.  Args are
    passed in depth first order.
    c                     t        || |g      S )N)r7   rw   r	  s      r9   r   z
Arg._match  s    S$dV,,r<   Nr   r  r   r   r>   r  r1   r?   r@   r   r   r4   r<   r9   r  r    s    
-r<   r  c                  (    e Zd ZdZddZddZddZy)	Ignoredz4
    Match an arg, but don't pass it to handler
    c                    t        ||       S r3   r  r	  s      r9   r   zIgnored._match  s    S$r<   c                     y)N*r4   rQ   s    r9   r   zIgnored.__repr__  s    r<   c                     y)Nz	Ignored()r4   )r6   pps     r9   pretty_printzIgnored.pretty_print  s    r<   Nr   r   r(  PatternPrettyPrinterr>   r0   )r1   r?   r@   r   r   r   r)  r4   r<   r9   r#  r#    s     r<   r#  c                  @     e Zd ZdZd fdZddZddZd	 fdZ xZS )

KeywordArgD
    Capture a kwarg which will become an input to the handler.
    c                0    t         |           || _        y r3   r   rR   namer6   r1  r   s     r9   rR   zKeywordArg.__init__      	r<   c                "    d| j                   dS )NzKeywordArg(r   r1  rQ   s    r9   r   zKeywordArg.__repr__  s    TYYM++r<   c                6    t        || | j                  |i      S )Nr8   )rw   r1  r	  s      r9   r   zKeywordArg._match  s    S$		4'899r<   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S r3   typingcastr   r   r  r1  r6   r   r   s     r9   r  zKeywordArg.pattern_eq  5    D%(w!%(DTYY%**-DDr<   r1  r0   r>   rT   r   r   r  	r1   r?   r@   r   rR   r   r   r  r   r   s   @r9   r-  r-    s#    ,:E Er<   r-  c                  L     e Zd ZU dZded<   d fdZd	dZd
dZd fdZ xZ	S )ExclusiveKeywordArgr.  r0   r1  c                0    t         |           || _        y r3   r0  r2  s     r9   rR   zExclusiveKeywordArg.__init__  r3  r<   c                "    d| j                   dS )NzExclusiveKeywordArg(r   r5  rQ   s    r9   r   zExclusiveKeywordArg.__repr__  s    %dii]!44r<   c                    ||j                   v rt        d      S |j                   j                  |       t        || | j                  |i      S )Nzexclusive arg appears twicer7  )r   r   rl   rw   r1  r	  s      r9   r   zExclusiveKeywordArg._match  sH    3)))<==%%d+S$		4'899r<   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S r3   r9  r<  s     r9   r  zExclusiveKeywordArg.pattern_eq	  r=  r<   r>  r   r   r  )
r1   r?   r@   r   rA   rR   r   r   r  r   r   s   @r9   rA  rA    s*     I5:E Er<   rA  c                       e Zd ZU dZded<   ded<   	 d	 	 	 	 	 d fdZeedd              ZddZ	dd	Z
dd
Z	 	 	 	 	 	 ddZddZddZd fdZ xZS )_TargetExprz7
    Base class for filtering match by node.target
    zlist[FnsType]fnszOrderedSet[FnsType]fns_setc                Z   t         |           t        |      st        |t              r|gn
t        |      }|D ]O  t        t        j                  j                        s(|j                  fdj                         D               Q || _        t        |      | _        || _        y )Nc              3  6   K   | ]  }t        |        y wr3   )getattr)r`   overloadrI   s     r9   rc   z'_TargetExpr.__init__.<locals>.<genexpr>  s     PX72x0Ps   )r   rR   callabler   r0   r   r[   _opsOpOverloadPacketr   	overloadsrH  r!   rI  r   )r6   rH  r   rI   r   s      @r9   rR   z_TargetExpr.__init__  s     	}
3(<se$s) 	QB"ejj99:

PPP	Q !#
r<   c                     y r3   r4   rQ   s    r9   opz_TargetExpr.op#  s    r<   c                   | j                   d   }t        |t              s|j                  }t	        | j                         dkD  rd| dS | j                   d   t        t        |d       u rd| S | j                   d   t        t        |d       u rd| S t        | j                   d   t        j                  j                        rt        | j                   d         S |S )Nr   r+   [z, ...]torch.z	operator.)
rH  r   r0   r1   r   rL  r[   operatorrO  
OpOverload)r6   
first_reprs     r9   fns_reprz_TargetExpr.fns_repr'  s    XXa[
*c*#,,Jtxx=1zl&))XXa[GE:t<<J<((XXa[GHj$??zl++UZZ%:%:;txx{##r<   c                    | j                   t        u rd}n"| j                   dk7  rd| j                    d}nd}| j                  j                   d| j	                          | dS )Nz
, MULTIPLEr+   r   r    ()r   rO   r   r1   rZ  )r6   comma_userss     r9   r   z_TargetExpr.__repr__7  s^    ::!&KZZ1_tzzl!,KK..))*!DMMO+<[MKKr<   c                X    t        | j                  t              xs | j                  dkD  S )Nr+   )r   r   rM   rQ   s    r9   r  z_TargetExpr.has_multiple_users@  s     $**h/A4::>Ar<   c                    t         r3   NotImplementedErrorr  s      r9   r  z_TargetExpr.find_anchor_nodesC  s
     "!r<   c                    t        |t        j                  j                        xr2 |j                  | j                  k(  xr t        |      | j                  v S r3   )r   r[   r\   r   rS  extract_targetrI  )r6   r   s     r9   
_match_fnsz_TargetExpr._match_fnsH  sD    tUXX]]+ 5477"5t$4	
r<   c                    | |j                   v xs6 | j                  t        u xs" t        |j                        | j                  k(  S r3   )r   r   rO   r   r	  s      r9   _match_usersz_TargetExpr._match_usersO  s=    CKK -zzX%-4::$**,	
r<   c                    t        j                  t        |      }t        |   |      xrO | j
                  |j
                  k(  xr4 | j                  |j                  k(  xr | j                  |j                  k(  S r3   )r:  r;  r   r   r  rS  rH  r   r<  s     r9   r  z_TargetExpr.pattern_eqV  sf    D%(Gu% *588#*EII%* 

ekk)		
r<   )r+   )rH  z!Union[FnsType, Sequence[FnsType]]r   zUnion[Multiple, int]r>   rT   r   r   r  r   r   r>   r   )r   r   r   r   r>   r   r  )r1   r?   r@   r   rA   rR   r   r   rS  rZ  r   r  r  re  rg  r  r   r   s   @r9   rG  rG    s     
   UV4=Q	    LB""+D"	7"



 
r<   rG  .c                       e Zd ZdZdd	 	 	 	 	 	 	 	 	 d fdZe	 	 	 	 	 	 dd       Ze	 	 	 	 	 	 dd       ZddZddZ	dd	Z
	 	 	 	 	 	 dd
Zd fdZ xZS )_TargetArgsExprzE
    Base class for filtering match by node.{target,args,kwargs}
    r+   )_usersc               j   t         |   ||       t        |      | _        t	        |      | _        t        d t        j                  ||j                               D              r| j                  | _        n| j                  | _        | j                  | j                  | j
                        | _        y )Nc              3  R   K   | ]  }t        |t        t        t        f       ! y wr3   )r   r   r   r   r`   xs     r9   rc   z+_TargetArgsExpr.__init__.<locals>.<genexpr>r  s&      
 q4u-.
s   %')r   rR   r   r7   r   r8   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)r6   rH  rl  r7   r8   r   s        r9   rR   z_TargetArgsExpr.__init__h  s     	f%$K	6l 
__T6==?;
 
  ..DL..DL $TYY Dr<   c                n    g | |j                         }t        |       g|j                         }||fS r3   )rt  r   r   )r7   r8   rt  specs       r9   rw  z_TargetArgsExpr.simple_flatten{  s:     +4*&--/*D	*FKKM*t|r<   c                    t         t        t        t        t        t        idfdt        j                  | |ffd      }t        j                  |      \  }}||fS )Nc                    t        |       }j                  |      }|!t        j                   ||       fd      S | S )Nc                    t        |       v S r3   typerp  type_mappings    r9   r   zF_TargetArgsExpr.pytree_flatten.<locals>.convert_type.<locals>.<lambda>  s    d1g&= r<   is_leaf)r  rj   pytreer   )rp  cls
convert_fnconvert_typer  s      r9   r  z4_TargetArgsExpr.pytree_flatten.<locals>.convert_type  sG    q'C%))#.J% qM= 
 Hr<   c                    t        |       v S r3   r~  r  s    r9   r   z0_TargetArgsExpr.pytree_flatten.<locals>.<lambda>  s    d1g5 r<   r  )rp  r   r>   r   )r   r   r   r   r   r  r   tree_flatten)r7   r8   normalized_args_treeflatrz  r  r  s        @@r9   ru  z_TargetArgsExpr.pytree_flatten  s_    
 E%D*
		  &6N5 

 (()=>
dTzr<   c                   | j                         gt        t        | j                        | j                  j                         D cg c]  \  }}| d|  c}}}| j                  t        u r|j                  d       n-| j                  dk7  r|j                  d| j                          | j                  j                   ddj                  |       dS c c}}w )N=_users=MULTIPLEr+   _users=r]  r   r   )rZ  mapreprr7   r8   rp   r   rO   rl   r   r1   join)r6   ra   rb   r7   s       r9   r   z_TargetArgsExpr.__repr__  s    MMO
tyy!
 &*[[%6%6%89TQ1QCj9

 ::!KK)*ZZ1_KK'$**./..))*!DIIdO+<A>> :s   Cc           
        | j                         gfd| j                  D        | j                  j                         D cg c]  \  }}| dj	                  |        c}}}| j
                  t        u r|j                  d       n-| j
                  dk7  r|j                  d| j
                          d}| j                  j                   d|j                  |       dS c c}}w )	Nc              3  @   K   | ]  }j                  |        y wr3   )r)  )r`   rp  r(  s     r9   rc   z/_TargetArgsExpr.pretty_print.<locals>.<genexpr>  s     4Qbooa 4s   r  r  r+   r  r   r]  r   )rZ  r7   r8   rp   r)  r   rO   rl   r   r1   r  )r6   r(  ra   rb   r7   
joiner_strs    `    r9   r)  z_TargetArgsExpr.pretty_print  s    MMO
4$))4
 7;kk6G6G6IJda1R__Q'()J

 ::!KK)*ZZ1_KK'$**./
..))*!JOOD,A+B!DD Ks    C'c                   | j                  |      r+t        |j                        t        | j                        k7  rt        d||       S | j	                  ||      st        d|       S |j                  }|j
                  }t        |      t        | j
                        k  rddlm} t        |j                        sJ  ||j                  |j                  |j
                        }|t        d||       S |\  }}t        |      t        | j                        k(  rDt        |      t        | j
                        k\  r#|D ci c]  }|| j
                  v s|||    }}n/t        d||       S |D ci c]  }|| j
                  v s|||    }}| j                  ||      \  }}	| j                  \  }
}|	|k7  rt        d|	|      S t        |      t        |
      k(  sJ t        ||       }t        t        j                         |
|      D ]  \  }}}t!        |t"              r3|j%                  ||      }t'        |      s|c S |j)                  |       Jt!        |t*        j,                  j.                        s||k7  stt        d||      c S  |j0                  j3                  |       |j                  |j4                  | <   |S c c}w c c}w )Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionzargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})re  r   r7   r   rg  r8   torch.fx.operator_schemasr  rN  r   rv  rx  rw   r   rr  countr   rx   r  r   r   r[   r\   r   r}   rl   r~   )r6   r   r   _args_kwargsr  normalized_args_and_kwargsi
node_items	node_spec
self_items	self_specr   ry   
child_nodechild_matchs                   r9   r   z_TargetArgsExpr._match  ss   t$DII#dii.(HGtTT  s+2D99		++w<#dkk**DDKK((();TYY*& *1"#KTSWXX!;wu:TYY/CLCDT4T6=RdkkAQq'!*}RGR&@$  /6Jdkk9Iq'!*}JGJ $UG <
I $ 5 5
I	!5y)LL:#j/111#t&))//*;Z&T 		"Aw
';/!ii<,&&%J6*:O"94 		 	
t++		$7 S Ks    KK0KKc              #    K   | |j                   v r|j                   |     y| j                  d   D ]  }t        |t              s|j	                  ||      D ]d  }t        |t
        j                  j                        s(|j                  D ].  }||vs| j                  |      s| |j                  |       0 f  yw)a  
        This is used when we are matching a pattern with multiple outputs.
        There is a partial match (stored in ctx) and we want to walk
        this pattern to find a connection to an already-matched node.

        Yields candidate nodes that `self._match` might like.
        Nr   )r   rx  r   rx   r  r[   r\   r   r   re  add)r6   r   r  ry   
other_noder   s         r9   r  z!_TargetArgsExpr.find_anchor_nodes  s      3&&&%%d++,,Q/ 		3G';/")";";C"J 3J%j%((--@  * 0 0 3x/#t4&*
 (T 2	33		3s   ACACC&Cc                   t        j                  t        |      }t        |   |      xrW | j
                  d   |j
                  d   k(  xr6 t        d t        | j
                  d   |j
                  d         D              S )Nr+   c              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wr3   r   rx   r  r`   abs      r9   rc   z-_TargetArgsExpr.pattern_eq.<locals>.<genexpr>  6      Aq $.a#=Q16I   35r   )r:  r;  r   r   r  rx  allr   r<  s     r9   r  z_TargetArgsExpr.pattern_eq  s    D%(Gu% %%a(E,B,B1,EE  5 5a 8%:P:PQR:ST 	
r<   )
rH  z/Union[torch.fx.node.Target, str, Sequence[Any]]r7   r   rl  zUnion[int, Multiple]r8   r   r>   rT   )r7   r   r8   zMapping[Any, Any]r>   z9tuple[Sequence[Any], Union[_SimpleSpec, pytree.TreeSpec]]r   r*  r  r  r  )r1   r?   r@   r   rR   staticmethodrw  ru  r   r)  r   r  r  r   r   s   @r9   rk  rk  c  s     ()	E<E E %	E
 E 
E& %6	B  %6	B 8
?E1f33+D3	732	
 	
r<   rk  c                      e Zd ZdZdZy)CallFunctionzR
    Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
    call_functionNr1   r?   r@   r   rS  r4   r<   r9   r  r    s     
Br<   r  c                      e Zd ZdZdZy)
CallMethodzW
    Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
    call_methodNr  r4   r<   r9   r  r         
Br<   r  c                      e Zd ZdZdZy)
CallModulezP
    Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
    call_moduleNr  r4   r<   r9   r  r  "  r  r<   r  c                      e Zd ZdZddZy)_TargetExprVarArgsz[
    Matches a call_function node with any arguments which are passed into the pattern
    c                   | j                  |      st        d      S | j                  ||      st        d      S t        ||       }|j                  j                  |       |j                  |j                  | <   |j                  j                  |j                         |j                  j                  |j                         |S )Nfunction_mismatchmultiple_users)re  r   rg  rw   r}   rl   r   r~   r7   r   r8   rn   )r6   r   r   r   s       r9   r   z_TargetExprVarArgs._match/  s    t$233  s+/00#t	t++		$	dii 	$r<   Nr  r!  r4   r<   r9   r  r  *  s    r<   r  c                      e Zd ZdZy)CallFunctionVarArgsr  Nr1   r?   r@   rS  r4   r<   r9   r  r  >  s    	Br<   r  c                      e Zd ZdZy)CallMethodVarArgsr  Nr  r4   r<   r9   r  r  B      	Br<   r  c                      e Zd ZdZy)CallModuleVarArgsr  Nr  r4   r<   r9   r  r  F  r  r<   r  c                  B     e Zd ZdZdd fdZddZd	dZd
 fdZ xZS )ListOfz$
    Matches a repeated pattern
    c                b    t         |           t        |t              sJ || _        || _        y r3   )r   rR   r   rx   ry   r   )r6   ry   r   r   s      r9   rR   zListOf.__init__O  s,    ';///r<   c                N    | j                   j                   d| j                   dS Nr]  r   )r   r1   ry   rQ   s    r9   r   zListOf.__repr__U  $    ..))*!DLL>;;r<   c                (   t        |t        t        f      rt        |      dk(  rt	        d      S t        ||       }|j                         }d}t        |      D ]  \  }}t        |j                  ||j                        }|j                  | j                  |      }	|j                         }t        |	      s| j                  st	        d||	      c S {d}|j                  |	j!                                 |st	        d      S |j!                         S )Nr   non_listFr  zlist[{}]: {}Tzlist: no_match)r   r   r   r   r   rw   r  	enumerater   r   r   r  ry   r   r   r   r   )
r6   r   r   r   r   matchedr  r  	child_ctxr  s
             r9   r   zListOf._matchX  s    $u.#d)q.z**#t 88:&t_ 	+MAz$_J4D4DI $//$,,
CK'BBDOK(||&~q+FFGHH['')*	+ /00xxzr<   c                    t        j                  t        |      }t        |   |      xr@ | j
                  j	                  |j
                        xr | j                  |j                  k(  S r3   )r:  r;  r   r   r  ry   r   r<  s     r9   r  zListOf.pattern_eqp  sU    D%(Gu% .''6.-	
r<   F)ry   rx   r   r   r>   rT   r   )r   r|   r   r   r>   r  r  r?  r   s   @r9   r  r  J  s!    <0
 
r<   r  c                  ~     e Zd ZU ded<   d fdZedd       ZddZddZddZ		 	 	 	 	 	 ddZ
dd	Zd fd
Z xZS )MultiOutputPatternr   r   c                    t         |           t        |d   t              sJ t	        d |D              sJ |       t        |      | _        |d   j                  | _        y )Nr   c              3  H   K   | ]  }|d u xs t        |t                y wr3   )r   rx   ro  s     r9   rc   z.MultiOutputPattern.__init__.<locals>.<genexpr>  s#     Lq19:
1k ::Ls    ")r   rR   r   rG  r  r   r   rS  )r6   r   r   s     r9   rR   zMultiOutputPattern.__init__|  sU    '!*k222LGLLUgULG}!*--r<   c                h    t        j                  t        | j                  d         }|j                  S Nr   )r:  r;  rG  r   rH  )r6   outputs     r9   rH  zMultiOutputPattern.fns  s&     [$,,q/:zzr<   c                N    | j                   j                   d| j                   dS r  )r   r1   r   rQ   s    r9   r   zMultiOutputPattern.__repr__  r  r<   c                    | j                   D cg c]  }|j                  |       }}dd }| j                  j                   d|j	                  |       }| d}|S c c}w )Nz,
z  z([z
]))r   r)  r   r1   r  )r6   r(  rp  r7   r  str_outs         r9   r)  zMultiOutputPattern.pretty_print  sh    ,0LL9q"994&\
^^,,-R
0E/FGIT"	 :s   Ac                $   t        j                  t        | j                  d         }|j	                  ||      }t        |      s|S | j                  dd  D ]7  }|| j                  ||      }t        |      s|c S |j                  |       9 |S )Nr   r+   )r:  r;  rG  r   r  r   _match_from_anchorsr   )r6   r   r   r  r   ry   r  s          r9   r   zMultiOutputPattern._match  s    [$,,q/:IIfd#{H||AB' 	"G227C@KK(""HH[!	" r<   c                    t        |j                        }t        d      }|j                  |t	                     D ]3  }|j                  ||      }t        |      r|c S t        |      |_        5 |S )Nzno anchor found)r   r   r   r  r!   r  r   )r6   ry   r   priorr   r   s         r9   r  z&MultiOutputPattern._match_from_anchors  sl     S(()$%67--c:<@ 	.D		'4(A{"&u+C	. r<   c                    	 t        | j                  |j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wr  )r   r   r   r  r   r  s      r9   r  zMultiOutputPattern.match  s>    	DJJ?EEdDQQ 	H	s   03 	AAAAc                   t        j                  t        |      }t        |   |      xr] t        | j                        t        |j                        k(  xr0 t        d t        | j                  |j                        D              S )Nc              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wr3   r  r  s      r9   rc   z0MultiOutputPattern.pattern_eq.<locals>.<genexpr>  r  r  )	r:  r;  r   r   r  r   r   r  r   r<  s     r9   r  zMultiOutputPattern.pattern_eq  sm    D%(Gu% DLL!S%77 emm< 	
r<   )r   zSequence[Optional[PatternExpr]]r>   rT   )r>   z-Union[Callable[..., Any], str, Sequence[Any]]r   r*  r  )ry   rx   r   r   r>   r  r  r  )r1   r?   r@   rA   rR   r   rH  r   r)  r   r  r  r  r   r   s   @r9   r  r  y  s]    ((   
< ")5		
 	
r<   r  c                  J     e Zd ZdZd fdZedd       ZddZd	 fdZ xZ	S )
RepeatedExprzp
    Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
    c                R    t         |           || _        |j                  | _        y r3   )r   rR   inner_patternrS  )r6   r  r   s     r9   rR   zRepeatedExpr.__init__  s#    *""r<   c                .    | j                   j                  S r3   )r  rH  rQ   s    r9   rH  zRepeatedExpr.fns  s    !!%%%r<   c                   |j                  | j                  |      }t        |      s|S |j                  j	                  | j                         | j                  j                  |t                     D ]T  }t        | g|j                        j                  | j                  |      }t        |      s|c S |j                  |       V |S r  )
r  r  r   r   popr  r!   r   r   r   )r6   r   r   r   anchor_nodeanchor_ms         r9   r   zRepeatedExpr._match  s    IId(($/{H	
  --??Z\R 	K#TF$**=CC""KH H%HHX	 r<   c                    t        j                  t        |      }t        |   |      xr% | j
                  j	                  |j
                        S r3   )r:  r;  r   r   r  r  r<  s     r9   r  zRepeatedExpr.pattern_eq  sF    D%(w!%( 
T-?-?-J-J.
 	
r<   )r  rG  r>   rT   )r>   zSequence[FnsType]r  r  )
r1   r?   r@   r   rR   r   rH  r   r  r   r   s   @r9   r  r    s0    #
 & &"
 
r<   r  c                  Z    e Zd ZdZddZeej                  dd	d              Zd
dZ	ddZ
y)r+  z
    Serializes Patterns to executable python.
    XXX: currently only used and tested for fuse attention patterns. May not cover
    all patterns.
    c                z    t         j                  j                  j                         | _        i | _        i | _        y r3   )r[   r\   r   
_Namespace	namespacememoized_objs_namesmemoized_objs_pprQ   s    r9   rR   zPatternPrettyPrinter.__init__  s*    224;= 8:r<   c                   t               }t        | d      sJ | j                  |      }|j                  D cg c]#  }|j                  |    d|j                  |    % }}|j                  | d|        dj                  |      S c c}w )zU
        Serializes obj to python code with obj written out to `output_name`
        r)  )r(  z = 
)r+  hasattrr)  r  r  rl   r  )objoutput_namer(  out_strr   r  s         r9   runzPatternPrettyPrinter.run  s     "#sN+++""b") --
 %%c*+3r/B/B3/G.HI
 

 	S	23yy  
s   (B
c                    t        |t              r0| j                  j                  |      x}r|S | j	                  |      S t        |d      r|j                  |       S t        |      S )Nr)  )r   rk  r  rj   memoizer  r)  r  )r6   r  memoized_names      r9   r)  z!PatternPrettyPrinter.pretty_print  sb    c?+ $ 8 8 < <S AA}A$$||C((3'##D))Cyr<   c                    |j                  |       }|j                         }dD ]  }|j                  |d      } | j                  j	                  |d       }|| j
                  |<   || j                  |<   |S )N)zaten.rV  zprims.r\  )r)  rZ  replacer  create_namer  r  )r6   r  obj_strobj_nameprefixtmp_names         r9   r  zPatternPrettyPrinter.memoize  s{    ""4(<<>3 	4F''3H	4 >>--h=(0  %%,c"r<   NrS   )r  )r  rx   r  r0   r>   r0   )r  r   r>   r0   )r  rk  r>   r0   )r1   r?   r@   r   rR   r  r   cacher  r)  r  r4   r<   r9   r+  r+    s6    ;
 __!  !$		r<   r+  c                      e Zd Z	 	 	 	 ddZy)_PassDictsTypec                     y r3   r4   )r6   ra   s     r9   __getitem__z_PassDictsType.__getitem__   s     r<   N)ra    tuple[str, torch.fx.node.Target]r>   list[PatternEntry])r1   r?   r@   r  r4   r<   r9   r
  r
    s    !1!	!r<   r
  c                  F    e Zd ZU ded<   ded<   ddZ	 	 d		 	 	 	 	 	 	 d
dZy)PatternEntryrx   ry   Callable[[Match], bool]extra_checkc                    t         r3   ra  r6   r  r   r   s       r9   applyzPatternEntry.apply*  s    !!r<   Nc                   |Ht        | j                  d      sJ | j                  j                  D ]  }| j                  |||        y t	        |t
        t        f      rqt        | j                  d      sJ |r,|| j                  j                  |f   j                  d|        y || j                  j                  |f   j                  |        y t        j                  t        t           |      }|D ]  }| j                  |||        y )NrH  prependrS  r   )r  ry   rH  registerr   r   PatternMatcherPassrS  insertrl   r:  r;  r   r
  )r6   
pass_dictsr   r  rI   rp  s         r9   r  zPatternEntry.register-  s     >4<<///ll&& ?j"g>?
T+=$>?4<<...DLLOOV45<<QEDLLOOV45<<TBXn%=zJJ :a9:r<   r  rw   r   r   r   r   r>   rT   r   )r  /Union[_PassDictsType, Sequence[_PassDictsType]]r   z!Union[torch.fx.node.Target, None]r  r   r>   rT   )r1   r?   r@   rA   r  r  r4   r<   r9   r  r  %  sF    ((" 59	:C: 2: 	:
 
:r<   r  c                       e Zd ZU ded<   ddZy)LoweringPatternEntryCallable[..., Any]handlerc                    t        j                  | j                        t        j                  | j                  |            }|j	                  |      5  |j                  |t        |j                        |j                        }|j                  j                  |j                         |j                  |       d d d        |j                  d   |u sJ |j                          y # 1 sw Y   -xY w)N)r   wrapsr"  r   inserting_beforer  r   r7   r8   ro   rn   replace_all_uses_withr}   r   )r6   r  r   r   r"  r   s         r9   r  zLoweringPatternEntry.applyG  s    /)//$,,/	0A0A$,,PU0VW##D) 	4--guUZZ7H%,,WK##DII.&&{3	4 {{2$&&&	4 	4s   A'C((C1Nr  )r1   r?   r@   rA   r  r4   r<   r9   r   r   C  s    r<   r   c                  $    e Zd ZU dZded<   ddZy)GraphPatternEntryz8
    A pattern that runs a function on the FX graph
    r!  r"  c                    |j                  |      5   | j                  |g|j                  i |j                   d d d        y # 1 sw Y   y xY wr3   )r&  r"  r7   r8   r  s       r9   r  zGraphPatternEntry.applyY  sE    ##D) 	=DLL<<u||<	= 	= 	=s   +AANr  )r1   r?   r@   r   rA   r  r4   r<   r9   r)  r)  Q  s      =r<   r)  c                  F    e Zd ZU ded<   e	 	 	 	 	 	 	 	 	 	 dd       ZddZy)r   zCallable[..., list[Any]]normalize_argsc                    G fddt         j                  j                        }| j                         }t	        |      dk(  r|d   }n|d   sJ t        |d   j                  j                        }|D cg c]9  }t        |t         j                  j                        r|j                  |      |f; }	}t        |	t        j                  d            d   }	 	 	 	 	 	 	 	 	 	 d	dj                  |      5  t        |t         j                  j                        sJ   ||      j                    }
t        |
t         j                  j                        r|
g}
d
d	 	 	 	 	 	 dfdt	        |      t	        |
      k(  rt#        ||
      D ]  \  }} ||        nt	        |      dk(  sJ  |d   |
       d d d        | j%                          y c c}w # 1 sw Y   xY w)Nc                  0     e Zd ZdZdZdZd fdZ xZS )<ReplacementPatternEntry.replace_with_graph.<locals>.ReplacerNc                   |j                   dv rt        
| 	  |      S |j                  }| j	                  |      \  }}|j                   dk(  rt        |      sJ j                  |||      }t        |j                  |d       d|j                  v r|j                  d   |j                  d<   d|j                  v r}d|j                  vro|j                  d   |j                  d<   t        |j                  d   t        j                        r,d|j                  v sJ |j                  d   |j                  d<   |S |j                   dk(  rd	d
lm} t        
| 9  |||      }t        |t        j                  j                         st#        d| d| d      j$                  J  |j$                  t'        |            \  }}	j$                  j)                  |	|       j                  |	      S t#        d|       )N)placeholderr  r  Interpreter_Replacerr   r   r   tensor_metaget_attrr   )unique_graph_name_with_rootzNYI: replacement_graph.z is not a graph module. Got .z
unhandled )rS  r   run_noder   fetch_args_kwargs_from_envrN  r  ru   ro   r   r[   Tensortorch._higher_order_ops.utilsr5  r4  r\   GraphModulerb  owning_moduler0   register_module)r6   r   r   r7   r8   resultr5  sub_gm_
graph_namer   r   s             r9   r7  zEReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_noden  s   7777 7+D11#>>tDf77o-#F+++"00vFF"!'!%"8 *TYY6:>))DV:W$67		)e6;;.F-1YYu-=E*%dii&6E#0DII#==#=9==9QFKK6!M77j( #W-fdFCF%fehh.B.BC15fX=YZ`Yaabc  !..:::$?++S[%MAz ''77
FK >>*55)Jtf*=>>r<   r   r   r>   r   )r1   r?   r@   r  r  r4  r7  r   )r   r   s   @r9   Replacerr/  i  s    KKH,? ,?r<   rC  r+   r   )r   c                   | g}t        t        j                  j                            }|rc|j	                         }||vrK||vrGt        |d      r;|j                  |       ||j                  |<   |j                  |j                         |rby y )Nro   )
r!   r[   r\   r   r  r  r  ro   r   all_input_nodes)r   tag_name	tag_valueinput_stopsqueuevisitedr   s          r9   percolate_tagszBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tags  sy     FE /1Giikw&;.V,KK$)2CHHX&LL!4!45 r<   c                    | j                   dk7  ry | j                  t        j                  k7  ry t	        | j
                        dk(  sJ | j
                  d   S )Nr  r"   r+   )rS  r   rW  getitemr   r7   r   s    r9   maybe_getitemzAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitem  sJ    77o-;;("2"22499~***yy|#r<   c           	        | |J y t        | t        j                  j                        sJ |#| j	                  d        j                  |        y t        |t        j                  j                        rd|j                  vr%|j                  j                  | j                         dD ]2  }|| j                  v s 	||| j                  |   t                     4 | j	                  |       j                  |        y t        | j                  j                               }|D ]#  } |      }|t        d       
|||          % j                  |        y )Nr   )	recomputeac_graph_idzcan't handle)r   r[   r\   r   r'  r   ro   rn   r!   r   r   r   AssertionError)oldnewrF  old_usesuseridxr7   r   rO  rK  r  s         r9   r  z;ReplacementPatternEntry.replace_with_graph.<locals>.replace  sA    ;;&;!#uxx}}555;--d3$$S)c588==1CHH,1 %A #sxx/* #Xsxx/A:dCS --c2$$S):  		 01$ ,D'-C{,^<<D#c(+	,
   %r<   )
r   r   rF  r0   rG  r0   rH  r  r>   rT   rB  )rT  zUnion[torch.fx.Node, None]rU  z3Union[torch.fx.Node, Sequence[torch.fx.Node], None]r>   rT   )r[   r\   Interpreterr   r   r   r   r}   r   r   indexminrW  
itemgetterr&  r;  r  r   r   )r  r   r   r7   rC  r   	last_noder}   r   indicesr   rT  rU  rO  rK  r  s    ` `         @@@r9   r   z*ReplacementPatternEntry.replace_with_graphb  s
   1	?uxx++ 1	?f ))+|!$QI?"?a..445E &a/ Q#G 
 G)<)<Q)?@CI	6	6	6 	6 3		6
 	6( ##I. X	6/1E1EFFF9(#45994@K+uxx}}5*m$C&/C&HC& C& C&J < C$44 #L+ > &HCC%& <(A---Q5qX	6t 	k6X	6 X	6s   <>G>CG$$G-c           
         |j                   J | j                  |||j                    | j                  |j                  i |j                         y r3   )r   r   r,  r7   r8   r  s       r9   r  zReplacementPatternEntry.apply  sN    &&222##D<u||<		
r<   N)
r  rw   r   r   r   z+Union[torch.fx.Graph, torch.fx.GraphModule]r7   zSequence[torch.fx.Node]r>   rT   r  )r1   r?   r@   rA   r  r   r  r4   r<   r9   r   r   ^  sV    ,,uuu Gu &	u
 
u un
r<   r   c                     yr   r4   )r  s    r9   _return_truera  $  s    r<   c                F    t         j                  d| j                  |       y )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginfor1   )	search_fnr  s     r9   log_trace_failurerf  (  s    HHJ	r<   Fc                p   t         j                  |       |j                        }|s'|   j                  |rt	        |             yd       y||ryt        j                  dfd       t	        |      }|D ]&  |k(  s	|r yt        j                  dfd       ( |j                  |       y)aB  
    Check if a pattern is a duplicate. Because we ignore certain types in searching, but not
    in matching, use the graph to distinguish equivalent search patterns.

    Returns True if a duplicate is found and `skip_duplicates=True` is passed in. Errors if
    `skip_duplicates` is False and a duplicate is found.
    NFTc                     d  dS )NDuplicate pattern: z with no graphr4   )pattern_reprs   r9   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>I  s    ),~F r<   c                     d d  dS )Nri  z with duplicated match graph  r4   )	graph_strrj  s   r9   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>T  s    ),7TU^T__`a r<   )r+  r  rj   rl   r0   r[   _check)ry   r   seen_patternsskip_duplicatesequiv_pattern_reprsnew_graph_strrm  rj  s         @@r9   check_and_add_duplicate_patternrs  0  s     (++G4L'++L9l#**3u:I EII}F	

 JM( 
		)a	

 }-r<   r4   c
           
         g t        j                         j                  j                         d f	d}
d	fd}t        u rt        j                         ryt        j                  d      5  |D cg c]*  }t        |t
        j                        xr |j                  , c}t         |      \  }}n}d}t        |t              r|n|gD ]D  }t        |t              st        ||r|j                   nd|j"                  |	      s< ddd       y t%        ||
|      }|j'                  |       |j(                  cddd       S c c}w # 1 sw Y   yxY w)
a  
    Create a replacement rule based on example functions that get traced
    to create patterns.  This supports both training and inference when
    run on a joint forward+backward graph.

    Args:
        search_fn: traced to give original pattern
        replace_fn: traced to give replacement graph
        example_inputs: example inputs for initial trace
        trace_fn: fwd_only or joint_fwd_bwd
        pass_dict: dict of passes to register to
        extra_check: additional check to run on match(using real shapes)
    c           
       	 t              }|D ]+  }|| j                  vst        d| d| j                          t        t        j                  j                  |D cg c]  }| j                  |    c}d             g }t        j                  j                  j                        5  t              D ]  \  }}t        |   t        j                        s%|r"t        |   j                        r
 ddd       yt        j                  |   j                         |   j!                         |   j                  |   j"                  |      |<   t%        j&                  |   j(                  |   j!                               D ]C  t        t        j*                        st-        fd|D              s3|j/                         E  }|s3|rdfd}	  ||z         }g }
t3        t5        t7        |      t7              z         |j8                  j:                        D ]  \  }}|t7        |      k  r|
j/                  |j<                         0|j8                  j?                  |      5  |j8                  jA                  ||t7        |      z
           }|jB                  |_        |jE                  |       |j8                  jG                  |       ddd        |
|z   }n
	        }tI        ||	      }| jK                         d
   }|J |jM                  |      }tO        |      r |      rz       | _(        t7        | j:                        dk(  rJ| jP                  j8                  j:                  D ]'  }tS        |jT                  | j:                  d
   d       ) 	 ddd       y	 ddd       yc c}w # t        $ r}	t1        |	       Y d}	~	ddd       yd}	~	ww xY w# 1 sw Y   xY w# t        $ r}	t1        |	       Y d}	~	ddd       yd}	~	ww xY w# 1 sw Y   yxY w)z
        Often shapes get burned into the pattern, so our initial match ran with
        `ignore_types=(int, ...)`.

        Recheck the match with the correct shapes.
        z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                     | j                   d   S r   r   )r   s    r9   r   z8register_replacement.<locals>.check_fn.<locals>.<lambda>  s    QVVE] r<   NF)dtypedevicerequires_gradc              3  :   K   | ]  }t        |k7          y wr3   r   )r`   r  rb   s     r9   rc   z9register_replacement.<locals>.check_fn.<locals>.<genexpr>  s       ?>?1!q&9?s   c                 >     | t        |       t              z
  d   S r3   )r   )args_newr7   re  s    r9   search_fn_newz=register_replacement.<locals>.check_fn.<locals>.search_fn_new  s#    ((3x=3t93L3N*OPPr<   )argnamesexclusive_arg_namesscalar_workaroundr   r+   r   r   T)r|  r   r>   r   )+r   r8   r   r[   r\   r   _dynamor   detect_fake_moder  r   r9  r   rw  empty_stridedsizestriderx  rr  rs  shapeSymIntr  rl   rf  r   ranger   r   r}   r   inserting_afterr1  r1  r'  r   fx_to_patternr   r  r   r   ru   ro   )r  r~  r1  sym_argsr  gradspecific_patternr}  specific_graphr  sym_arg_namesr1  r   r   specific_pattern_matchr   r7   rb   argnames_staticr  r  
replace_fnry  r  re  search_fn_patternr   s                   @@r9   check_fnz&register_replacement.<locals>.check_fnu  s,    ( 	D5<<'"99A
/RWR^R^Q_a 	 HH089d#9;R
 (*]]  11$7 Z	$]3 /4d1gu||4 0a ?$	Z	 Z	 $11QQ("1gmm#Aw~~&*DG '__T!W]]DGNN<LM /%a63 ?CK? < %OOA.	//*  1#Q%)1-D)Q %'M*-c(mc$i78&,,22+ I; s8},)001C1CD$+11AA+N I'5';';'G'G (S]): ;(H /7mmHO'==hG*00;;KHI II   -x7H%)1)T)B
 $1"%(;&7	$  %%'*D###%5%;%;D%A"./K@V4W*2:t*D'u{{#q("44::@@ &%&VV%*[[^&3 sZ	 Z	t uZ	 Z	 :V ( %))Q7$QZ	 Z	L%I I ( %))Q7$EZ	 Z	@%AZ	 Z	s   !O/0P7 P7B*P7/P7%P7*O6BP7A(P )P7:	PB;P7P7	O=O8+P78O==P7 P
P7	P4P/"P7/P44P77Q c                     D cg c]  }| j                  |       }}t        dt        |       dz         D ].  }d| | vr n%|j                  | j                  d|              0 | r
J d|        |S c c}w )Nr+   	tangents_zleftover kwargs: )r  r  r   rl   )r8   r1  r7   r  r  s       r9   r,  z,register_replacement.<locals>.normalize_args  s    -<=T

4 ==q#f+/* 	5A1#f,KK

Yqc?34	5 9.vj99z >s   A7Ffunctionalize_rng_opsN)rp  )ry   r  r,  r  rw   r>   r   )r8   r   r>   rz   )inspect	signature
parametersr   joint_fwd_bwdr[   is_inference_mode_enabledfunctorch_configpatchr   r9  ry  gen_pattern_and_search_gmr   r  rs  r   ro  r   r  ry   )re  r  example_inputsr   r  r  r  r  r  rp  r  r,  rp  ry   gmpattern_matcher_passr  ry  s   `` ` ````       @@r9   register_replacementr  Z  so   2 H)))4??DDFGOp pd =  **, 
		e	< "ES%
@AJq%,,';AOO;%
 $3!#KGR (GB %Z:J
	!  .0BC2 "BHH(66$3	 !5" " 
	! * )

 	$E" "%
" "s1    E%/E 4AE%9'E%!E%++E% E%%E.zOrderedSet[str]_serialized_patternsc                V   dd}t         j                         st        dt                |j                  }ddlm}  |j                  d      5  t        ||||      }d d d        t        j                  |       }	|t        vrd}
t        j                  |       nd	}
 |       }t        t         | d
z  |
      5 }|
dk(  r|j                  |       n|j                  d       |j                  |	       |j                  d       d d d        |S # 1 sw Y   xY w# 1 sw Y   |S xY w)Nc                    t        j                  d      } t        j                  d      j                  |       }g }t        t        j
                  j                        D ]^  }t        t        j
                  j                  |      }	 t        |t              r't        |t        t        f      r|j                  |       ` dj                  |      }d| d}| | S # t        $ r Y w xY w)Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            a               # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor
            import operator

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr   dirr[   	_inductorpattern_matcherrL  r   r  
issubclassrx   rG  rl   	TypeErrorr  )auto_generated_msgfile_templatepattern_matcher_importsr1  attrformatted_importss         r9   get_file_templatez-_serialize_pattern.<locals>.get_file_template&  s    %__
 !
 &'&
( 	  #%778 	D5??::DADdD)j;4/ ,2248	 %MM*ABPQbPccij!2 344  s   7C	C*)C*z0Could not find serialized patterns directory at r   r#   Fr  )r  wr  z.pyz

r  r   )SERIALIZED_PATTERN_PATHis_dirr   r1   torch._functorchr$   r  gen_patternr+  r  r  r  openwrite)unique_namere  r  r   r  r  pattern_namer  ry   serialized_pattern
write_moder  fs                r9   _serialize_patternr    s)   &5P #))+>?V>WX
 	
 %%L;				e	< ViCTUV .11'{1S//
  .
%'M	%<.(<<j	I QGGM"GGFO	"#	 N)V V Ns   D=ADDD(	fx_passesserialized_patternszvlist[tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsc
                   t        |      }dt        j                  v rt        | ||||      }
nT|j                  }t        j                  d|       }|rt        ||       st        j                  d|        t        ||       }
t        j                  |      D ]'  }t        |t              s|j                  !d |_        ) t         j#                  |||||
f       t%        |||||||||
|	
       y )NPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)r  rp  )r   osenvironr  r1   	importlibimport_moduler  rc  warningrL  r  	tree_iterr   r(   constantr  rl   r  )r  re  r  r  r   r  r  r  r  rp  patr  r   r   s                 r9   gen_register_replacementr  ~  s    >*N+ NH>O
 !))##<\NK
 ;/KKV a%/  c:&3<<+C
  CL   &&	NH.?E 'r<   r  c                r   g t        j                  |       j                  j                         }|i }g }d}|D ]4  }||v r|j	                  ||          |j	                  ||          |dz  }6  || |      }	t        |	t        t        t        t        j                  t        j                  f|||      |	fS )Nr   r+   )ignore_typesr~  r  r  )r  r  r  r   rl   r  intfloatr   r[   rx  rw  )
re  r  r   r  r  r~  flat_inputs	input_idxargname	search_gms
             r9   r  r    s     A""9-88==?@H KI ''09:~i89NI K0IudELL%++F/ 3	
 		 	r<   c                &    t        | ||||      d   S r  )r  )re  r  r   r  r  s        r9   r  r    s&     %>8->@S	 	r<   r  c                     d fd}|S )z
    Register an aten to inductor IR replacement pattern.  The decorated
    function is saved and then called a lowering time allowing direct
    pattern to inductor IR conversion.
    c                n    t        |       sJ t        |       j                         d| _        | S )Nry   r  r"  r  T)rN  r   r  _inductor_lowering_functionr"  r  	pass_dictry   r  s    r9   	decoratorz,register_lowering_pattern.<locals>.decorator  s;       g	

(9g(
..2+r<   r"  r!  r>   r!  r4   ry   r  r  r  r  s   ```` r9   register_lowering_patternr    s      r<   c                     d fd}|S )zk
    Register a pattern that runs a function on the FX graph, allowing
    custom transformation code.
    c                `    t        |       sJ t        |       j                         | S )Nr  r  )rN  r)  r  r  s    r9   r  z)register_graph_pattern.<locals>.decorator	  s3       g	

(9g(
.r<   r  r4   r  s   ```` r9   register_graph_patternr    s      r<   c                B    |t        t        | j                              u S r3   )r   iterr}   )r   r   s     r9   is_start_of_fx_graphr    s    4U[[)***r<   z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)c                    | j                   dk7  ry| t        j                  j                  j                  j
                  t        j                  j                  j                  j
                  fv S )NinductorF)r  r[   r   r  accumulate_grad_defaultresize_storage_bytes_)rS  s    r9   "fixme_incorrect_inductor_schema_opr    sW    	||z! 		++33		0088  r<   c                   t        | j                  t        j                  j                        r5t        | j                        s | j                  j                  j                  S t        | j                  t        j                  j                  j                        ry| j                  dk(  rAt        | j                        sJ t        j                  | j                  j                        rLy| j                  dk(  r<t        | j                  t               sJ t        j                  | j                        ry| j"                  j%                  d      d uS )NFr  Tr  out)r   r   r[   rO  rX  r  _schema
is_mutable_higher_order_opsauto_functionalizeAutoFunctionalizedrS  rN  _mutation_op_researchr1   r0   r8   rj   rN  s    r9   is_mutation_opr  )  s    UZZ**0={{""---	U,,??RR
 ww/!$$$!!$++"6"67	M	!$++s+++!!$++.;;??5!--r<   c                    d| j                   v sJ d|j                   v sJ | j                   d   |j                   d   k(  S Nmutation_region_idr   )r  r  s     r9   same_mutation_regionsr  =  sE    166)))166)))66&'1662F+GGGr<   c                ,   |}d|j                   vr3t        | |      s'|j                  }d|j                   vrt        | |      s'|j                   j                  dd      }||ur0|j                  }t        |      r|dz  }||j                   d<   ||ur0|S )Nr  r   r+   )ro   r  prevrj   r   r  )r   r   r   r  s       r9   get_mutation_region_idr  C  s    A
aff
,5I%QR5SFF aff
,5I%QR5S$8!<
4-FF!!#'9#$	 4-
 r<   c                V    dt        t        | j                              j                  vS r  )r   r  r}   ro   r  s    r9   "should_compute_mutation_region_idsr  P  s!    tD,='>'C'CCCr<   c                h    d}| j                   D ]!  }t        |      r|dz  }||j                  d<   # y )Nr   r+   r  )r}   r  ro   )r   r  nds      r9   compute_mutation_region_idsr  T  s=    kk ;"!#(:$%;r<   c                  B     e Zd Z	 d	 	 	 d fdZddZddZd	dZ xZS )
r  c                    t         |           t        t              | _        || _        t        t              | _        y r3   )r   rR   r   r   patternsrs   ro  )r6   rs   r   s     r9   rR   zPatternMatcherPass.__init__]  s:     	  	 # >I=Nr<   c                     | j                   |   S r3   )r	  )r6   items     r9   r  zPatternMatcherPass.__getitem__m  s    }}T""r<   c                   | j                   syt        |t        j                  j                        r|j
                  }nJt        |t        j                  j                        r|}|j                  }nt        dt        |             t        |      rt        |       t        j                  t        |      }d}g }d}| j                   D ]0  \  }}|dk(  rd}|j                  |j!                  ||d             2 |r"|j                  |j!                  dd             | j"                  | j"                  nd}	t        |t        j                  j                        sJ t%        ||	      5  t'        t(        j*                  j-                  |      d	      D ]  }
t/        |
      }|
j0                  dk(  r|
j0                  |f| j                   vr9t3        |
d
      rG| j                   |
j0                  |f   D ]-  }|
j4                  r t|j6                  j9                  |
      }t;        |      r,t=        t?        tA        ||jB                                    dk7  rdtD        jF                  jI                  d      |
jJ                  k(  r-tL        jO                  d|
|
jP                  ||j6                         t;        |      s|jS                  |      s|dz  }|jU                  |||
       tV        d   dxx   dz  cc<   tV        d   dxx   t=        |jB                        z  cc<   0  	 d d d        |S # 1 sw Y   |S xY w)Nr   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Fr  T)rS  r   sort)rS  r  r  )reverse)allow_cpu_inputsr+   !TORCHINDUCTOR_PATTERN_MATCH_DEBUGz
%s%s %s %sr  pattern_matcher_countpattern_matcher_nodes),r	  r   r[   r\   r;  r   Graphr<  r   r  r  r  r   r   r  rl   
find_nodesrs   r    sortedrr  rs  from_iterablerd  rS  r-   r   ry   r  r   r   r!   r  r}   r  r  rj   r1  rc  r  r7   r  r  r   )r6   r  r   get_mutation_region_id_partialr  r}   has_call_modulerS  r   rs   r   entryr   s                r9   r  zPatternMatcherPass.applyp  s   }}b%((../HHEEHHNN+E$$B\]abd]e\fg  .e4'.)2):):"E*
& -- 	QJB]""&U--F-OP		Q
 LL))])GH&*nn&@DNNFW	"ehh22333#B	2  	Vy<<UCTR V'-77m+(= 
 9PUV!]]DGGV+<= VE||++D1A !&s+I177'ST 
 !zz~~&IJdiiWL$		1emmT{u'8'8';
Aud3 ,-DEJE ,-DEQWWUE'VV 	VB C 	VB s   .EL:L:AL::Mc                8    | j                   j                          y r3   )r	  clearrQ   s    r9   r  zPatternMatcherPass.clear  s    r<   r3   )rs   Optional[str]r>   rT   )r  r  r>   r  )r  +Union[torch.fx.GraphModule, torch.fx.Graph]r>   r  rS   )r1   r?   r@   rR   r  r  r  r   r   s   @r9   r  r  \  s2     $(O O 
O #>@r<   r  c                     t         r3   ra  r   s     r9   _not_implementedr    s    
r<   c                  	
 |xs i }|j                         D ci c]  \  }}||
 c}}
t        
      t        |      k(  sJ 	 d	 	 	 	 	 d
fdt        j                         	 G 	fddt        j
                  j                        }t        | t        j
                  j                        sJ  ||       j                         }t        |t              st        t        j                  |            S |S c c}}w )z
    Convert an FX graph into a PatternExpr.  This is useful for simple
    patterns that can only match single functions and fixed-length lists.
    c                    ||n}t        | t        t        f      r| v rt        |          S t	        |       |v r
t               S t        | t              rt        d | D              r| r
t               S | S )Nc              3  <   K   | ]  }t        |t                y wr3   )r   r#  )r`   ys     r9   rc   z5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>  s     &I!z!W'=&Is   )r   r  r  r-  r  r#  r   r  )rp  ignore_types_overridecurrent_ignore_typesr  inv_scalar_workarounds      r9   process_argz"fx_to_pattern.<locals>.process_arg  sy     &;%F!L 	 a%&10E+E3A6777**9a3&Iq&I#Ia9r<   c                  l     e Zd ZeZeZeZ	 	 	 	 	 	 	 	 dfdZ	 	 	 	 	 	 	 	 dfdZd fdZ	 xZ
S ) fx_to_pattern.<locals>.Converterc                    t              }|t              k  r|   }n1r|j                  d      sJ |}nt        j                  dd|      }|}|v rt        |      S t        |      S )Ntangentz_\d+$r\  )r   r   
startswithresubrA  r-  )	r6   r   r7   r8   r   r1  r~  argnumr  s	         r9   r1  z,fx_to_pattern.<locals>.Converter.placeholder  su     VA3x= {((333"f5***400!$''r<   c                b   	}|t         j                  k(  rt        d D              f	 	 	 	 	 d	fd}|}t        j                  |||f      \  }}t
        v r>|D cg c]
  } ||       }}|j                         D ci c]  \  }}| ||       }}}t        |g|i |S c c}w c c}}w )Nc              3  2   K   | ]  }|t         us|  y wr3   )r  )r`   ts     r9   rc   zAfx_to_pattern.<locals>.Converter.call_function.<locals>.<genexpr>  s      Q1C<Qs   c                     | |      S r3   r4   )rp  r$  r'  s     r9   process_arg_fn_implzKfx_to_pattern.<locals>.Converter.call_function.<locals>.process_arg_fn_impl  s     'q*?@@r<   rp  rK   r$  zOptional[Sequence[type[Any]]]r>   zUnion[T, KeywordArg, Ignored])rW  rM  r   r  r   r   rp   r  )
r6   r   r7   r8   process_arg_fnr4  r  ra   r  r'  s
           r9   r  z.fx_to_pattern.<locals>.Converter.call_function  s     )N))) LQ Q#/Q LAA+HA
 3A "5!??>D&>JLD&|#378aq)88;A<<>J41a!^A..JJ8888 9Js   B&B+c                l   t         |   |      }|j                  dk(  rzt        |t              rj|j
                  d   }t        |t              sJ t        |      t        |      k(  sJ t        ||      D ]  \  }}t        |j                        |_	        ! |S t        |j                        |_	        |S )Nr  r   )
r   r7  rS  r   r   r7   r   r   r   r   )r6   r   rvr7   rr   r   s         r9   r7  z)fx_to_pattern.<locals>.Converter.run_node  s    !!$BttxJr5$9vvay!$
3332w#d)+++!"dm -FAs!#))nAG- I qww<Ir<   )r   r0   r7   r   r8   Mapping[str, Any]r>   z&Union[ExclusiveKeywordArg, KeywordArg])r   r0   r7   r   r8   r:  r>   rx   )r   r   r>   r   )r1   r?   r@   r  r  r  r4  r1  r  r7  r   )r   r~  r/  r  r  r'  s   @r9   	Converterr)    sr    &&#	(	(  	( &		(
 4	((	9	9  	9 &		9
 	96
	 
	r<   r;  r3   r5  )rp   r   rr  r  r[   r\   rY  r   r;  r  rx   r  r  tree_leaves)r  r  r~  r  r  ra   rb   r;  ry   r/  r&  r'  s    `` `    @@@r9   r  r    s     */R.?.E.E.GHdaQTH$%->)???? FJ%B	& __F> >EHH(( >@ b%((..///m!Gg{+!&"4"4W"=>>No Is   DT)r   get_decomp_fnc                  t               5  | |       n	t               } t        | |d      | }ddd       ddlm} |r, |j
                         |j
                  j                          j                          |S # 1 sw Y   OxY w)z>Build a normalized inference graph, for use with fx_to_patternNreal)tracing_moder+   remove_noop_ops)r   r,   r   fx_passes.post_gradrB  r   eliminate_dead_code	recompile)rI   r7   r   r=  decompositionsr  rB  s          r9   r   r      s     
"	# E,8MO>Q>S 	 >WRf=tD	E 5!
$$&LLNIE Es   %A>>Bc           
        d	 	 	 	 	 	 	 	 dfd}t         j                  j                  d      5   t        | d |t	               dd      |  ddd       sJ ddlm}  |j                         dd	lm	} t               }t        t         j                  j                  j                  j                  t!        d
      t!        d            }t#        ||t$              j'                  |j(                         |j+                  j                         t         j,                  j                  j/                         j                  _        j                  j3                          j5                          S # 1 sw Y   *xY w)z=Build a normalized training graph, for use with fx_to_patternNc                <    rJ t        |       t        | |fi |S r3   )clone_graphr'   )joint_graphinputsr8   r  s      r9   record_joint_graphz)joint_fwd_bwd.<locals>.record_joint_graph?  s(     v% f???r<   c                    t        |       S r3   )r&   )gr  s     r9   r   zjoint_fwd_bwd.<locals>.<lambda>J  s    + r<   TF)partition_fnrF  keep_inference_input_mutations
enable_logr+   rA  )pointless_viewr   r  )ry   r"  r  )rJ  rJ   rK  r   r8   r   r>   z1tuple[torch.fx.GraphModule, torch.fx.GraphModule])r[   _guardstracingr%   r,   rC  rB  r   fx_passes.joint_graphrR  r  r  r   atenviewr  r-  r)  ra  r  r	  r  r\   CodeGen_codegenrD  rE  )rI   r7   rL  rB  rR  matcher_passry   r  s          @r9   r  r  :  sJ    *.B@)@3@@LO@	:@ 
		t	$ 	
++.0+/	
 	 I24BHH5%'L		##Z%6
68JG \h|$$%rxx  ..0BHHHH  "LLNI? s   E77Fc                    g }t         j                  j                  | j                  | j                  f|j
                         |S r3   )r[   r\   r   r7   r8   rl   )r   r7   s     r9   r  r  i  s1    )+D	HHaffahh'5Kr<   c                T   t        t        | j                              }t        t        j
                  j                            }t        t               }d }|r|j                         }t        |      D cg c]	  }||vs| }}|r||d      j                  |       n^|j                  |       |r|j                  |ur|j                  |       |}|j                  t        |j                  |d                   |r|s!t        |      t        | j                        k(  sJ y c c}w )Nr$  r4   )r   r   r}   r!   r[   r\   r   r   r  r  rl   r  r   r   r   )r   pendingreadywaitingcursorr   rp  waiting_fors           r9   stable_topological_sortrb  o  s     8EKK()G uxx}}%'E $G F
{{}"'+@Q%q@@ KO$++D1IIdO&++T1d#F NN8GKKb$9:;   3u:U[[)99999 As   4	D%>D%c                j     t         j                  t        j                         d fd              }|S )z0Wrapper around lazy init functions in fx_passes/c                 :   t         d   j                         } t        j                  j	                  d       5  t               5  t               5          }d d d        d d d        d d d        | t         d<   S # 1 sw Y   $xY w# 1 sw Y   (xY w# 1 sw Y   ,xY w)Nr  )r   rk   r[   rS  rT  r   r)   )counters_refr>  rI   s     r9   	lazy_initz%init_once_fakemode.<locals>.lazy_init  s      
+002]]""4( 	*@*B 	NDT 	TF	 	 	  ,	 	 	 	 	 	s;   BBA9BB9B>BB	
BB)r>   r   )r   r  r%  )rI   rf  s   ` r9   init_once_fakemoderg    s4     ____R	  	 r<   c                     d fd}|S )z2Function for extra_check to put pass behind a flagc                $    t        t              S r3   )rL  r$   )r  r1  s    r9   
flag_checkzconfig_flag.<locals>.flag_check  s    vt$$r<   )r  rw   r>   r   r4   )r1  rj  s   ` r9   config_flagrk    s    % r<   c                L     G d dt               } ||       j                         S )Nc                        e Zd Zd fdZ xZS )clone_graph.<locals>.CopyGraphc                J   t         |   |      }t        |t        j                  j
                        rn|j                  j                  j                  |j                         | j                  j                  j                  |j                  d       |j                  _        |S r3   )r   r7  r   r[   r\   Proxyr   ro   rn   	new_graph_graph_namespacer  r1  )r6   rr   r   r   s      r9   r7  z'clone_graph.<locals>.CopyGraph.run_node  sq    w'1H(EHHNN3""))(--8%)^^%D%D%P%PMM4&" Or<   )rr   r   r>   r   )r1   r?   r@   r7  r   r   s   @r9   	CopyGraphrn    s    	 	r<   rs  )r*   	transform)input_graphrs  s     r9   rI  rI    s$    K  [!++--r<   _seen_patternsc                    t        | j                        |kD  r| j                  |   S |y | j                  j                  |      S r3   )r   r7   r8   rj   )r   
arg_number
kwarg_names      r9   get_arg_valuerz    s?     499~
"yy$$		{{z**r<   c           	        |g}t        |t        j                  j                        r7|j	                  |j                         D cg c]  }t        ||       c}       | D cg c]  }|j                  |v s| c}S c c}w c c}w r3   )r   r[   rO  rP  r   rQ  rL  r   )r}   rI   rH  rM  r   s        r9   filter_nodesr|    sf    $C"ejj112

",,.IhGB)IJ"9TdkkS&8D99 J9s   A>#B7Bc                    | j                   dk(  rPt        | j                  t              sJ t	        | j
                  j                  | j                        j                  S | j                  S )zFor call_function and call_method, we directly use the target function;
    For call_module, the target is string, and we treat the module class
     as a function.
    r  )rS  r   r   r0   r   r   r<  r   rN  s    r9   rd  rd    sO    
 ww-$++s+++114;;?III;;r<   )r\  )rq   r{   rr   r   rs   r0   r>   rT   )r   r  r>   zTypeIs[Match]r  )re  r!  r  r   r>   rT   r  )
ry   rx   r   zOptional[torch.fx.Graph]ro  zdict[str, list[Optional[str]]]rp  r   r>   r   )re  r/   r  rC   r  Iterable[Any]r   rG   r  r  r  r  r  )Union[dict[str, Union[float, int]], None]r  Sequence[str]r  zUnion[PatternExpr, None]rp  r   r>   r   )r  r0   re  r/   r  r   r   rG   r  r  r>   rx   )r  r0   re  r/   r  rC   r  r~  r   rG   r  r  r  r  r  r  r  r  rp  r   r>   rT   )Nr4   )re  r/   r  r   r   rG   r  r  r  r  r>   z(tuple[PatternExpr, torch.fx.GraphModule])re  r/   r  r   r   rG   r  r  r  r  r>   rx   )
ry   rx   r  r  r  r
  r  r   r>   z2Callable[[Callable[..., Any]], Callable[..., Any]])r   r   r   r   r>   r   )rS  ztorch._ops.OpOverloadr>   r   ri  )r  r   r  r   r>   r   )r   r   r   r   r>   r  )r   r   r>   r   )r   r   r>   rT   )r7   r   r8   r   r>   r   )r4   r4   Nr4   )r  r  r  zSequence[type[Any]]r~  r  r  r  r  r  r>   rx   )
rI   r!  r7   r   r   r   r=  zOptional[Callable[..., Any]]r>   rJ   )rI   r!  r7   r   r>   rJ   )r   r   r>   zlist[torch.fx.node.Argument])rI   r!  r>   zCallable[[], Any])r1  r0   r>   zCallable[[Match], Any])ru  rJ   r>   rJ   r3   )r   r   rx  r  ry  r  r>   r   )r}   zIterable[torch.fx.Node]rI   r   r>   r|   )r   r   r>   ztorch.fx.node.Target)r   
__future__r   r   dataclassesr   r  r  rr  loggingrW  r  r-  r  r:  abcr   r   collectionsr   collections.abcr   r   r	   r
   r   pathlibr   r   r   r   r   r   r   r   typing_extensionsr   r   r[   torch._guardstorch.fxtorch.utils._pytreer   r   r  torch._dispatch.pythonr   torch._dynamo.utilsr   torch._prims_commonr   torch._subclasses.fake_tensorr   "torch.fx.experimental.proxy_tensorr   %torch.fx.experimental.symbolic_shapesr   torch.fx.graph_moduler   torch.fx.immutable_collectionsr   r   (torch.fx.passes.graph_transform_observerr    torch.utils._ordered_setr!   
_functorchr$   r  _functorch.aot_autogradr%   r&   _functorch.partitionersr'   _subclassesr(   r)   r\   r*   r\  decompositionr,   loweringr-   	getLoggerr1   rc  r   rV  primsConstantr   r  r/   rC   rG   rK   r   Targetr0   FnsTyperM   rO   ru   rw   r   r   r  r   r   rx   r  r#  r-  rA  rG  r   _SimpleSpecrk  r  r  r  r  r  r  r  r  r  r  r+  r
  	dataclassr  r   r)  r   ra  rf  rs  r  r  rA   r  __file__parentr  r  r  r  r  r  r  r  r  compiler  r  r  r  r  r  r  r  r  r  no_gradr   enable_gradr  r  rb  rg  rk  rI  rv  rz  r|  rd  r4   r<   r9   <module>r     s  !F #         	 	   # # N N  N N N *    $ $ ; ( 0 @ 6 G + I K / 3 C 7 4   . ; g!yy~~		x./>x >> >#h # CL $$c)
*? ? : IK

(5
BE
	
4| |~, 8 E;&''
 '
T 1#  1F-+ -k E E(E+ E4O
+ O
d CHol
k l
^?    (, * * ,
[ ,
^F
 F
R#
; #
L4 4n!X ! : : :: 
< 
 
 	= 	= 	= B
l B
 B
J "	''#' 2' 	'
 
'` ,8CG)+26! " 	
 @ ) A ' 0  
D )3 o 4LLL "L 	L
 AL L^ x.//+=@UU     $ ,8CG)+!555 5 "	5
 5 @5 )5 A5 '5 5 
5p e4
 DH)+!  A	
 ' . 5L DH)+				!		 		 A			
 '		 		 ,8
 ( 	
  84 ,8
 ( 	
  8,+ "**VW	.(H
D;U Up )+ CG)+e3e%e e A	e
 'e eP 
 #'26
  	
 0  2 + +\!:H&. #-, . GK+
+%(+6C++:r<   