
    ia                     T    d dl mZ d dlmZ ddlmZmZ ddlmZ d
dZ	d Z
ddZd	 Zy)    )
MethodTypeN   )is_hpu_availableis_transformer_engine_available)GatheredParametersc                 h   t               st        d      t               r&ddl}t	        |d      st
        j                  |_        nddlm} | j                         D ]  \  }}t        |t
        j                        r|r|r|j                  du}|j                  g}|r|j                  |j                         t        |d      5  t!        d |j                  j"                  D              r	 ddd        y|j                  |j$                  |j&                  ||j                  j(                        }	|	j                  j+                  |j                         |r%|	j                  j+                  |j                         t-        | ||	       ddd       ?t        |t
        j                        r|r|rt        |j                  |j                  gd      5  |j                  du}|j                  |j.                  d   |j0                  |j                  j(                        }	|	j                  j+                  |j                         |r%|	j                  j+                  |j                         ddd       t-        | |	       1t        ||j                        r|s|r|j                  du}t        j                  |j$                  |j&                  ||j                  j(                        }
|
j                  j+                  |j                         |r%|
j                  j+                  |j                         t-        | ||
       t        ||j                        r|s|rt        j                  |j.                  d   |j0                  |j                  j(                        }
|
j                  j+                  |j                         |
j                  j+                  |j                         t-        | ||
       t3        ||||	        y# 1 sw Y   xY w# 1 sw Y   xY w)
zu
    Recursively converts the linear and layernorm layers of a model to their `transformers_engine` counterpart.
    zBUsing `convert_model` requires transformer_engine to be installed.r   N	LayerNorm)modifier_rankc              3   ,   K   | ]  }|d z  dk7    yw)   r   N ).0ps     m/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/accelerate/utils/transformer_engine.py	<genexpr>z convert_model.<locals>.<genexpr>2   s     @qq2v{@s   )biasparams_dtype)epsr   )to_transformer_engine_convert_linear_convert_ln)r   ImportErrorr   intel_transformer_enginehasattrnnr	   transformer_engine.pytorchpytorchnamed_children
isinstanceLinearr   weightappendr   anyshapein_featuresout_featuresdtypecopy_setattrnormalized_shaper   convert_model)modelr   r   r   tenamemodulehas_biasparams_to_gather	te_module
new_modules              r   r+   r+      s5    +,^__-r;'<<BL/,,. 2ffbii(-B{{$.H & ''4#$4AF 
0@FMM,?,?@@
0 
0 II&&(;(;(Y_YfYfYlYl & 	   &&v}}5NN((5tY/
0 
0 -2GK#V]]FKK$@PQR 6!;;d2LL)@)@)CbhbobobubuLv	  &&v}}5NN((56 E4+		*3H_{{$.H""F$7$7hU[UbUbUhUhJ ##FMM2%%fkk2E4,-6KP[f&=&=a&@fjj_e_l_l_r_rsJ##FMM2OO!!&++.E4,&; /'	[2
0 
06 6s    	(P;BPBP'P$	'P1	c                     t               st        d      t               rddl}|j                  }n)ddlm} |j                  |j                  |j                  f}| j                         D ]  }t        ||      s y y)zS
    Returns whether a given model has some `transformer_engine` layer or not.
    zRUsing `has_transformer_engine_layers` requires transformer_engine to be installed.r   NTF)r   r   r   r   r    r   r   r	   TransformerLayermodulesr   )r,   r-   module_cls_to_checkms       r   has_transformer_engine_layersr9   _   so     +,noo- ii/!||RYY8K8KL]]_ a,-     c                      t               st        d      t               rddlm nddlm  fd} |_        |S )z
    Wrapper for a model's forward method to apply FP8 autocast. Is context aware, meaning that by default it will
    disable FP8 autocast during eval mode, which is generally better for more accurate metrics.
    zLUsing `contextual_fp8_autocast` requires transformer_engine to be installed.r   )fp8_autocastc                 t    xs | j                   } |      5   |i |cd d d        S # 1 sw Y   y xY w)N)enabled
fp8_recipe)training)selfargskwargsr>   r<   r?   model_forwarduse_during_evals       r   forwardz(contextual_fp8_autocast.<locals>.forward   s=    !2T]]'jA 	2 $1&1	2 	2 	2s   .7)r   r   r   r   r<   r   __wrapped__)rD   r?   rE   rF   r<   s   ``` @r   contextual_fp8_autocastrH   v   s7    
 +,hii9;2 (GNr:   c                    t               st        d      t               rddlm} d}d}n,ddlmc m} ddlm} |j                  j                         \  }}||j                         ni }d|v rt        |j                  |d         |d<   |j                  dd      }|j                  dd      }|r|st        d	|       |r1d
|v rt        d      d|v rt        d       |j                   di |}	n |j"                  di |}	t%        | j&                  |	|      }
t)        | j&                  d      rt+        |
|       | _        | S |
| _        | S )zC
    Applies FP8 context manager to the model's forward method
    zGUsing `apply_fp8_autowrap` requires transformer_engine to be installed.r   NFz,MXFP8 block scaling is not available on HPU.
fp8_formatuse_autocast_during_evaluse_mxfp8_block_scalingz&MXFP8 block scaling is not available: amax_compute_algoz=`amax_compute_algo` is not supported for MXFP8 block scaling.amax_history_lenz<`amax_history_len` is not supported for MXFP8 block scaling.__func__r   )r   r   r   intel_transformer_engine.reciperecipe transformer_engine.common.recipecommonr   r   fp8check_mxfp8_support	to_kwargsgetattrFormatpop
ValueErrorMXFP8BlockScalingDelayedScalingrH   rF   r   r   )r,   fp8_recipe_handler	te_recipeis_fp8_block_scaling_availablemessager-   rC   rE   rL   r?   new_forwards              r   apply_fp8_autowraprb      s\    +,cdd;).&@ 	=</24&&2L2L2N/&/A/M))+SUFv&y'7'79MN|jj!;UCO$jj)BEJ'EA'KLL&(\]]'[\\0Y00:6:
-Y--77
)%--_UKu}}j)";6 L $Lr:   )TTT)F)typesr   torch.nnr   importsr   r   
operationsr   r+   r9   rH   rb   r   r:   r   <module>rg      s)      F *BJ.0,r:   