
    id3              	       :   d dl Z d dlmZ d dlmZmZmZmZ d dlZej                  j                         r	d dlm	c mZ ddlmZmZmZmZ ddlmZ ddlmZ dd	lmZmZ  ee      Zd
ZdZe G d d             Zdej>                  j@                  dedee!ef   ddfdZ"dej>                  j@                  dee!ef   ddfdZ# G d de      Z$ G d de      Z% G d dejL                  jN                        Z( G d d      Z)dej>                  j@                  de!deej>                  j@                  eej>                  j@                     f   fdZ*dej>                  j@                  de!deej>                  j@                  eej>                  j@                     f   fdZ+y)     N)	dataclass)DictListTypeUnion   )ContextParallelConfigContextParallelInputContextParallelModelPlanContextParallelOutput)
get_logger)unwrap_module   )HookRegistry	ModelHookzcp_input---{}zcp_output---{}c                   B    e Zd ZU dZeeef   ed<   dZe	ed<   ddefdZ
y)ModuleForwardMetadataNcached_parameter_indices_cls
identifierc                    |xs i }||v r||   dd fS | j                   5| j                   j                  |d       }|t        d| d      ||   d|fS | j                  t        d      t	        t        j                  | j                  j                        j                  j                               }|dd  }t        |      D ci c]  \  }}||
 c}}| _         || j                   vrt        d| d      | j                   |   }|t        |      k\  rt        d| d	t        |       d
      ||   d|fS c c}}w )NTzParameter 'z' not found in cached indices.Fz$Model class is not set for metadata.r   z4' not found in function signature but was requested.z	Expected z arguments but got .)r   get
ValueErrorr   listinspect	signatureforward
parameterskeys	enumeratelen)selfr   argskwargsindexr   iparams           j/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/diffusers/hooks/context_parallel.py_get_parameter_from_args_kwargsz5ModuleForwardMetadata._get_parameter_from_args_kwargs0   sX   2*%tT11((41155j$GE} ;zl:X!YZZ;u,,99CDD'++DII,=,=>IINNPQ
^
BKJBW(Xha(X%T:::{:,6jkll--j9CIy/B3t9+QOPPE{E5(( )Ys   D;) N)__name__
__module____qualname__r   r   strint__annotations__r   r   r*   r+       r)   r   r   +   s+    /3d38n3D$)# )r2   r   moduleparallel_configplanreturnc           	         t         j                  d|j                   d|        |j                         D ]1  \  }}t	        | |      }t        |t              s|g}t         j                  d|dt        |       d       |D ]  }t        |t              r"t        ||      }t        j                  |      }nt        |t        t        t        f      rUt        |t              r|g}t        d |D              st        d|       t!        ||      }t"        j                  |      }nt        dt%        |             t'        j(                  |      }	|	j+                  ||        4 y	)
z"Apply context parallel on a model.z(Applying context parallel with CP mesh: z and plan: z*Applying ContextParallelHook to module_id=z identifying a total of z modulesc              3   <   K   | ]  }t        |t                y wN)
isinstancer   .0xs     r)   	<genexpr>z)apply_context_parallel.<locals>.<genexpr>d   s     WA:a)>?Ws   z?Expected all elements of cp_model_plan to be CPOutput, but got .Unsupported context parallel model plan type: N)loggerdebug_meshitems_get_submodule_by_namer:   r   r"   dictContextParallelSplitHook%_CONTEXT_PARALLEL_INPUT_HOOK_TEMPLATEformatr   tupleallr   ContextParallelGatherHook&_CONTEXT_PARALLEL_OUTPUT_HOOK_TEMPLATEtyper   check_if_exists_or_initializeregister_hook)
r3   r4   r5   	module_idcp_model_plan	submodulemhook	hook_nameregistrys
             r)   apply_context_parallelrW   N   sX    LL;O<Q<Q;RR]^b]cde$(JJL 4 	=*69=	)T*"IB	|C[\_`i\j[kkstu 	4A-./OAHHS	M,A4+OPm-BC%2OMWWW$'fgtfu%vww0PBII)T	 #QRVWdReQf!ghh#AA!DH""43	44r2   c                    |j                         D ]  \  }}t        | |      }t        |t              s|g}|D ]  }t	        j
                  |      }t        |t              rt        j                  |      }nHt        |t        t        t        f      rt        j                  |      }nt        dt        |             |j                  |         y )Nr?   )rC   rD   r:   r   r   rN   rE   rG   rH   r   rI   rL   r   rM   remove_hook)r3   r5   rP   rQ   rR   rS   rV   rU   s           r)   remove_context_parallelrZ   n   s    $(JJL , 	=*69=	)T*"I 	,A#AA!DH-.AHHS	M,A4+OPBII)T	 #QRVWdReQf!ghh  +	,,r2   c                   z     e Zd Zdededdf fdZd Zd Zd Zd	e	j                  d
ede	j                  fdZ xZS )rF   metadatar4   r6   Nc                 L    t         |           || _        || _        d | _        y r9   )super__init__r\   r4   module_forward_metadatar#   r\   r4   	__class__s      r)   r_   z!ContextParallelSplitHook.__init__   s%     .'+$r2   c                 R    t        |      j                  }t        |      | _        |S )N)r   )r   rb   r   r`   )r#   r3   clss      r)   initialize_hookz(ContextParallelSplitHook.initialize_hook   s$    F#--'<#'F$r2   c           	      D   t        |      }| j                  j                         D ]k  \  }}t        |t              r|j
                  r$| j                  j                  |||      \  }}}	|Ht        |t        j                        r| j                  ||      }nt        |t         t        f      rt        |      t        |      k7  r$t        dt        |       dt        |       d      g }
t        |      D ]O  \  }}t        j                  |      r$||   j
                  s| j                  |||         }|
j!                  |       Q |
}nt        dt#        |             |r|||<   H|	|	t        |      k  r|||	<   _t        d| d       t        |      |fS )Nz"Expected input model plan to have  elements, but got r   zUnsupported input type: z9An unexpected error occurred while processing the input 'z'. Please open an issue at https://github.com/huggingface/diffusers/issues and provide a minimal reproducible example along with the full stack trace.)r   r\   rC   r:   r
   split_outputr`   r*   torchTensor_prepare_cp_inputrI   r"   r   r!   	is_tensorappendrM   )r#   r3   r$   r%   	args_listnamecpm	input_valis_kwargr&   sharded_input_valr'   r=   s                r)   pre_forwardz$ContextParallelSplitHook.pre_forward   s   J	,,. '	ID##349I9I *.)E)E)e)ei*&Ix   )U\\2 229cB	Ie}5y>SX-$<S^<LL_`cdg`h_iijk  %'!%i0 0DAqq)#a&2E2E 221c!f=%,,Q/0 .	 #;DO;L!MNN(t"us9~'=#,	%  OPTv V? @ G'	R Y''r2   c           	         t        |t        j                        }t        |t        t        f      xr t        d |D              }|s|st        dt        |       d      |r|gn
t        |      }| j                  j                         D ]g  \  }}t        |t              r|j                  s#|t        |      k\  rt        d| dt        |       d      ||   }| j                  ||      }|||<   i |r|d   S t	        |      S )Nc              3   P   K   | ]  }t        |t        j                           y wr9   r:   ri   rj   r;   s     r)   r>   z8ContextParallelSplitHook.post_forward.<locals>.<genexpr>   s     Bocd:aQVQ]Q]C^Bo   $&CExpected output to be a tensor or a list/tuple of tensors, but got r   zIndex z$ out of bounds for output of length r   )r:   ri   rj   r   rI   rJ   r   rM   r\   rC   r
   rh   r"   rk   )r#   r3   outputrl   is_tensor_listr&   rp   current_outputs           r)   post_forwardz%ContextParallelSplitHook.post_forward   s   vu||4	#FT5M:osBohnBo?obcghncobppqrss&&DL----/ 	+JE3c#78@P@PF# 6%0TUXY_U`Taab!cdd#E]N!33NCHN*F5M	+ &vay85=8r2   r=   cp_inputc                 *   |j                   S|j                         |j                   k7  r6t        j                  d|j                    d|j                          d       |S t        j                  ||j                  | j                  j                        S )NzExpected input tensor to have z dimensions, but got z' dimensions, split will not be applied.)	expected_dimsdimr@   warning_onceEquipartitionShardershard	split_dimr4   _flattened_mesh)r#   r=   r~   s      r)   rk   z*ContextParallelSplitHook._prepare_cp_input   s    !!-!%%'X=S=S2S01G1G0HH]^_^c^c^e]f  gN  O H'--a1C1CTEYEYEiEijjr2   )r,   r-   r.   r   r	   r_   re   rt   r}   ri   rj   r
   rk   __classcell__rb   s   @r)   rF   rF      sZ    ,!9 ,La ,fj ,
,(\9&k5<< k;O kTYT`T` kr2   rF   c                   2     e Zd Zdededdf fdZd Z xZS )rK   r\   r4   r6   Nc                 >    t         |           || _        || _        y r9   )r^   r_   r\   r4   ra   s      r)   r_   z"ContextParallelGatherHook.__init__   s     .r2   c                 P   t        |t        j                        }|r|g}n@t        |t        t        f      rt        d |D              st        dt        |       d      t        |      }t        |      t        | j                        k7  r.t        dt        | j                         dt        |       d      t        | j                        D ]C  \  }}|	t        j                  ||   |j                  | j                  j                        ||<   E |r|d   S t	        |      S )Nc              3   P   K   | ]  }t        |t        j                           y wr9   rw   r;   s     r)   r>   z9ContextParallelGatherHook.post_forward.<locals>.<genexpr>   s     ;h\]Jq%,,<W;hrx   ry   r   zExpected output to have rg   r   )r:   ri   rj   r   rI   rJ   r   rM   r"   r\   r!   r   unshard
gather_dimr4   r   )r#   r3   rz   rl   r'   rp   s         r)   r}   z&ContextParallelGatherHook.post_forward   s   vu||4	XFVdE]3;hag;h8hbcghncobppqrssfv;#dmm,,7DMM8J7KK^_bci_j^kklmnn. 	vFAs{,44VAYPTPdPdPtPtuF1I	v
 &vay85=8r2   )r,   r-   r.   r   r	   r_   r}   r   r   s   @r)   rK   rK      s%    /!9 /La /fj /
9r2   rK   c                   ,    e Zd Zed        Zed        Zy)AllGatherFunctionc                     || _         || _        t        j                  j	                  |      | _        t        j                  j                  |      | _        t        j                  |||      S )N)group)
r   r   ri   distributedget_world_size
world_sizeget_rankrankfuncolall_gather_tensor)ctxtensorr   r   s       r)   r   zAllGatherFunction.forward   sU    	**99%@$$--e4''5AAr2   c                 ~    t        j                  || j                  | j                        }|| j                     d d fS )Nr   )ri   chunkr   r   r   )r   grad_outputgrad_chunkss      r)   backwardzAllGatherFunction.backward   s2    kk+s~~377K388$dD00r2   N)r,   r-   r.   staticmethodr   r   r+   r2   r)   r   r      s*    B B 1 1r2   r   c            	          e Zd Zedej
                  dedej                  j                  j                  dej
                  fd       Z
edej
                  dedej                  j                  j                  dej
                  fd       Zy)r   r   r   meshr6   c                     |j                         |   |j                         z  dk(  sJ d       |j                  |j                         |      t        j                  j	                  |j                                  S )Nr   zHTensor size along dimension to be sharded must be divisible by mesh sizer   )sizer   ri   r   r   	get_grouprd   r   r   r   s       r)   r   zEquipartitionSharder.shard  sj     {{}S!DIIK/14 	
V	
4 ||DIIKS|1%2C2C2L2LT^^M]2^__r2   c                 p    |j                         }t        j                  |||j                               }|S r9   )
contiguousr   applyr   r   s       r)   r   zEquipartitionSharder.unshard  s0    ""$"((dnn6FGr2   N)r,   r-   r.   classmethodri   rj   r0   r   device_mesh
DeviceMeshr   r   r+   r2   r)   r   r      s    
`5<< 
`c 
`9J9J9V9V9a9a 
`fkfrfr 
` 
` U\\  5;L;L;X;X;c;c hmhtht  r2   r   modelro   c                 X    |j                  d      dkD  rt        d      t        | |      S )N*r   z.Wildcard '*' can only be used once in the name)countr   _find_submodule_by_name)r   ro   s     r)   rD   rD     s+    zz#IJJ"5$//r2   c                    |dk(  r| S d|v r|j                  dd      n|df\  }}|dk(  rjt        | t        j                  j                        st        d      g }| D ]2  }t        ||      }t        |t              s|g}|j                  |       4 |S t        | |      rt        | |      }t        ||      S t        d| d| j                  j                   d      )N r   r   r   z-Wildcard '*' can only be used with ModuleList'z' is not a submodule of ')splitr:   ri   nn
ModuleListr   r   r   extendhasattrgetattrrb   r,   )r   ro   
first_atomremaining_name
submodulesrR   subsubmoduless          r)   r   r     s    rz7:d{C!3r
JS%!4!45LMM
 	-I3I~NMmT2!.m,		-
 5*%z2I*9nEEq,EeooF^F^E__`abbr2   ),r   dataclassesr   typingr   r   r   r   ri   r   is_available)torch.distributed._functional_collectives_functional_collectivesr   models._modeling_parallelr	   r
   r   r   utilsr   utils.torch_utilsr   hooksr   r   r,   r@   rG   rL   r   r   Moduler/   rW   rZ   rF   rK   autogradFunctionr   r   rD   r   r+   r2   r)   <module>r      s    ! * *  	!!#>>   - * 
H	(7 %)9 & ) ) )D4HHOO4*4 s,,
-4 
	4@,EHHOO ,4E]@];^ ,cg ,"Tky Tkn9	 961// 1 *0%((// 0 0uxxX\]b]e]e]l]lXmGmAn 0c588?? c# c%Y]^c^f^f^m^mYnHnBo cr2   