i ddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZmZddlZddlmZd d lmZmZmZmZd d lmZd d lmZmZd d lmZm Z m!Z!d dl"m#Z#ee$Z%dZ&dZ'd0dZ(d0dZ)dZ*d1dZ+d1dZ,d2dZ-d1dZ.d3de/de/de0fdZ1 d4de/de/de0de0fdZ2dejfjhd ejjfd!Z6dejfjhd"e7fd#Z8d$ejrjtd%e7fd&Z;dejfjhfd'Zd+Z?d,e7d(e7fd-Z@d.ee ejfjhe/fd(eAejfjfd/ZCy)5N) defaultdict)Iterable) nullcontext)Path)CallableUnion) get_logger)FSDP_MODEL_NAMEOPTIMIZER_NAMESAFE_WEIGHTS_NAME WEIGHTS_NAME)get_module_class_from_name)get_non_persistent_buffers is_peft_model)get_module_children_bottom_upis_compiled_modulesave)is_torch_versionctdtjvrdtjd<dtjd<y)z[ Enables RAM efficient loading of Hugging Face models for FSDP in the environment. ACCELERATE_USE_FSDPTrueFSDP_CPU_RAM_EFFICIENT_LOADINGNosenvirone/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/accelerate/utils/fsdp_utils.py!enable_fsdp_ram_efficient_loadingr!'s. BJJ.,2 ()39BJJ/0rc*dtjd<y)z\ Disables RAM efficient loading of Hugging Face models for FSDP in the environment. FalserNrrrr "disable_fsdp_ram_efficient_loadingr$1s4;BJJ/0rc|r%t|rddlm}|||jS|ddlm}|||S|j S)Nr)get_peft_model_state_dict adapter_name)get_model_state_dictoptions)rpeftr&active_adapter'torch.distributed.checkpoint.state_dictr) state_dict)model adapter_only sd_optionsr&r)s r _get_model_state_dictr38sI e,2(U=Q=QRRP#E:>>!!rc|r&t|rddlm}||||jS|ddlm}||||S|j |S)Nr)set_peft_model_state_dictr')set_model_state_dictr*)rr,r5r-r.r6load_state_dict)r0r/r1r2r5r6s r _set_model_state_dictr8GsO e,2( I]I]^^P#E:zJJ$$Z00rc d}|jdk(rXddlm}ddlm}||j |j k(t|jddt|jdd}|S) Nr r)StateDictOptions StateDictTypeoffload_to_cpuF rank0_only)full_state_dict cpu_offloadbroadcast_from_rank0) fsdp_versionr.r:2torch.distributed.fsdp.fully_sharded_data_parallelr<state_dict_typeFULL_STATE_DICTgetattrstate_dict_config) fsdp_pluginr2r:r<s r _prepare_sd_optionsrIVsjJ1$LT%'77=;X;XX = =?OQVW!()F)F V[!\ rcddlmcm}ddlm}ddlm}ddlm} tj|d|j| jk(r1|jdkD} | |j_| |j_|j dk(r2|j||j|j|j"n t%} t'|} | 5t)||| } |j| jk(r|dk(r t*d n t*d |d }tj,j/||}|j0dk(rt2j5d |t7j8| |t2j5d |ni|j| j:k(r|dk(rt*d |j0d nt*d |d |j0d }tj,j/||}t2j5d |t7j8| |t2j5d |n|j| j<k(rtj,j/|t*d |}tj|dt2j5d |d| i} |j9| |j?||t2j5d |dddy#1swYyxYw)NrDefaultSavePlannerFullyShardedDataParallelr;Texist_okr r1r2.bin_zSaving model to zModel saved to _rankr0r/storage_writerplanner) torch.distributed.checkpoint distributed checkpoint,torch.distributed.checkpoint.default_plannerrLrCrNr<rmakedirsrDrE num_processesrGr=r>rBoptim_state_dict_configrrIr3r pathjoin process_indexloggerinfotorchrLOCAL_STATE_DICTSHARDED_STATE_DICTFileSystemWriter)rH acceleratorr0 output_dir model_indexr1dist_cprLFSDPr<is_multi_processctxr2r/ weights_nameoutput_model_fileckpt_dirs r save_fsdp_modelrrgs22OcPKK T*""m&C&CC'44q87G %%43C %%0  # #q (  ;.. 0M0M{OrOr ] %[1J 6*5|Xbc  & &-*G*G G7Ba7Go.d3P_O``abmannrMsL " Z F ((A- ./@.ABC :'89 o.?-@AB  ( (M,J,J J!###5)B)B(C4H'(+eK ? JJz#4 5 KK/*;)<= >  ( (M,L,L Lww||J?2C1[M0RSH KK4 0 KK*8*5 6!:.J LL%&77A*,   KK/(4 5?666s H#LL cddlmcm}ddlm}ddlm}ddlm} |j|j| jk(r1|jdkD} | |j_ | |j_|jdk(r2|j||j|j|j n t#} t%|} | 5|j| jk(rt'||urK|j(dk7r<|j*s0|j,s|jdk(r t/d dddy|dk(r t0dn t0d|d} t2j4j7|| }t8j;d ||j* xs |j<}|rt?j@|d }ni}t8j;d |no|j| jBk(r|dk(rt0d |j(dnt0d|d |j(d} t2j4j7|| }t8j;d |t?j@|d }t8j;d |n|j| jDk(rt0|vr)t2j4j7|t0d|n|}t8j;d |dtG||| i}|jA||jI|||d}t8j;d |tK||| }ddd|S#1swYSxYw)Nr)DefaultLoadPlannerrMr;r zzSet the `sync_module_states` flag to `True` so that model states are synced across processes when initializing FSDP objectrRrSzLoading model from T weights_onlyzModel loaded from rTr0rQ)r/storage_readerrW)&rXrYrZr[rtrCrNr<wait_for_everyonerDrEr]rGr=r>rBr^rrItyperais_fsdp2sync_module_states ValueErrorr rr_r`rbrcis_main_processrdloadrerfr3FileSystemReaderr8)rHrhr0 input_dirrjr1rkrtrlr<rmrnr2roinput_model_file load_modelr/rq load_results r load_fsdp_modelrsi22OcP!!#""m&C&CC'44q87G %%43C %%0  # #q (  ;.. 0M0M{OrOr ] %[1J -q  & &-*G*G GE{$&;+D+D+IR]RfRf"55+:R:RVW:W$3-q-q8Ca7Go.d3P_O``abmannrMsL!ww||I|D  KK-.>-?@ A(111P[5P5PJ"ZZ(8tL  KK,-=,>? @  ( (M,J,J J!###5)B)B(C4H'(+eK-?@ A$44HJ KK,-=,>? @  ( (M,L,L L&&y8 Y?*;1[M(JK  KK-hZ8 9!#8\fp#qrJ LL%&77A*,   $G,J KK,XJ7 8+E:Leop [-q\ ]-q\ sA)M+H!M++M5c0ddlmcm}ddlm}ddlm}ddlm} tj|d|jdk(r2|j||j|j|jn t} t|} | 5|jdk(rdd lm} | ||| } n|j%||} |j| j&k(r|j(dk(r|dk(r t*d n t*d |d }tj,j/||}t0j3d |t5j6| |t0j3d|ntj,j/|t*d |}tj|dt0j3d ||j7d| i|j9||t0j3d|dddy#1swYyxYw)NrrKrMr;TrOr r )get_optimizer_state_dictr*rRrSzSaving Optimizer state to zOptimizer state saved in optimizerrU)rXrYrZr[rLrCrNr<rr\rBrDrGr^rrIr.roptim_state_dictrErar r_r`rbrcrdrrg)rHrhrr0rioptimizer_indexrkrLrlr<rnr2r optim_stateoptim_state_nameoutput_optimizer_filerqs r save_fsdp_optimizerrs22OcPKK T*  # #q (  ;.. 0M0M{OrOr ] %[1J @  # #q ( X25)ZXK//yAK  & &-*G*G G((A-/>!/C~&d+NK[[\]l\mmqIr!)+ ZAQ(R% 89N8OPQ ;(=> 78M7NOPww||J>2B!OCT0UVH KK4 0 KK4XJ? @ LL'5&77A*,   KK3H:> ?5@@@s E8H  Hcxddlmcm}ddlm}ddlm} |j |jdk(r2|j||j|j|jn t} t|} | 5|j| jk(rd} |jdk(s|jjs |dk(r t dn t d|d} t"j$j'|| }t(j+d|t-j.|d } t(j+d |nt |vr)t"j$j'|t d|n|}t(j+d |d |j1i} |j/| ||j3| | d } t(j+d||jdk(r&|j5||| }|j7|nddlm}|||| | dddy#1swYyxYw)NrrMr;r rRrSzLoading Optimizer state from TruzOptimizer state loaded from zLoading Optimizer from r) checkpoint_idrwzOptimizer loaded from )r0optimr)set_optimizer_state_dictr*)rXrYrZrCrNr<rxrBrDrGr^rrIrErar>r rr_r`rbrcrdr~r/roptim_state_dict_to_loadr7r.r)rHrhrr0rrr1rkrlr<rnr2roptimizer_nameinput_optimizer_filerq flattened_osdrs r load_fsdp_optimizerrs'22cP!!#  # #q (  ;.. 0M0M{OrOr ] %[1J !X  & &-*G*G GK((A-[5X5X5c5c/>!/C~&d+NK[[\]l\mmqIr(*ww||I~'N$ ;*:!O;L(MN  KK1(< =& (<(<(>?K LL&&77A   &k2K KK0 ; <  # #q ( 99Yit9uM  % %m 4 X $UI{J WC!X!X!Xs >F)H00H9checkpoint_dir save_pathsafe_serializationcpddlmcm}ddlmcmcm}i}t |}|j d|j||j||jd|r |tz n|tz }t|jdk(r|t|d}t||||S)z Passthrough to `torch.distributed.checkpoint.format_utils.dcp_to_torch_save` Will save under `save_path` as either `model.safetensors` or `pytorch_model.bin`. rNTrO)rwrWno_distr )r)rXrYrZ)torch.distributed.checkpoint.format_utils format_utilsrmkdir_load_state_dictr_EmptyStateDictLoadPlannerrrlenkeyslistr)rrrrkdist_cp_format_utilsr/s r )_distributed_checkpoint_to_merged_weightsrLs32LLJYI OOTO"))//?$??A * 2D --UaIaI :?? "Z 0 34 Y3EF r output_pathremove_checkpoint_dirct|}ddlm}tdds t d|j s|dz j }|dz j }d|d }|r#|r!|d z }|d |d |d z }|dz }t ||r|dz }|d|dz }t ||r|dz }|d|dz }t ||}|j rltjd|t|||} tjd| |r-tjd|tj||jy)a? Merge the weights from sharded FSDP model checkpoints into a single combined checkpoint. Should be used if `SHARDED_STATE_DICT` was used for the model. Weights will be saved to `{output_path}/model.safetensors` if `safe_serialization` else `pytorch_model.bin`. Note: this is a CPU-bound process. Args: checkpoint_dir (`str`): The directory containing the FSDP checkpoints (can be either the model or optimizer). output_path (`str`): The path to save the merged checkpoint. safe_serialization (`bool`, *optional*, defaults to `True`): Whether to save the merged weights with safetensors (recommended). remove_checkpoint_dir (`bool`, *optional*, defaults to `False`): Whether to remove the checkpoint directory after merging. r) PartialStatez>=z2.3.0z/`merge_fsdp_weights` requires PyTorch >= 2.3.0`pytorch_model_fsdp_0 optimizer_0zTried to load from z) but couldn't find a valid metadata file.zE However, potential model and optimizer checkpoint directories exist.zPlease pass in either z/pytorch_model_fsdp_0 or z /optimizer_0zinstead.z8 However, a potential model checkpoint directory exists.zPlease try passing in z/pytorch_model_fsdp_0 instead.z< However, a potential optimizer checkpoint directory exists.z/optimizer_0 instead.zMerging FSDP weights from z.Successfully merged FSDP weights and saved to z"Removing old checkpoint directory N) raccelerate.staterrr|existsr}rbrcrshutilrmtreerx) rrrrrmodel_path_existsoptimizer_path_existserrstaters r merge_fsdp_weightsrhs(.)N- D' *JKK  "+.DDLLN!/-!? G G I#N#33\] !6 Z ZC +N+;;TUcTddpq qC : Co  M MC +N+;;YZ ZCo# Q QC +N+;;PQ QCo NE  00@AB=nk[mn  DYKPQ KK<^.param_init_fn_tied_params$D) ///> 3HAu%yL(bi(//2 3 v&%2$7$7$9 7 FL* 7 $V,=,36:+FL(FJ6 7 7 r)rFsplitr` get_submodulerrdnnModule) rr0r _tied_namesnamermodrrrs ` @r ensure_weights_retiedrs%!5t._infer_parameter_dtypes =55jAI%,E?$C! !:!g{?P?PTYTgTg?g    . .7M%OOM$B)@)@)BMQQ =0:0A0A#q0I -O-++O._cast_and_contiguouss.  YYUY+F &&(F r)srcgroup)rrT)assign)torch.distributedrYtorch.distributed.tensorrrr/r}ziprvalues device_meshdetachr device_type isinstanceto_local broadcastrWORLD placementsrdemptysizerr7)rhr0rdistrrmeta_sharded_sd sharded_sdrrr full_param sharded_paramrsharded_tensorrr full_tensors r fsdp2_load_full_state_dictrs%C&&(OJR$""7:7==?OLbLbLd7e 4 3 $Zm'33K#**,// 0G0GHJ*g.(002 NN:1DJJ4D4DN E.z; H`H`aN+A, (M= 2.-Q^_N%3Jz "! 4&*9)>)>)@ 4 %J '33K++m&8&8&:;CZCZbobubuvK NN;ATZZ5E5EN F.{KIaIabN+A, (M= 2.-Q^_N%3Jz " 4 *T2 Lrrmappingcddlm}i}d||< |jD]%}|dDcgc]}||jc}|d<'ycc}w#t$r t dwxYw)a Switches the parameters of the optimizer to new ones (sharded parameters in usual case). This function modifies the optimizer in-place. Args: optimizer (`torch.optim.Optimizer`): Optimizer instance which contains the original model parameters mapping (`dict`): Mapping from the original parameter (specified by `data_ptr`) to the sharded parameter Raises: KeyError: If a parameter in the optimizer couldn't be switched to its sharded version. This should never happen and indicates a bug. If we kept the original params instead of raising, the training wouldn't be numerically correct and weights wouldn't get updated. r)r _local_tensorparamszA parameter in the optimizer couldn't be switched to its sharded version. This breaks the training. Please raise an issue on GitHub.N)rr param_groupsdata_ptrKeyError)rrraccessor_mapping param_groupps r !fsdp2_switch_optimizer_parametersrs~1 /W $11 YKBMhBW$XQWQZZ%8$XK ! Y$X   S   sA AA A Acdddlm}t|jj|}t |dddD]v\}}t |jddkDr|jdd\}}nd}|}|r|j|n|}||s[||d }|j||x|S) a8 Applies the activation checkpointing to the model. Args: accelerator (`Accelerator`): The accelerator instance model (`torch.nn.Module`): The model to apply the activation checkpointing to Returns: `torch.nn.Module`: The model with the activation checkpointing applied r)checkpoint_wrapperT) return_fqnsNrrr F)preserve_rng_state) ;torch.distributed.algorithms._checkpoint.checkpoint_wrapperrfsdp2_prepare_auto_wrap_policyrrHrrrrrregister_module) rhr0rauto_wrap_policy_func layer_namelayer parent_name child_name parent_modules r fsdp2_apply_acr >s;;;L;L;X;XZ_`:5dSTWUWX = E z$ % )&0&7&7Q&? #KK#J.policys%99Afe,C&DE Ercbtd|jD}|jkDS)Nc3<K|]}|jyw)N)numel).0rs r zAfsdp2_prepare_auto_wrap_policy..policy..s#K!AGGI#Ks)sum parametersmin_num_params)rmodule_num_paramsrGs r r\z.fsdp2_prepare_auto_wrap_policy..policys. ##Kv7H7H7J#K K $|'B'BB Br)torch.distributed.fsdp.wraprVrWauto_wrap_policyr functoolspartialfuncrFrrZsetrr|addrdrrbool) rGr0rVrWfnno_split_modulesrZ layer_classtransformer_clsr\r[s ` @r rrsf  & &B"i''( WW ))"5*=tD  #! (,-=(>%  5 5 A,8,V,V )"%%8 9K8 LO& #N{m[i!jkk # ' ' 8  9  F588?? Ft F M * * C588?? Ct C Mrc ddlm}|di|S)a Returns a `GradScaler` for FSDP2, as the current implementation of `get_grad_scaler` doesn't accept other args. We need this as current `get_grad_scaler` accepts only `distributed_type` as arg, which doesn't differentiate between FSDP1 and FSDP2 r) GradScalerr)torch.amp.grad_scalerrs)kwargsrss r get_fsdp2_grad_scalerrvs 1   r named_paramsc|jDcic]\}}|jdd|}}}|jDcic]+\}}|jdr|jddn||-}}}|jDcic]\}}|jdd|}}}|Scc}}wcc}}wcc}}w)a6Removes parameter name modifiers in order to map them back to their original names. See huggingface/accelerate#3554 for more context. Args: named_params (`dict`): The named parameters dictionary to canonicalize. Returns: `dict`: The canonicalized named parameters dictionary z._checkpoint_wrapped_modulez _orig_mod.z ._orig_mod)rreplace startswith)rwrLrMs r fsdp2_canonicalize_namesr|sQ]PbPbPde1AII;R@!CeLeXdXjXjXlPTPQSTq||L'A ,#q!KL@L?Q?Q?STtq!AIIlB/2TLT  fUsB/0B5 B;modulesc|| tSg}t|tretj|}g}|j D]9\}}|j |s|j||j|;|}|D]*}|jt|j,t|S)zConverts modules to parameters where modules can be a string or list of torch.nn.Module Args: modules (`Union[Iterable[torch.nn.Module], str]`): List of modules Returns: `set[torch.nn.Parameter]`: List of parameters ) rkrstrrecompile named_modules fullmatchrrextendrrc)r}r0rrcregmapped_modulesrrs r r9r9$su J'3jj!!//1 .LD&}}T" &!%%f- .!5$v002345 z?r)FN)rF)r)T)TF)Dr>rhrrrrD collectionsrcollections.abcr contextlibrpathlibrtypingrrrdloggingr constantsr r rr dataclassesrmodelingrrotherrrrversionsrr<rbr!r$r3r8rIrrrrrrrmrrrrrrdictrr Optimizerrr rTrrvr|rk Parameterr9rrr rs  #$"" WW3?JJ& H :; " 1"76tEP-@`0Xfccgk:kp44&)4?C4cg4n+$+$+$\O588??OTOd 1F1F QU >uxxBoEHHOOood22HV[V^V^VeVeUfhlUlLm2j 4D& 8EHHOO,c1 2  r