iddlmZddlmZmZddlZddlmcmZ ddlmZddl m Z ddl m Z ddlmZmZmZdd lmZdd lmZdd lmZmZdd lmZmZdd lmZmZddlm Z ddl!m"Z"m#Z#m$Z$ddl%m&Z&m'Z'ddl(m)Z)m*Z*ddl+m,Z,e)r ddl-m.Z.m/Z/nd\Z.Z/edGddej`Z1Gddej`Z2Gddej`Z3GddZ4dZ5ed d=d!Z6d"ejnd#e8d$ejnfd%Z9 d>d&ej`d'ejnd(ejnd)ejnd*ejndzd+e:d,e:d-e e"fd.Z;ee6Gd/d0ej`Ze?e>Z@Gd2d3ej`ZAGd4d5eZBe#Gd6d7eZCe#Gd8d9eCZDe#Gd:d;eCe ZEgd<ZFy)?)Callable)AnyOptionalN)nn)Cache)GenerationMixin)use_kernel_forward_from_hubuse_kernel_func_from_hubuse_kernelized_func)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)check_model_inputsmaybe_autocast)is_causal_conv1d_availableis_torchdynamo_compiling) Lfm2Config)causal_conv1d_fncausal_conv1d_updateNNRMSNormc,eZdZdfd ZdZdZxZS) Lfm2RMSNormct|tjt j ||_||_y)z: Lfm2RMSNorm is equivalent to T5LayerNorm N)super__init__r Parametertorchonesweightvariance_epsilon)self hidden_sizeeps __class__s p/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/transformers/models/lfm2/modeling_lfm2.pyr'zLfm2RMSNorm.__init__2s1 ll5::k#:; #c"|j}|jtj}|j dj dd}|tj ||jzz}|j|j|zS)NT)keepdim) dtypetor)float32powmeanrsqrtr,r+)r- hidden_states input_dtypevariances r1forwardzLfm2RMSNorm.forward:sy#)) %((7  $$Q',,R,>% Ht?T?T4T(UU {{]--k:::r2c^t|jjd|jS)Nz, eps=)tupler+shaper,r-s r1 extra_reprzLfm2RMSNorm.extra_reprAs* ))*+6$2G2G1HIIr2)gư>)__name__ __module__ __qualname__r'r@rE __classcell__r0s@r1r$r$0s$;Jr2r$c eZdZUejed<d deffd Ze d dedzde dde dzde d e ffd Z ejed ZxZS)Lfm2RotaryEmbeddinginv_freqNconfigct||j|_|j|_||_|j j d|_|j}|jdk7rt|j}||j |\}|_ |jd|d|jd|jdy)N rope_typedefaultrMF) persistentoriginal_inv_freq) r&r'max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrNrope_parametersrPcompute_default_rope_parametersrattention_scalingregister_bufferclone)r-rNdevice rope_init_fnrMr0s r1r'zLfm2RotaryEmbedding.__init__Hs "("@"@$*$B$B! 44[A!%!E!E >>Y &.t~~>L+7 V+L($( ZeD 0(..2BuUr2r\z torch.deviceseq_lenreturnz torch.Tensorc|jd}t|ddxs|j|jz}d}d|t j d|dtj j|tj|z zz }||fS) a Computes the inverse frequencies according to the original RoPE implementation Args: config ([`~transformers.PreTrainedConfig`]): The model configuration. device (`torch.device`): The device to use for initialization of the inverse frequencies. seq_len (`int`, *optional*): The current sequence length. Unused for this type of RoPE. Returns: Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). rope_thetahead_dimNg?rr4r7r\r7) rWgetattrr.num_attention_headsr)arangeint64r8float)rNr\r^basedimattention_factorrMs r1rXz3Lfm2RotaryEmbedding.compute_default_rope_parametersXs&%%l3fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjj k )))r2cN|jddddfjj|jdddj |j }|dddddfj}t |j jtr/|j jdk7r|j jnd}t|d5|j|jzjdd}tj||fd }|j|jz}|j|jz} dddj |j  j |j fS#1swYAxYw) Nrr5rmpscpuF) device_typeenabledr4rkrc)rMriexpandrCr8r\ isinstancetypestrr transposer)catcosrYsinr7) r-x position_idsinv_freq_expandedposition_ids_expandedrpfreqsembryrzs r1r@zLfm2RotaryEmbedding.forwardvsR!MM$4-8>>@GG HZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk  U C 5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C  5 vvAGGv$cff177f&;;;  5 5s BFF$NNNN)rFrGrHr)Tensor__annotations__rr' staticmethodrintrBrirXno_gradrr@rIrJs@r1rLrLEsllVzV $(+/"*T!*(*t* ~u$ % **:U]]_ <  ..:$'(G(GJ[([$\!$*$<$<&)A)AAAE&JbJbb%!))F..0AN))F..0AN))-v/A/ANr2c|jtj|j||j |zSr)rFsilurr)r-r{s r1r@zLfm2MLP.forwards/wwqvvdggaj)DGGAJ677r2)rFrGrHrr'r@rIrJs@r1rrs Oz O8r2rc eZdZdZdZdZdZdZejdfde de dejdejezdzfdZ dd ej d ej d e d eeefdzd eej ej ff dZdej*fdZdd e dzd e fdZdej d e d ee e ffdZde fdZd e fdZdZy)Lfm2HybridConvCachea Attention and conv cache for Lfm2. It stores the Key and Value states as a list of tensors, one for each layer. Attention layer cache shape: `[batch_size, num_heads, seq_len, head_dim]`. Conv layer cache shape: `[batch_size, hidden_size, L_cache-1]`. NFrNmax_batch_sizer7r\cg|_g|_||_|j|_|jj d|_|j |_||_g|_|tj|nd}t|jD]}tj|j|j|j |j|}|jj||jjtj g|jjtj gy)Nfull_attention)r7r\) key_cache value_cacher layer_typesindexfirst_attention_layer conv_L_cache_dtype conv_cacher)r\rangenum_hidden_layerszerosr.appendtensor)r-rNrr7r\_ conv_states r1r'zLfm2HybridConvCache.__init__s ,!--%)%5%5%;%;>) $ * * , 1(2DNN9 %*6D  Y '(- 4>>)3Lj2Y_a(bDNN9 %*/))T5E5Ei5PR^4_eg*hD  Y '~~i($*:*:9*EEEr2beam_idxctt|jD]:}|j|jr|j|j}|j|j d|j ||j|<|j|j}|j|j d|j ||j|<|j|js|j|j}|j|j d|j ||j|<=y)zDReorders the cache for beam search, given the selected beam indices.rN) rlenrrr\ index_selectr8rr)r-rrr\s r1 reorder_cachez!Lfm2HybridConvCache.reorder_caches#s4>>23 mI~~i(..0 299,0NN9,E,R,RSTV^VaVabhVi,jy))))4;;.2.>.>y.I.V.VWXZbZeZeflZm.n  +y)//13::-1__Y-G-T-TUVX`XcXcdjXk-l * mr2c|j|dk7r |jn|}t|j|ks |j|j dk(ry|j|j dS)zYReturns the sequence length of the cached states. A layer index can be optionally passed.rrr)rrrrrrCr-rs r1get_seq_lengthz"Lfm2HybridConvCache.get_seq_lengthsm372B2B92MQa2aD..gp t~~ ) +t~~i/H/N/N/PTU/U~~i(..r22r2cache_positioncVd}|jd}|j}||z}||fS)aB Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns (i.e. sliding_window, chunk_size), for each layer. r)rCr)r-rrfull_mask_kv_offset query_lengthpast_seen_tokens kv_lengths r1get_mask_sizesz"Lfm2HybridConvCache.get_mask_sizess@ %++A. ..0 #33 ---r2 max_lengthc|dkr|jt|z }|j|krytt|jD]l}|j|j s!|j|dd|ddf|j|<|j |dd|ddf|j |<ny)z"Crop the cache to the given lengthrN.)rabsrrrrr)r-ridxs r1cropzLfm2HybridConvCache.crop s >,,.Z@J    J . T^^,- SC~~c"((*&*nnS&9#{ {A:M&Ns#(,(8(8(=c;J;PQ>Q(R  % Sr2c,t|jSr)rrrDs r1__len__zLfm2HybridConvCache.__len__s4>>""r2ctt|jD]}|j|j!yr)rrrzero_rs r1resetzLfm2HybridConvCache.resets4s4??34 /I OOI & , , . /r2r)r)rFrGrH__doc__ris_compileablerrr)r9rrr7r\rvr'rdictrrBr LongTensorrrrrrrr2r1rrsBNNIK #]],0 666{{ 6  s"T) 6F/3 FLLFllF F 38nt+ F u||U\\) * FB me&6&6 m3d 333 .U\\ .c .eTWY\T\o . Ss S##/r2rc|dd|jddzf}|d|jddzdf}tj| |fdS)z*Rotates half the hidden dims of the input..Nr5r4rr)rCr)rx)r{x1x2s r1 rotate_halfr sZ 3"!''"+"" " #B 3 q " " #B 99rc2YB ''r2rotary_pos_embc|j|}|j|}||zt||zz}||zt||zz}||fS)aApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. ) unsqueezer)qkryrz unsqueeze_dimq_embedk_embeds r1apply_rotary_pos_embr'sY& -- &C -- &C3w;q>C/0G3w;q>C/0G G r2r=n_repr_c|j\}}}}|dk(r|S|dddddddddfj|||||}|j|||z||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)rCrsreshape)r=rbatchnum_key_value_headsslenrbs r1 repeat_kvrAso 2?1D1D.E h z!!Qa"23::5BUW\^bdlmM  (;e(CT8 TTr2modulequerykeyvalueattention_maskscalingdropoutkwargsc Tt||j}t||j} tj||j dd|z} |#|ddddddd|j df} | | z} t jj| dtjj|j} t jj| ||j} tj| | } | j ddj} | | fS)Nr4rrr5)rkr7)ptrainingr)rnum_key_value_groupsr)matmulrwrCr functionalsoftmaxr9r8r7rr contiguous) rrrrrrrrrr attn_weights causal_mask attn_outputs r1eager_attention_forwardrMs3 ; ;$>&B\B\$\!}}d* ii 2 2F4N4NQUQ^Q^4^ejk ii 2 2F4N4NQUQ^Q^4^ejk ii 2 2F4N4NQUQ^Q^4^ejk  &"<"'$ t+ '$ -t3 '$ ((4/ '$ u||U\\D00 1'$r2rc|N|jddkDr<|jddkDr*|j}||dddddfzj|}|S)zm Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66 Nrr)rCr7r8)r=rr7s r1apply_mask_to_padding_statesrsa !n&:&:1&=&AnFZFZ[\F]`aFa##&1d )CCGGN r2c reZdZdedeffd Z d dejdedzdejdzdejdzfd Z d dejdedzdejdzdejdzfd Z d d ejdedzdejdzdejdzfd Z xZ S) Lfm2ShortConvrNrc 2t|||_||_|j|_|j |_tj|j|j|j |j|j|j dz |_ tj|jd|jz|j|_ tj|j|j|j|_y)Nr) in_channels out_channels kernel_sizegroupsrpaddingrr)r&r'rNrrL_cache conv_biasrrConv1dr.convrin_projrrs r1r'zLfm2ShortConv.__init__s  "** $$ II**++ %%LL1$   yy!3!3Q9K9K5KRVR[R[\  &"4"4f6H6HtyyY r2Nr{rrrct||}|j|jdd}|jdd\}}}||z}|jj j |jj jd|jj jd} |c|ddkDr[t|jd|j|j| |jjd} | jd} n|dtjj!||j"|j$dz df} |j|jj'| t)|| |jjd} || z} |j+| jddj-} | S)Nr5rrrrrr4) activation)rr!rwchunkr r+r sizer squeezerrrrrrpadrrCcopy_rrr) r-r{rrrBCxBCBx conv_weightsconv_outrys r1cuda_kernels_forwardz"Lfm2ShortConv.cuda_kernels_forwards )N ;ll1o''B/))A2)&1a Uyy'',,TYY-=-=-B-B1-EtyyGWGWG\G\]^G_`  &>!+>:  H ))"-H*]]..rDLL288B<4OQR3ST **4>>:@@L'L$))..UYZH L MM!++b"-88: ;r2c|jd}t||}|j|jdd}|j dd\}}}||z} |5|ddkDr,|j |j } |jd|jdz }| jdd} | j| j| j| dddd|f<|j |j j| tj| j| j|j j"dddddfzd} |j$r| |j j$z } | j'd} n~|dt(j*j-| |j| jdz df} |j |j j| |j!| d d|f} || z} | jddj/} |j1| } | S) Nrr5rrrrr)shiftsdimsrd.)rCrr!rwr$rrclamprrollr8r\r7r(r)sumr r+rrrrr'rr) r-r{rrrseqlenr)r*r+r,rr.r/s r1 slow_forwardzLfm2ShortConv.slow_forwards (N ;ll1o''B/))A2)&1a U  &>!+>:@@Lyy}S'6'\2H L KKB  * * , MM! r2r=ctr6d|jjvrts|j ||||S|j ||||S)Ncuda)is_fast_path_availabler\rurr0r8)r-r=rrrs r1r@zLfm2ShortConv.forwardsP "f 0D0D0I0I&IRjRl,,]O^]kl l  Q_``r2r)rFrGrHrrr'r)rrrr0r8r@rIrJs@r1rrsZZZ27;26.2  << -t3 ((4/   t+ J7;26.2 $ <<$-t3$((4/ $  t+ $R7;26.2 a|| a-t3 a((4/ a  t+ ar2rc eZdZdedeffd Z d dejdeejejfdzdejdzdejdzd e dzd ejdzd ejfd Z xZ S)Lfm2DecoderLayerrNrcft||j|dk(|_|jrt |||_nt |||_t||_ t|j|j|_ t|j|j|_y)Nrr)r&r'ris_attention_layerr self_attnrr r feed_forwardr$r.r operator_normffn_normrs r1r'zLfm2DecoderLayer.__init__ s "("4"4Y"?CS"S  " "*69=DN%fi8DI#FO(););Q#F$6$6FOOL r2Nr=rrr|rrr_c |}|jr,|jd|j||||||d|\}} n$|j|j||||}||z}||j |j |z}|S)N)r=rrr|rr)r=rrrr)r?r@rBr rArC) r-r=rrr|rrrresidualrs r1r@zLfm2DecoderLayer.forward,s!  " "-t~~ "00?$7-) /-    M1!II"00? /-- &M &0 %(9(9$-- :V(WW r2)NNNNN) rFrGrHrrr'r)rrBrrr@rIrJs@r1r=r=s Mz Mc MIM.2046:26||#5<<#=>E t+  &&-  -t3 ((4/ r2r=cJeZdZUeed<dZdZdgZdgZdZ dZ dZ dZ dZ eedZy) Lfm2PreTrainedModelrNmodelTr=rF)r= attentionsN)rFrGrHrrbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr=r_can_record_outputsrr2r1rGrGNsQ &*#+,#4"5N""&)#r2rGceZdZdeffd Zee ddejdzdejdzdejdzde dzdejdzd e dzd ejdzd e ed efd ZxZS) Lfm2ModelrNc  t|||j|_|j|_t j |j|j|j|_t jt|jDcgc]}t||c}|_ t||_d|_t#|j|j$|_|j)ycc}w)N)rNFr)r&r' pad_token_id padding_idx vocab_sizer Embeddingr. embed_tokens ModuleListrrr=layersrL rotary_embgradient_checkpointingr$rembedding_norm post_initrs r1r'zLfm2Model.__init__bs  !.. ++LL):):F|<|jd} t|j| |j |j }|F||jnd} tj| | |jdz|j }||jd}t|j|||||} |jddk7r|nd} |} |j| |}|jd|jjD]!}|jr| n| }|| f|||||d|} #|j!| } t#| | S) Nz:You must specify exactly one of input_ids or inputs_embedsr)rNrr7r\r)r\)rN input_embedsrrrr|)r|)rrr|rr)last_hidden_stater) ValueErrorr[rCrrNr7r\rr)rgrr r^r]rr?r`r)r-rbrr|rrcrdrr batch_sizerrlinear_attentionr=r decoder_layer layer_masks r1r@zLfm2Model.forwardrs  -t"; <YZ Z   --i8M 0&,,Q/J1{{:TZZX\XcXcO  !CRC^==?de "\\ "2]5H5H5K"KTaThThN  )33A6L(;;&))+%  .;-@-@-Cq-H>d% "oom,oW"[[)H4;;+H+HI M(5(H(HN^J))$7) /- M ++M: &++  r2)NNNNNNN)rFrGrHrr'rrr)rrr FloatTensorboolrrrr@rIrJs@r1rUrU`sz .2.2046:26!%26@ ##d*@  t+@ &&- @ -t3 @ ((4/ @ $;@ ((4/@ +,@  !@ @ r2rUcbeZdZddiZddiZddgdgfiZfdZee dd e jdzd e jdzd e jdzd e dzd e jdzde jdzdedzde jdzdee jzdeedefdZxZS)Lfm2ForCausalLMzlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputr=logitsct||t||_|j|_t j |j|jd|_|jy)NFr) r&r'rUrHrYrrr.rqra)r-rNr0s r1r'zLfm2ForCausalLM.__init__sU  v&  ++yy!3!3V5F5FUS  r2Nrbrr|rrclabelsrdrlogits_to_keeprr_c z|jd|||||||d| } | j} t| tr t | dn| } |j | dd| ddf}d}|*|j d|||jjd| }t||| j| j| jS)a Example: ```python >>> from transformers import AutoTokenizer, Lfm2ForCausalLM >>> model = Lfm2ForCausalLM.from_pretrained("meta-lfm2/Lfm2-2-7b-hf") >>> tokenizer = AutoTokenizer.from_pretrained("meta-lfm2/Lfm2-2-7b-hf") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```)rbrr|rrcrdrN)rsrurY)lossrsrr=rIr) rHrgrtrslicerq loss_functionrNrYrrr=rI)r-rbrr|rrcrurdrrvroutputsr= slice_indicesrsrxs r1r@zLfm2ForCausalLM.forwards@,64:: , )%+') ,  ,  11 8B>SV8W~ot4]k mA}a,?@A  %4%%pVFt{{OeOepiopD%#33!//))   r2) NNNNNNNNr)rFrGrH_tied_weights_keys_tp_plan_pp_planr'rrr)rrrrmrnrrrrr@rIrJs@r1rprps/*,GH23H_-z:;H.2.204(,26*.!%26-.8 ##d*8  t+8 &&- 8  8 ((4/ 8   4'8 $;8 ((4/8 ell*8 +,8  8 8 r2rp)rprUrG)r)r )Gcollections.abcrtypingrrr)torch.nn.functionalrrr cache_utilsr generationr integrationsr r r masking_utilsr modeling_layersrmodeling_outputsrrmodeling_rope_utilsrrmodeling_utilsrrprocessing_utilsrutilsrrr utils.genericrrutils.import_utilsrrconfiguration_lfm2r causal_conv1drr Moduler$rLrrrrrrrrirrrkernel_modulesallr;rr=rGrUrp__all__rr2r1rs:(%  )ff/9OKF&II?V*DD-7**Y'J"))J(J(><"))>