i4dZddlZddlmZddlmZmZmZddlmZ ddl m Z m Z m Z ddlmZmZddlmZmZmZmZmZmZmZmZdd lmZdd lmZdd lmZm Z m!Z!dd l"m#Z#m$Z$d dl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6ddl7m8Z8e!jre:Z;Gdde'Z<Gdde6Z=Gdde&Z>Gdde1Z?Gdde5Z@Gdde3ZAGdde(ZBe Gdd eZCGd!d"e4ZDGd#d$e-ZEGd%d&e,ZFGd'd(e2ZGGd)d*e)ZHGd+d,e+ZIGd-d.e/ZJGd/d0e*ZKGd1d2e0ZLGd3d4e.ZMgd5ZNy)6zPyTorch ERNIE model.N)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)Cache DynamicCacheEncoderDecoderCache)create_bidirectional_maskcreate_causal_mask),BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputNextSentencePredictorOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)Unpack)TransformersKwargsauto_docstringlogging)can_return_tuplecheck_model_inputs)BertCrossAttentionBertEmbeddings BertEncoderBertForMaskedLMBertForMultipleChoiceBertForNextSentencePredictionBertForPreTrainingBertForPreTrainingOutputBertForQuestionAnsweringBertForSequenceClassificationBertForTokenClassification BertLayerBertLMHeadModelBertLMPredictionHead BertModel BertPoolerBertSelfAttention) ErnieConfigceZdZdZfdZ d dej dzdej dzdej dzdej dzdejdzd ed ejfd Z xZ S) ErnieEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.ct|||j|_|jr0tj|j |j |_yy)N)super__init__ use_task_idnn Embeddingtask_type_vocab_size hidden_sizetask_type_embeddings)selfconfig __class__s q/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/transformers/models/ernie/modular_ernie.pyr4zErnieEmbeddings.__init__AsL  !--   (* V5P5PRXRdRd(eD % N input_idstoken_type_ids task_type_ids position_ids inputs_embedspast_key_values_lengthreturnc|||j}n|jdd}|\}} ||jdd|| |zf}|t|drT|jj |j dd} t j| d|} | j || }n:t j|t j|jj}||j|}|j|} |j| j}|| z} |j|} | | z} |jrR|:t j|t j|jj}|j!|}| |z } |j#| } |j%| } | S)NrArr.)dimindex)dtypedevice)sizerChasattrrAexpandshapetorchgatherzeroslongrLword_embeddingstoken_type_embeddingstoposition_embeddingsr5r: LayerNormdropout)r;r@rArBrCrDrE input_shape batch_size seq_lengthbuffered_token_type_idsrV embeddingsrXr:s r>forwardzErnieEmbeddings.forwardHs  #..*K',,.s3K!, J  ,,Q0FVlIl0l-lmL  !t-.*.*=*=*D*D\EWEWXYEZ\^*_'*/,,7NTU]i*j'!8!?!? J!W!&[ SWSdSdSkSk!l   00;M $ : :> J&(()>)E)EF "%:: "66|D"55    $ % KuzzRVRcRcRjRj k #'#<#<]#K . .J^^J/ \\*- r?)NNNNNr) __name__ __module__ __qualname____doc__r4rQ LongTensor FloatTensorintTensorr` __classcell__r=s@r>r1r1>sQf.226150426&'3##d*3((4/3''$. 3 &&- 3 ((4/ 3!$3 3r?r1c eZdZy)ErnieSelfAttentionNrarbrcr?r>rlrl~r?rlc eZdZy)ErnieCrossAttentionNrmrnr?r>rqrqror?rqc eZdZy) ErnieLayerNrmrnr?r>rsrsror?rsc eZdZy) ErniePoolerNrmrnr?r>rururor?ruc eZdZy)ErnieLMPredictionHeadNrmrnr?r>rwrwror?rwc eZdZy) ErnieEncoderNrmrnr?r>ryryror?rycneZdZeZdZdZdZdZdZ dZ e e e dZej fdZxZS)ErniePreTrainedModelernieT) hidden_states attentionscross_attentionsct||t|tr t j |j yt|tryt j|jtj|jjdjdt j |jyy)zInitialize the weightsrH)r.rHN)r3 _init_weights isinstancerwinitzeros_biasr1copy_rCrQarangerPrOrA)r;moduler=s r>rz"ErniePreTrainedModel._init_weightss f% f3 4 KK $  0 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h KK-- .1r?)rarbrcr/ config_classbase_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrsrlrq_can_record_outputsrQno_gradrrirjs@r>r{r{sXL&*#N"&#(/ U]]_//r?r{ceZdZdgZdfd Zee ddejdzdejdzdejdzdejdzdejdzd ejdzd ejdzd ejdzd e dzd e dzdejdzde e de ejezfdZdZxZS) ErnieModelrsct|||||_d|_t ||_t ||_|r t|nd|_ |jy)NF) r3r4r<gradient_checkpointingr1r_ryencoderrupooler post_init)r;r<add_pooling_layerr=s r>r4zErnieModel.__init__sU v& &+#)&1#F+ ->k&)D  r?Nr@attention_maskrArBrCrDencoder_hidden_statesencoder_attention_maskpast_key_values use_cachecache_positionkwargsrFc |jjr| | n|jj} nd} |jr%|jr| rt j dd} | rd| b||jjr4tt|jt|jnt|j} | | td|#|j|||j} n!||jdd} n td| \}}| |jn |j}| | jnd}| tj |||z| } |j#|||||| }|j%||||| | \}}|j&|f|||| | | |d | }|d}|j(|j)|nd}t+|||j, S) task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] NFzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...)r<zDYou cannot specify both input_ids and inputs_embeds at the same timerHz5You have to specify either input_ids or inputs_embedsr)rL)r@rCrArBrDrE)rrembedding_outputrrr)rrrrrrrC)last_hidden_state pooler_outputr)r< is_decoderrrtraininglogger warning_onceis_encoder_decoderr r ValueError%warn_if_padding_and_no_attention_maskrMrLget_seq_lengthrQrr__create_attention_masksrrr r)r;r@rrArBrCrDrrrrrrr[r\r]rLrErencoder_outputssequence_output pooled_outputs r>r`zErnieModel.forwards10 ;; ! !%.%:  @U@UII  & &4==##p" 0)4 8V8V$L $DlZ^ZeZeFfg!5   ]%>cd d  "  6 6y. Q#..*K  &',,.s3KTU U!, J%.%:!! @T@TETE`!?!?!Afg  !"\\*@BX[eBentuN??%)''#9+ 261M1M)#9-"7)+ 2N2 ..'$,,   )"7#9+)%    *!,8< 8O O4UY ;-'+;;  r?c|jjrt|j||||}nt|j||}|t|j|||}||fS)N)r< input_embedsrrr)r<rr)r<rrr)r<rr r )r;rrrrrrs r>rz"ErnieModel._create_attention_masks$sx ;; ! !/{{--- / N7{{--N " -%>{{-5&; & "555r?)T) NNNNNNNNNNN)rarbrc_no_split_modulesr4rrrQrhrboolrrtupler r`rrirjs@r>rrsR% *..2.2-1,0-1596:(,!%.2_ <<$&_  t+_  t+ _ ||d* _ llT) _ ||d*_  %||d2_ !& t 3_ _ $;_  t+_ +,_  u|| K K_ _ D 6r?rc eZdZy)ErnieForPreTrainingOutputNrmrnr?r>rrGror?rcbeZdZdddZee ddejdzdejdzdejdzdejdzd ejdzd ejdzd ejdzd ejdzd ee de eje zfdZ y)ErnieForPreTrainingcls.predictions.bias'ernie.embeddings.word_embeddings.weightzcls.predictions.decoder.biaszcls.predictions.decoder.weightNr@rrArBrCrDlabelsnext_sentence_labelrrFc |j|f|||||dd| } | dd\} } |j| | \} }d}|u|st}|| jd|jj |jd}||jdd|jd}||z}t || || j| jS)a: task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, ErnieForPreTraining >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.prediction_logits >>> seq_relationship_logits = outputs.seq_relationship_logits ``` TrrArBrCrD return_dictNrrH)lossprediction_logitsseq_relationship_logitsr}r~) r|clsrviewr< vocab_sizerr}r~)r;r@rrArBrCrDrrroutputsrrprediction_scoresseq_relationship_score total_lossloss_fctmasked_lm_lossnext_sentence_losss r>r`zErnieForPreTraining.forwardQs^$**   ))'%'    *1!&48HH_m4\11  "5"A')H%&7&<&rrKs(>*S *..2.2-1,0-1&*37H <<$&H  t+H  t+ H ||d* H llT) H ||d*H  t#H #\\D0H +,H  u|| 8 8H H r?rc"eZdZee ddej dzdej dzdej dzdej dzdej dzdej dzdej dzd ej dzd ej dzd eej dzd edzd ej dzde ej zde e de ej e zfdZy)ErnieForCausalLMNr@rrArBrCrDrrrrrrlogits_to_keeprrFc | d} |j|f|||||||| | | dd |}|j}t| tr t | dn| }|j |dd|ddf}d}| *|j d|| |jjd|}t|||j|j|j|jS)a task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` NFT) rrArBrCrDrrrrrr)logitsrr)rrrr}r~rrn)r|rrrgslicer loss_functionr<rrrr}r~r)r;r@rrArBrCrDrrrrrrrrrr} slice_indicesrrs r>r`zErnieForCausalLM.forwards<  I@J A ))'%'"7#9+)A A   11 8B>SV8W~ot4]k -=!(;<=  %4%%pVFt{{OeOepiopD0#33!//))$55   r?) NNNNNNNNNNNNr)rarbrcrrrQrhlistrrgrrrrr`rnr?r>rrsc*..2.2-1,0-1596:&*59!%.2-.? <<$&?  t+?  t+ ? ||d* ? llT) ? ||d*?  %||d2? !& t 3?  t#? ell+d2? $;?  t+? ell*? +,? u|| @ @!? ? r?rceZdZdddZee ddejdzdejdzdejdzdejdzd ejdzd ejdzd ejdzd ejdzd ejdzdee de eje zfdZ y)ErnieForMaskedLMrrrNr@rrArBrCrDrrrrrFc @|j|f|||||||dd| } | d} |j| } d}| Ft}|| jd|jj | jd}t || | j| jS)as task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` T)rrArBrCrDrrrrNrHrrr}r~) r|rrrr<rrr}r~)r;r@rrArBrCrDrrrrrrrrrs r>r`zErnieForMaskedLM.forwards4$**   ))'%'"7#9    "!* HH_5  ')H%&7&<&rrs(>*S *..2.2-1,0-1596:&*2 <<$&2  t+2  t+ 2 ||d* 2 llT) 2 ||d*2  %||d22 !& t 32  t#2 +,2  u|| ~ -2 2 r?rc8eZdZee d dej dzdej dzdej dzdej dzdej dzdej dzdej dzd eed e ej e zfd Z y) ErnieForNextSentencePredictionNr@rrArBrCrDrrrFc |j|f|||||dd|} | d} |j| } d} |2t} | | jdd|jd} t | | | j | j S)a task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring). Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt") >>> outputs = model(**encoding, labels=torch.LongTensor([1])) >>> logits = outputs.logits >>> assert logits[0, 0] < logits[0, 1] # next sentence was random ``` Trr.NrHrr)r|rrrrr}r~)r;r@rrArBrCrDrrrrseq_relationship_scoresrrs r>r`z&ErnieForNextSentencePrediction.forward!sZ$**   ))'%'      "&((="9!  ')H!)*A*F*Fr1*Mv{{[]!_ *#*!//))   r?NNNNNNN) rarbrcrrrQrhrrrrr`rnr?r>rr s*..2.2-1,0-1&*D <<$&D  t+D  t+ D ||d* D llT) D ||d*D  t#D +,D  u|| : :D D r?rc8eZdZee d dej dzdej dzdej dzdej dzdej dzdej dzdej dzd eed e ej e zfd Z y) ErnieForSequenceClassificationNr@rrArBrCrDrrrFc |j|f|||||dd|} | d} |j| } |j| } d} ||jj|j dk(rd|j_nl|j dkDrL|j tjk(s|j tjk(rd|j_nd|j_|jjdk(rIt} |j dk(r&| | j|j} n| | |} n|jjdk(r=t} | | jd|j |jd} n,|jjdk(rt} | | |} t| | | j | j" S) a^ task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Trr.N regressionsingle_label_classificationmulti_label_classificationrHr)r|rZ classifierr< problem_type num_labelsrKrQrTrgrsqueezerrrrr}r~)r;r@rrArBrCrDrrrrrrrs r>r`z&ErnieForSequenceClassification.forwardks0$**   ))'%'       ]3 /  {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+- B @&++b/R))-II,./'!//))   r?r) rarbrcrrrQrhrrrrr`rnr?r>rrjs*..2.2-1,0-1&*B <<$&B  t+B  t+ B ||d* B llT) B ||d*B  t#B +,B  u|| 7 7B B r?rc8eZdZee d dej dzdej dzdej dzdej dzdej dzdej dzdej dzd eed e ej e zfd Z y) ErnieForMultipleChoiceNr@rrArBrCrDrrrFc ||jdn|jd} |!|jd|jdnd}|!|jd|jdnd}|!|jd|jdnd}|!|jd|jdnd}|1|jd|jd|jdnd}|j|f|||||dd|} | d} |j | } |j | } | jd| } d}|t }|| |}t|| | j| jS)a9 input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) task_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr.rHTrr) rPrrMr|rZrrrr}r~)r;r@rrArBrCrDrr num_choicesrrrreshaped_logitsrrs r>r`zErnieForMultipleChoice.forwards`-6,Aiooa(}GZGZ[\G] >G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  $**   ))'%'       ]3 / ++b+6  ')HOV4D("!//))   r?r) rarbrcrrrQrhrrrrr`rnr?r>rrs*..2.2-1,0-1&*U <<$&U  t+U  t+ U ||d* U llT) U ||d*U  t#U +,U  u|| 8 8U U r?rc8eZdZee d dej dzdej dzdej dzdej dzdej dzdej dzdej dzd eed e ej e zfd Z y) ErnieForTokenClassificationNr@rrArBrCrDrrrFc J|j|f|||||dd|} | d} |j| } |j| } d} |r`z#ErnieForTokenClassification.forwards,$**   ))'%'    "!*,,71  ')HFKKDOOrr s*..2.2-1,0-1&*. <<$&.  t+.  t+ . ||d* . llT) . ||d*.  t#. +,.  u|| 4 4. . r?rcXeZdZee d dej dzdej dzdej dzdej dzdej dzdej dzdej dzd ej dzd eed e ej e zfd Z y)ErnieForQuestionAnsweringNr@rrArBrCrDstart_positions end_positionsrrFc |j|f|||||dd| } | d} |j| } | jdd\} }| jdj } |jdj }d}||t |j dkDr|jd}t |j dkDr|jd}| j d}|jd|}|jd|}t|}|| |}|||}||zd z }t|| || j| j S) rTrrr.rH)rIN) ignore_indexr)r start_logits end_logitsr}r~) r| qa_outputssplitr contiguouslenrMclamprrr}r~)r;r@rrArBrCrDrrrrrrrrr ignored_indexr start_lossend_losss r>r`z!ErnieForQuestionAnswering.forwardBs*$**   ))'%'    "!*1#)<<r<#: j#++B/::< ''+668   &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M-33A}EO)//=AM']CH!,@J M:H$x/14J+%!!//))   r?r) rarbrcrrrQrhrrrrr`rnr?r>rrAs*..2.2-1,0-1/3-1< <<$&<  t+<  t+ < ||d* < llT) < ||d*< ,< ||d*< +,<  u|| ; ;< < r?r) rrrrrrrrrr{)OrdrQtorch.nnr6rrrrr cache_utilsrr r masking_utilsr r modeling_outputsr rrrrrrrmodeling_utilsrprocessing_utilsrutilsrrr utils.genericrrbert.modeling_bertrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-configuration_ernier/ get_loggerrarr1rlrqrsrurwryr{rrrrrrrrrr__all__rnr?r>rs AA&CCJ   .&@@A&-   H %=n=@ *  ,    *  0  ; /?//2T6T6n 8 P ,P fB B J: : zG %BG TE %BE PX 2X v1 "<1 h? 8? D r?