idZddlmZddlmZddlZddlmZddlmZddl m Z dd l m Z dd lmZdd lmZmZmZdd lmZmZdd lmZddlmZmZmZmZddlmZej@e!Z"GddejFZ$ d;dejFdejJdejJdejJdejJdzde&de&fdZ'GddejFZ(GddejFZ)Gdd ejFZ*Gd!d"ejFZ+Gd#d$ejFZ,Gd%d&eZ-Gd'd(ejFZ.eGd)d*eZ/eGd+d,e/Z0Gd-d.ejFZ1Gd/d0ejFZ2eGd1d2e/Z3eed34Gd5d6eZ4ed74Gd8d9e/Z5gd:Z6y)S>STzz&"<"<=   ELL)G)GHOOPWXej  N input_idstoken_type_idsr inputs_embedsreturnc||j}n|jdd}|d}||jddd|f}|:tj|tj|jj }||j |}|j|}||z}|j|} || z }|j|}|j|}|S)Nrrdtypedevice) sizerr2zeroslongrBr'r+r)r,r0) r6r;r<rr= input_shape seq_lengthr+ embeddingsr)s r9forwardzSplinterEmbeddings.forward:s  #..*K',,.s3K ^  ,,Q  ^ J"%:: "66|D)) ^^J/ \\*- r:)NNNN) __name__ __module__ __qualname____doc__r"r2 LongTensor FloatTensortuplerI __classcell__r8s@r9rr)szQ  .2260426 ##d*((4/&&-  ((4/   r:rmodulequerykeyvalueattention_maskscalingr0c tj||jdd|z}|#|ddddddd|jdf} || z}tj j |dtjj|j}tj j|||j}tj||} | jddj} | |fS)Nrr)dimrA)ptrainingr) r2matmul transposeshaper functionalsoftmaxfloat32torAr0r^ contiguous) rSrTrUrVrWrXr0kwargs attn_weights causal_mask attn_outputs r9eager_attention_forwardrk\s<<s}}Q':;gEL!$Q1o " o%=> #k1 ==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K  $$r:c eZdZfdZ ddej dej dzdedzdeej fdZ xZ S) SplinterSelfAttentionc$t||j|jzdk7r2t |ds&t d|jd|jd||_|j|_t|j|jz |_|j|jz|_ tj|j|j|_ tj|j|j|_ tj|j|j|_tj|j |_|j |_|jdz|_y)Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()g)r!r"r%num_attention_headshasattr ValueErrorr7intattention_head_size all_head_sizerLinearrTrUrVr.attention_probs_dropout_probr0attention_dropoutrXr5s r9r"zSplinterSelfAttention.__init__usC     : : :a ?PVXhHi#F$6$6#78 445Q8   #)#=#= #&v'9'9Fc |jdd}g|d|j}|j|j|j dd}|j |j|j dd}|j |j|j dd} tj|jjt} | |||| |f|jsdn |j|jd|\} } | jg|dj!} |r| | f} | S| f} | S)NrrrZ)r0rX)rarurTviewr`rUrVr get_interfacer7_attn_implementationrkr^ryrXreshaperf)r6rzrWr{rgrF hidden_shape query_states key_states value_statesattention_interfacerjrhoutputss r9rIzSplinterSelfAttention.forwardsT$))#2. CCbC$*B*BC zz-055lCMMaQRS XXm,11,?II!QO zz-055lCMMaQRS (?(M(M KK , ,.E) %8      % $}}C$2H2HLL %  % ! \*k));;;;FFH 1B; -JUr:NF rJrKrLr"r2TensorrOboolrPrIrQrRs@r9rmrmtsW6048). ||))D0 $;  u||  r:rmcneZdZfdZdej dej dej fdZxZS)SplinterSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNr) r!r"rrwr%denser,r-r.r/r0r5s r9r"zSplinterSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= r:rz input_tensorr>cr|j|}|j|}|j||z}|SNrr0r,r6rzrs r9rIzSplinterSelfOutput.forward7 =1  ]3 }|'CD r:rJrKrLr"r2rrIrQrRs@r9rr1> U\\RWR^R^r:rc eZdZfdZ ddej dej dzdedzdeej fdZ xZ S) SplinterAttentioncbt|t||_t ||_yr)r!r"rmr6routputr5s r9r"zSplinterAttention.__init__s& )&1 (0 r:NrzrWr{r>c n|j|f||d|}|j|d|}|f|ddz}|SN)rWr{rr)r6r)r6rzrWr{rg self_outputsattention_outputrs r9rIzSplinterAttention.forwards\!tyy  )/    ;;|A F#% QR(88r:rrrRs@r9rrsW148). ||))D0 $;  u||  r:rcVeZdZfdZdej dej fdZxZS)SplinterIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r!r"rrwr%intermediate_sizer isinstance hidden_actstrr intermediate_act_fnr5s r9r"zSplinterIntermediate.__init__s] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $r:rzr>cJ|j|}|j|}|Sr)rr)r6rzs r9rIzSplinterIntermediate.forwards& =1 00? r:rrRs@r9rrs#9U\\ellr:rcneZdZfdZdej dej dej fdZxZS)SplinterOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) r!r"rrwrr%rr,r-r.r/r0r5s r9r"zSplinterOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= r:rzrr>cr|j|}|j|}|j||z}|Srrrs r9rIzSplinterOutput.forwardrr:rrRs@r9rrrr:rc eZdZfdZ d dej dej dzdedzdeej fdZ dZ xZ S) SplinterLayerct||j|_d|_t ||_t ||_t||_ y)Nr) r!r"chunk_size_feed_forward seq_len_dimr attentionr intermediaterrr5s r9r"zSplinterLayer.__init__sI '-'E'E$*6208$V, r:NrzrWr{r>c |j|f||d|}|d}|dd}t|j|j|j|}|f|z}|Sr)rrfeed_forward_chunkrr) r6rzrWr{rgself_attention_outputsrr layer_outputs r9rIzSplinterLayer.forwards"0 " )/"  "  2!4(,0  # #T%A%A4CSCSUe  /G+r:cL|j|}|j||}|Sr)rr)r6rintermediate_outputrs r9rz SplinterLayer.feed_forward_chunks,"//0@A{{#68HI r:r) rJrKrLr"r2rrOrrPrIrrQrRs@r9rrs\-48). ||))D0 $;  u||  .r:rceZdZfdZe d dej dejdzdedzdedzdedzde ej e zf d Z xZ S) SplinterEncoderct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}wr) r!r"r7r ModuleListrangenum_hidden_layersrlayergradient_checkpointing)r6r7ir8s r9r"zSplinterEncoder.__init__sN  ]]5IaIaCb#caM&$9#cd &+#$dsA#NrzrWr{output_hidden_states return_dictr>c |rdnd}|rdnd}t|jD])\} } |r||fz}| |||fi|} | d}|s!|| dfz}+|r||fz}t|||S)Nrrlast_hidden_staterz attentions) enumeraterr ) r6rzrWr{rrrgall_hidden_statesall_self_attentionsr layer_module layer_outputss r9rIzSplinterEncoder.forward"s#7BD$5b4(4 POA|#$58H$H!(! M*!,M &9]1=Mc  ||n|jj}||n|jj}||n|jj}| | t d|#|j |||j } n!||j dd} n t d| \} } | |jn |j} |tj| | f| }|&tj| tj| }|j|| }|j||||}|j||||d }|d }t||j |j" S) a token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) NzDYou cannot specify both input_ids and inputs_embeds at the same timerz5You have to specify either input_ids or inputs_embeds)rBr@)r;rr<r=T)rWr{rrrr)r7r{ruse_return_dictrs%warn_if_padding_and_no_attention_maskrCrBr2onesrDrEget_extended_attention_maskrHrr rzr)r6r;rWr<rr=r{rrrgrF batch_sizerGrBextended_attention_maskembedding_outputencoder_outputssequence_outputs r9rIzSplinterModel.forwardls:2C1N-TXT_T_TqTq$8$D $++JjJj &1%cd d  "  6 6y. Q#..*K  &',,.s3KTU U!, J%.%:!! @T@T  !"ZZ*j)A6RN  !"[[EJJvVN150P0PQ_al0m??%)' +  ,, 2/!5 ' *!,-)77&11  r:)NNNNNNNN)rJrKrLrMr"rrrrr2rrrPr rIrQrRs@r9rrTs /0*..2.2,0-1)-,0#'J <<$&J  t+J  t+ J llT) J ||d* J  $;J #TkJ D[J   J J r:rcXeZdZdfd Zdej dej fdZxZS)SplinterFullyConnectedLayerct|||_||_t j |j|j|_t||_t j|j|_ yr) r!r" input_dim output_dimrrwrr act_fnr,)r6rrrr8s r9r"z$SplinterFullyConnectedLayer.__init__sV "$YYt~~t? Z( doo6r:inputsr>cl|j|}|j|}|j|}|Sr)rrr,)r6rrzs r9rIz#SplinterFullyConnectedLayer.forwards2 6*  M2 }5 r:)gelurrRs@r9rrs#7ellu||r:rc(eZdZdZfdZdZxZS)QuestionAwareSpanSelectionHeadzf Implementation of Question-Aware Span Selection (QASS) head, described in Splinter's paper: ct|t|j|j|_t|j|j|_t|j|j|_t|j|j|_tj|j|jd|_ tj|j|jd|_ y)NF)bias) r!r"rr%query_start_transformquery_end_transformstart_transform end_transformrrwstart_classifierend_classifierr5s r9r"z'QuestionAwareSpanSelectionHead.__init__s %@ASASU[UgUg%h"#>v?Q?QSYSeSe#f :6;M;MvOaOab89K9KVM_M_` " &*<*P>PW\ ] ii(:(:F? ''1a0 ||M:> ++N; ##Aq!,\\-: Z''r:)rJrKrLrMr"rIrQrRs@r9rrs \(r:rc`eZdZfdZe ddej dzdej dzdej dzdej dzdej dzdejdzd ejdzd edzd edzd edzd ejdzde e zfdZ xZ S)SplinterForQuestionAnsweringct||t||_t ||_|j |_|jyrr!r"rrr splinter_qassquestion_token_idrr5s r9r"z%SplinterForQuestionAnswering.__init__C  %f- ;FC!'!9!9 r:Nr;rWr<rr=start_positions end_positionsr{rrquestion_positionsr>c | | n|jj} d} | |Dtjtj||j j d}nJtj|jdtj|j|j}|jd} d} |j||||||| | }|d}|j|| \}}| r"|jd |jd }}|d|d |z tj |j"j$zz}|d |z tj |j"j$zz}d}||t'|jd kDr|jd}t'|jd kDr|jd}|jd }|j)d||j)d|t+| }|||}|||}||zd z }| s||f|d dz}||f|zS|St-||||j.|j0 S) a token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be the only one for which start_logits and end_logits are calculated and they will be of shape `(batch_size, sequence_length)`. NFr)r\r)rAlayoutrBTrWr<rr=r{rrr ignore_indexrZlossrrrzr)r7rr2argmaxeqrrtrDrCrErrBrrrsqueezefinforAminlenclamp_rr rzr)r6r;rWr<rr=rrr{rrrrgquestion_positions_were_none"question_position_for_each_examplerrrr total_loss ignored_indexloss_fct start_lossend_lossrs r9rIz$SplinterForQuestionAnswering.forwardsH&1% @R@RST@U*L  %'1~+=\M_M_A`AdAd*ddL#q>'9U[[IYIY=Z=^=^&^^J  &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M  " "1m 4  M 2']CH!,@J M:H$x/14J"J/'!"+=F/9/EZMF* Q6 Q+%!!//))   r: NNNNNNNNNNN) rJrKrLr"rr2rrNrrPr rIrQrRs@r9rrs#*..2.2,0-13715)-,0#'6:b <<$&b  t+b  t+ b llT) b ||d* b ))D0b ''$.b  $;b #Tkb D[b ",,t3b  - -b b r:rzB Class for outputs of Splinter as a span selection model. ) custom_introceZdZUdZdZej dzed<dZej dzed<dZ ej dzed<dZ e ej dzed<dZ e ej dzed<y)SplinterForPreTrainingOutputa loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when start and end positions are provided): Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. start_logits (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length)`): Span-start scores (before SoftMax). end_logits (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length)`): Span-end scores (before SoftMax). Nr rrrzr) rJrKrLrMr r2rOrrrrzrPrrr:r9r2r2es|&*D%  d ")-1L%##d*1+/J!!D(/59M5**+d2926Je''(4/6r:r2z Splinter Model for the recurring span selection task as done during the pretraining. The difference to the QA task is that we do not have a question, but multiple question tokens that replace the occurrences of recurring spans instead. ceZdZfdZe ddej dzdej dzdej dzdej dzdej dzdejdzd ejdzd edzd edzd edzd ejdzde e zfdZ dej dej fdZ xZ S)SplinterForPreTrainingct||t||_t ||_|j |_|jyrrr5s r9r"zSplinterForPreTraining.__init__rr:Nr;rWr<rr=rrr{rrrr>c `| | n|jj} | | | td| | td| |j|} |j ||||||| | } | d}|j \}}}|j || \}}| j d}||jdj|||}|d|z tj|jjzz}|d|z tj|jjzz}d}|||jdtd|dz |jdtd|dz t|jj }||j#||z||j#||z}||j#||z||j#||z}||zdz }| s||f| ddz}||f|zS|St%|||| j&| j( S) a input_ids (`torch.LongTensor` of shape `(batch_size, num_questions, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. start_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be the only one for which start_logits and end_logits are calculated and they will be of shape `(batch_size, sequence_length)`. NzCquestion_positions must be specified in order to calculate the lossz>question_positions must be specified when input_embeds is usedrrrrrZr)r7r TypeError_prepare_question_positionsrrCrrr4r2r$rAr%r'maxrr&r~r2rzr)r6r;rWr<rr=rrr{rrrrgrrrsequence_lengthr\rr num_questions attention_mask_for_each_questionr*r,r-r.rs r9rIzSplinterForPreTraining.forwardsn&1%$? @ (T[[5M5MNH!!!*}"H%x/14J"J/'!"+=F/9/EZMF* Q6 Q+%!!//))   r:c4tj||jjk(\}}tj|}tj |j d|jf|jjtj|j}t|j d|j dk(dtj|Dcgc]}tj|c}}||||f<|Scc}w)Nrr@z?All samples in the batch must have at least one question token.)r2wherer7rbincountfullrCr9r&rErBrcatr3)r6r;rowsflat_positionsr;rncolss r9r8z2SplinterForPreTraining._prepare_question_positions s${{9 8U8U+UVnt, JJ ^^A  1 1 3 4 KK $ $**##     q !Y^^A%6 6 M yy=Aa%,,q/AB . $*Bs)Dr/)rJrKrLr"rr2rrNrrPr2rIr8rQrRs@r9r4r4|s?*..2.2,0-13715)-,0#'6:y <<$&y  t+y  t+ y llT) y ||d* y ))D0y ''$.y  $;y #Tky D[y ",,t3y  - -y y vU\\ellr:r4)rr4rrr)r})7rMcollections.abcr dataclassesrr2rtorch.nnrrr activationsr modeling_layersr modeling_outputsr r r modeling_utilsrr pytorch_utilsrutilsrrrrconfiguration_splinterr get_loggerrJloggerModulerrfloatrkrmrrrrrrrrrrrr2r4__all__rr:r9rVs$! %&!9ZZF6 3   H %//t% II% <<% % << % LL4' %  %%05BII5r   2 299  RYY #.#N* bii* Zioiic +c c L"))$#(RYY#(Ln #:n n b  7;7  7"V4VVr r: