
    iP                     Z   d dl Zd dl mZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZ  G d dej>                        Z  G d dej>                        Z!	 	 d4dej>                  dejD                  dejD                  dejD                  dejD                  dz  de#dz  de#dee   fdZ$ G d dej>                        Z% G d dej>                        Z& G d d ej>                        Z' G d! d"ej>                        Z( G d# d$ej>                        Z) G d% d&e      Z*e G d' d(e             Z+ G d) d*ej>                        Z, G d+ d,ej>                        Z-e G d- d.e+             Z. ed/0       G d1 d2e+             Z/g d3Z0y)5    N)Callable   )initialization)ACT2FN)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstring	torch_int)can_return_tuplecheck_model_inputs   )IJepaConfigc                   f     e Zd ZdZdef fdZddej                  dedej                  fdZ	 xZ
S )	IJepaPatchEmbeddingsz
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    configc                    t         |           |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        t        j                  ||||      | _        y )Nr   r   )kernel_sizestride)super__init__
image_size
patch_sizenum_channelshidden_size
isinstancecollectionsabcIterablenum_patchesnnConv2d
projection)selfr   r   r   r   r    r%   	__class__s          r/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/transformers/models/ijepa/modeling_ijepa.pyr   zIJepaPatchEmbeddings.__init__   s    !'!2!2F4E4EJ
$*$7$79K9Kk#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$$(&))L+:^hi    pixel_valuesinterpolate_pos_encodingreturnc                    |j                   \  }}}}|| j                  k7  rt        d| j                   d| d      |sV|| j                  d   k7  s|| j                  d   k7  r2t        d| d| d| j                  d    d| j                  d    d		      | j	                  |      j                  d
      j                  dd
      }|S )NzoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got .r   r   zInput image size (*z) doesn't match model (z).   )shaper   
ValueErrorr   r(   flatten	transpose)r)   r-   r.   
batch_sizer   heightwidth
embeddingss           r+   forwardzIJepaPatchEmbeddings.forward.   s    2>2D2D/
L&%4,,,!../yaI  (++u8J/J (% 9+,Adooa.@-AE  __\2::1=GG1M
r,   F)__name__
__module____qualname____doc__r   r   torchTensorboolr<   __classcell__r*   s   @r+   r   r      s;    j{ jELL D ]b]i]i r,   r   c            	            e Zd ZdZddededdf fdZdej                  de	d	e	dej                  fd
Z
	 	 ddej                  dej                  dz  dedej                  fdZ xZS )IJepaEmbeddingszb
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
    r   use_mask_tokenr/   Nc                    t         |           |r4t        j                  t	        j
                  dd|j                              nd | _        t        |      | _	        | j                  j                  }t        j                  t	        j                  d||j                              | _        t        j                  |j                        | _        |j                   | _        || _        y )Nr   )r   r   r&   	ParameterrB   zerosr    
mask_tokenr   patch_embeddingsr%   randnposition_embeddingsDropouthidden_dropout_probdropoutr   r   )r)   r   rI   r%   r*   s       r+   r   zIJepaEmbeddings.__init__D   s    Q_",,u{{1a9K9K'LMei 4V <++77#%<<A{FL^L^0_#` zz&"<"<= ++r,   r;   r9   r:   c                 0   |j                   d   }| j                  j                   d   }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  }|j                   d   }|| j
                  z  }|| j
                  z  }	t        |dz        }
|j                  d|
|
|      }|j                  dddd      }t        j                  j                  |||	fdd	      }|j                  dddd      j                  dd|      }|S )
a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   g      ?r   r   r3   bicubicF)sizemodealign_corners)r4   rP   rB   jit
is_tracingr   r   reshapepermuter&   
functionalinterpolateview)r)   r;   r9   r:   r%   num_positionspatch_pos_embeddim
new_height	new_widthsqrt_num_positionss              r+   r.   z(IJepaEmbeddings.interpolate_pos_encodingN   s#    !&&q)0066q9 yy##%+*F6UZ?+++22r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nr,   r-   bool_masked_posr.   c                 x   |j                   \  }}}}| j                  ||      }|Z|j                   d   }	| j                  j                  ||	d      }
|j	                  d      j                  |
      }|d|z
  z  |
|z  z   }|r|| j                  |||      z   }n|| j                  z   }| j                  |      }|S )N)r.   r   rU   g      ?)	r4   rN   rM   expand	unsqueezetype_asr.   rP   rS   )r)   r-   rg   r.   r8   _r9   r:   r;   
seq_lengthmask_tokensmasks               r+   r<   zIJepaEmbeddings.forwardu   s     (4'9'9$
Avu**<Rj*k
&#))!,J//00ZLK",,R088ED#sTz2[45GGJ $#d&C&CJPVX]&^^J#d&>&>>J\\*-
r,   r=   NF)r>   r?   r@   rA   r   rD   r   rB   rC   intr.   
BoolTensorr<   rE   rF   s   @r+   rH   rH   ?   s    { D T %5<< % %UX %]b]i]i %T 48).	ll ))D0 #'	
 
r,   rH   modulequerykeyvalueattention_maskscalingrS   kwargsc                    ||j                  d      dz  }t        j                  ||j                  dd            |z  }|#|d d d d d d d |j                  d   f   }||z   }t
        j                  j                  |d      }t
        j                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )	NrU         r3   r   rc   )ptrainingr   )rW   rB   matmulr7   r4   r&   r^   softmaxrS   r   
contiguous)
rs   rt   ru   rv   rw   rx   rS   ry   attn_weightsattn_outputs
             r+   eager_attention_forwardr      s     **R.D( <<s}}Q':;gEL!'1a399R=(@A#n4==((2(>L==((6??([L,,|U3K''1-88:K$$r,   c                   z     e Zd Zdef fdZdej                  deej                  ej                  f   fdZ xZ	S )IJepaSelfAttentionr   c                 2   t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      || _        |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        |j                  | _        | j                  dz  | _        d| _        t        j                  |j                  | j                  |j                         | _        t        j                  |j                  | j                  |j                         | _        t        j                  |j                  | j                  |j                         | _        y )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads r1   r{   F)bias)r   r   r    num_attention_headshasattrr5   r   rq   attention_head_sizeall_head_sizeattention_probs_dropout_probdropout_probrx   	is_causalr&   Linearqkv_biasrt   ru   rv   r)   r   r*   s     r+   r   zIJepaSelfAttention.__init__   sF    : ::a?PVXhHi"6#5#5"6 7334A7 
 #)#=#= #&v'9'9F<V<V'V#W !558P8PP"??//5YYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
r,   hidden_statesr/   c           
         |j                   d   }|d| j                  | j                  f} | j                  |      j                  | j                  dd      } | j                  |      j                  | j                  dd      } | j                  |      j                  | j                  dd      }t        j                  | j                  j                  t              } || |||d | j                  | j                  | j                  sdn| j                         \  }}	|j#                         d d | j$                  fz   }
|j'                  |
      }||	fS )Nr   rU   r   r3           )r   rx   rS   r|   )r4   r   r   ru   r`   r7   rv   rt   r   get_interfacer   _attn_implementationr   r   rx   r   r   rW   r   r\   )r)   r   r8   	new_shape	key_layervalue_layerquery_layerattention_interfacecontext_layerattention_probsnew_context_layer_shapes              r+   r<   zIJepaSelfAttention.forward   sF   "((+
D$<$<d>V>VV	0DHH]+00)<FFq!L	4djj/44i@JJ1aP4djj/44i@JJ1aP(?(M(MKK,,.E)
 *=nnLL#}}C$2C2C	*
& #0"4"4"6s";t?Q?Q>S"S%--.EFo--r,   )
r>   r?   r@   r   r   rB   rC   tupler<   rE   rF   s   @r+   r   r      s:    ]{ ](.U\\ .eELL%,,<V6W .r,   r   c                   x     e Zd ZdZdef fdZdej                  dej                  dej                  fdZ xZ	S )IJepaSelfOutputz
    The residual connection is defined in IJepaLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r   c                     t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        y N)	r   r   r&   r   r    denserQ   rR   rS   r   s     r+   r   zIJepaSelfOutput.__init__   sB    YYv1163E3EF
zz&"<"<=r,   r   input_tensorr/   c                 J    | j                  |      }| j                  |      }|S r   r   rS   r)   r   r   s      r+   r<   zIJepaSelfOutput.forward   s$    

=1]3r,   
r>   r?   r@   rA   r   r   rB   rC   r<   rE   rF   s   @r+   r   r      s=    
>{ >
U\\  RWR^R^ r,   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )IJepaAttentionr   c                 b    t         |           t        |      | _        t	        |      | _        y r   )r   r   r   	attentionr   outputr   s     r+   r   zIJepaAttention.__init__   s&    +F3%f-r,   r   r/   c                 R    | j                  |      \  }}| j                  ||      }|S r   )r   r   )r)   r   self_attn_outputrl   r   s        r+   r<   zIJepaAttention.forward   s,    "nn];!-}=r,   	r>   r?   r@   r   r   rB   rC   r<   rE   rF   s   @r+   r   r      s*    .{ .
U\\ ell r,   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )IJepaIntermediater   c                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )r   r   r&   r   r    intermediate_sizer   r!   
hidden_actstrr   intermediate_act_fnr   s     r+   r   zIJepaIntermediate.__init__   s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$r,   r   r/   c                 J    | j                  |      }| j                  |      }|S r   )r   r   )r)   r   s     r+   r<   zIJepaIntermediate.forward  s&    

=100?r,   r   rF   s   @r+   r   r      s*    9{ 9U\\ ell r,   r   c                   t     e Zd Zdef fdZdej                  dej                  dej                  fdZ xZS )IJepaOutputr   c                     t         |           t        j                  |j                  |j
                        | _        t        j                  |j                        | _	        y r   )
r   r   r&   r   r   r    r   rQ   rR   rS   r   s     r+   r   zIJepaOutput.__init__  sB    YYv779K9KL
zz&"<"<=r,   r   r   r/   c                 T    | j                  |      }| j                  |      }||z   }|S r   r   r   s      r+   r<   zIJepaOutput.forward  s.    

=1]3%4r,   r   rF   s   @r+   r   r     s8    >{ >
U\\  RWR^R^ r,   r   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )
IJepaLayerz?This corresponds to the Block class in the timm implementation.r   c                 r   t         |           |j                  | _        d| _        t	        |      | _        t        |      | _        t        |      | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   eps)r   r   chunk_size_feed_forwardseq_len_dimr   r   r   intermediater   r   r&   	LayerNormr    layer_norm_epslayernorm_beforelayernorm_afterr   s     r+   r   zIJepaLayer.__init__  s    '-'E'E$'/-f5!&) "V-?-?VEZEZ [!||F,>,>FDYDYZr,   r   r/   c                     | j                  |      }| j                  |      }||z   }| j                  |      }| j                  |      }| j	                  ||      }|S r   )r   r   r   r   r   )r)   r   hidden_states_normattention_outputlayer_outputs        r+   r<   zIJepaLayer.forward%  si    !22=A>>*<= )=8 ++M:((6 {{<?r,   r   rF   s   @r+   r   r     s/    I[{ [U\\ ell r,   r   c                       e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdZeedZ ej$                         d	ej(                  ej*                  z  ej,                  z  d
dfd       Zy)IJepaPreTrainedModelr   ijepar-   )imageTrH   r   )r   
attentionsrs   r/   Nc                    t        |t        j                  t        j                  f      rct	        j
                  |j                  d| j                  j                         |j                   t	        j                  |j                         yyt        |t        j                        r?t	        j                  |j                         t	        j                  |j                         yt        |t              rct	        j
                  |j                  d| j                  j                         |j                   t	        j                  |j                         yyy)zInitialize the weightsr   )meanstdN)r!   r&   r   r'   inittrunc_normal_weightr   initializer_ranger   zeros_r   ones_rH   rP   rM   )r)   rs   s     r+   _init_weightsz"IJepaPreTrainedModel._init_weightsG  s     fryy"))45v}}3DKK<Y<YZ{{&FKK( '-KK$JJv}}%0v99IfIfg  ,F--. - 1r,   )r>   r?   r@   r   __annotations__base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backendr   r   _can_record_outputsrB   no_gradr&   r   r'   r   r    r,   r+   r   r   6  s    $O!&*#*L9N"&#(
 U]]_/BII		$9BLL$H /T / /r,   r   c                   H     e Zd Zdef fdZdej                  defdZ xZ	S )IJepaEncoderr   c                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w rp   )
r   r   r   r&   
ModuleListrangenum_hidden_layersr   layergradient_checkpointing)r)   r   rl   r*   s      r+   r   zIJepaEncoder.__init__X  sN    ]]fF^F^@_#`1Jv$6#`a
&+# $as   A#r   r/   c                 d    t        | j                        D ]  \  }} ||      } t        |      S )N)last_hidden_state)	enumerater   r   )r)   r   ilayer_modules       r+   r<   zIJepaEncoder.forward^  s5    (4 	8OA|(7M	8 ??r,   )
r>   r?   r@   r   r   rB   rC   r   r<   rE   rF   s   @r+   r   r   W  s)    ,{ ,@U\\ @o @r,   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )IJepaPoolerr   c                     t         |           t        j                  |j                  |j
                        | _        t        |j                     | _	        y r   )
r   r   r&   r   r    pooler_output_sizer   r   
pooler_act
activationr   s     r+   r   zIJepaPooler.__init__f  s>    YYv1163L3LM
 !2!23r,   r   r/   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )r   r   )r)   r   first_token_tensorpooled_outputs       r+   r<   zIJepaPooler.forwardk  s6     +1a40

#566r,   r   rF   s   @r+   r   r   e  s*    4{ 4
U\\ ell r,   r   c                        e Zd Zddededef fdZdefdZ ed      e		 	 	 dd
e
j                  d	z  de
j                  d	z  ded	z  dee   def
d              Z xZS )
IJepaModelFr   add_pooling_layerrI   c                    t         |   |       || _        t        ||      | _        t        |      | _        t        j                  |j                  |j                        | _        |rt        |      nd| _        | j                          y)z
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether to use a mask token for masked image modeling.
        )rI   r   N)r   r   r   rH   r;   r   encoderr&   r   r    r   	layernormr   pooler	post_init)r)   r   r  rI   r*   s       r+   r   zIJepaModel.__init__v  sm     	 )&P#F+f&8&8f>S>ST->k&)D 	r,   r/   c                 .    | j                   j                  S r   )r;   rN   )r)   s    r+   get_input_embeddingszIJepaModel.get_input_embeddings  s    ///r,   )tie_last_hidden_statesNr-   rg   r.   ry   c                    |t        d      | j                  j                  j                  j                  j
                  }|j
                  |k7  r|j                  |      }| j                  |||      }| j                  |      }|j                  }| j                  |      }| j                  | j                  |      nd}	t        ||	      S )z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rg   r.   )r   pooler_output)r5   r;   rN   r(   r   dtypetor  r   r  r	  r	   )
r)   r-   rg   r.   ry   expected_dtypeembedding_outputencoder_outputssequence_outputr  s
             r+   r<   zIJepaModel.forward  s     ?@@ 99DDKKQQ/'??>:L??/Tl + 
 ,0<<8H+I);;..98<8OO4UY)O[hiir,   )FFNNN)r>   r?   r@   r   rD   r   r   r  r   r   rB   rC   rr   r   r   r	   r<   rE   rF   s   @r+   r  r  t  s    { t ]a $0&: 0 u5 -13704	jllT)j ))D0j #'+	j
 +,j 
$j  6jr,   r  a  
    IJepa Model transformer with an image classification head on top (a linear layer on top of the final hidden states)
    e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune IJepa on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    )custom_introc                        e Zd Zdef fdZee	 	 	 d
dej                  dz  dej                  dz  de	dz  de
e   def
d	              Z xZS )IJepaForImageClassificationr   c                 .   t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        | j                          y )NF)r  r   )r   r   
num_labelsr  r   r&   r   r    Identity
classifierr
  r   s     r+   r   z$IJepaForImageClassification.__init__  ss      ++%@
 OUN_N_bcNc"))F$6$68I8IJikititiv 	r,   Nr-   labelsr.   ry   r/   c                     | j                   |fd|i|}|j                  }| j                  |j                  d            }d}| | j                  ||| j
                  fi |}t        |||j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r.   r   r}   N)losslogitsr   r   )	r   r   r  r   loss_functionr   r
   r   r   )	r)   r-   r  r.   ry   outputsr  r!  r   s	            r+   r<   z#IJepaForImageClassification.forward  s      /9djj/
%=/
 /

 "33!5!5!!5!<=%4%%ffdkkLVLD$!//))	
 	
r,   r  )r>   r?   r@   r   r   r   r   rB   rC   rD   r   r   r
   r<   rE   rF   s   @r+   r  r    s    
{ 
  -1&*04	
llT)
 t#
 #'+	

 +,
 

  
r,   r  )r   r  r  )Nr   )1collections.abcr"   r   rB   torch.nnr&    r   r   activationsr   modeling_layersr   modeling_outputsr   r	   r
   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.genericr   r   configuration_ijepar   Moduler   rH   rC   floatr   r   r   r   r   r   r   r   r   r   r  r  __all__r   r,   r+   <module>r2     s    $   & ! 9 b b F & B B A ,$299 $NNbii Nn !%II%<<% 
% <<	%
 LL4'% T\% % '(%:/. /.dbii "	RYY 			 
")) 
+ < /? / /@@299 @"))  6j% 6j 6jr .
"6 .
.
b Pr,   