iOddlmZddlZddlmZddlmZddlm Z ddl m Z m Z ddl mZmZddlmZdd lmZmZmZdd lmZd d lmZmZmZmZmZmZmZd d l m!Z!d dl"m#Z#m$Z$ejJe&Z'Gdde Z(GddeZ)Gdde#Z*GddeZ+GddeZ,GddeZ-GddeZ.eGddeZ/Gdd eZ0Gd!d"eZ1gd#Z2y)$)CallableN)initialization)PreTrainedConfig)BaseModelOutputBaseModelOutputWithPooling)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringlogging)check_model_inputs)CLIPMLP CLIPAttention CLIPEncoderCLIPEncoderLayerCLIPVisionEmbeddingsCLIPVisionModelCLIPVisionTransformer)eager_attention_forward)VisionRotaryEmbeddingapply_rotary_pos_emb_visioncFeZdZdZdZdZ dfd ZxZS)MLCDVisionConfiga This is the configuration class to store the configuration of a [`MLCDVisionModel`]. It is used to instantiate a MLCD vision encoder according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the vision encoder of the MLCD [DeepGlint-AI/mlcd-vit-bigG-patch14-336](https://huggingface.co/DeepGlint-AI/mlcd-vit-bigG-patch14-336) architecture. Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. Args: hidden_size (`int`, *optional*, defaults to 1664): Dimensionality of the encoder layers and the pooler layer. intermediate_size (`int`, *optional*, defaults to 8192): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. projection_dim (`int`, *optional*, defaults to 1024): Dimensionality of text and vision projection layers. num_hidden_layers (`int`, *optional*, defaults to 48): Number of hidden layers in the Transformer encoder. num_attention_heads (`int`, *optional*, defaults to 16): Number of attention heads for each attention layer in the Transformer encoder. num_channels (`int`, *optional*, defaults to 3): The number of input channels. image_size (`int`, *optional*, defaults to 336): The size (resolution) of each image. patch_size (`int`, *optional*, defaults to 14): The size (resolution) of each patch. hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the layer normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. initializer_factor (`float`, *optional*, defaults to 1.0): A factor for initializing all weight matrices (should be kept to 1, used internally for initialization testing). Example: ```python >>> from transformers import MLCDVisionConfig, MLCDVisionModel >>> # Initializing a MLCDVisionConfig with DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> configuration = MLCDVisionConfig() >>> # Initializing a MLCDVisionModel (with random weights) from the DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> model = MLCDVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```mlcd_vision_model vision_configc t|di|||_||_||_||_||_||_||_||_ | |_ | |_ | |_ | |_ | |_y)N)super__init__ hidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_groups num_channels patch_size image_sizeinitializer_rangeinitializer_factorattention_dropoutlayer_norm_eps hidden_act)selfr#r$r%r&r'r(r*r)r/r.r-r+r,kwargs __class__s o/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/transformers/models/mlcd/modular_mlcd.pyr"zMLCDVisionConfig.__init__dsz" "6"&!2!2#6 $8!($$!2"4!2,$) ii 0riPgelugh㈵>g{Gz??)__name__ __module__ __qualname____doc__ model_typebase_config_keyr" __classcell__r2s@r3rr*sH4 l%J%O%%r4rc eZdZy)MLCDMLPN)r<r=r>r r4r3rErEsr4rEc4eZdZdededej fdZy)MLCDRotaryEmbeddingnum_patches_heightnum_patches_widthreturnctj||jjj dj d|}tj||jjj dj |d}tj |j|jgd}t||}tj||jj|jj}tj||j}||jd} | S)a} Calculate the Rotary Position Embedding (RoPE) for MLCDVisionModel based on the grid size. Args: num_patches_height (int): Number of patches in the height dimension. num_patches_width (int): Number of patches in the width dimension. Returns: torch.Tensor: Rotary positional embeddings for the given grid size. )devicer7rdim)rLdtype) torcharangeinv_freqrL unsqueezeexpandstackflattenmaxrPouter) r0rHrIhpos_idswpos_idspos_ids max_grid_sizeseqrotary_pos_emb_fullrotary_pos_embs r3forwardzMLCDRotaryEmbedding.forwards  LL+DMM4H4H I S STU V ] ]^`bs t  LL*4==3G3G H R RST U \ \]oqs t  ++x//183C3C3EFBO.0AB ll=1E1ET]]M`M`a#kk#t}}=-W5==a@r4N)r<r=r>intrQTensorrar r4r3rGrGs ##%,,r4rGc\eZdZdeffd Zdej dejfdZxZ S)MLCDVisionEmbeddingsconfigc(t|||`yN)r!r"position_embeddingr0rfr2s r3r"zMLCDVisionEmbeddings.__init__s   #r4 pixel_valuesrJcT|jd}|jjj}|j|j |}|j dj dd}|jj|dd}tj||gd}|S)NrrPrr7rMrN) shapepatch_embeddingweightrPtorW transposeclass_embeddingrUrQcat)r0rk batch_size target_dtype patch_embeds class_embeds embeddingss r3razMLCDVisionEmbeddings.forwards!''* ++2288 ++LOO,O,OP #++A.88A> ++22:q"E YY l;C r4) r<r=r>rr"rQ FloatTensorrcrarBrCs@r3reres-$/$ E$5$5 %,, r4receZdZdZdeffd Z d dejdeejejfdejdzde e d eejejdzff d Z xZ S) MLCDAttentionzMulti-headed attention with RoPE. Refer to papers: - Attention is all you need: https://huggingface.co/papers/1706.03762 - RoFormer: Enhanced Transformer with Rotary Position Embedding: https://huggingface.co/papers/2104.09864 rfcTt|||j|_d|_y)NF)r!r"r' is_causalrjs r3r"zMLCDAttention.__init__s%  $*$?$?!r4N hidden_statesposition_embeddingsattention_maskr1rJc |jdd\}}|j|j|||j|jf}|j |j|||j|jf}|j |j|||j|jf} |djdj} |djdj} t||| | \}}|jddddj}|jddddj}| jddddj} tj|jjt } | |||| |f|j"sdn |j$|j&|j(d|\} }| jddddj} | j+||d} |j-| } | jdddj} | |fS)NrMrr7rrr:)dropoutscalingr~)rnq_projreshape num_headshead_dimk_projv_projrTfloatrpermute contiguousr get_interfacerf_attn_implementationrtrainingrscaler~viewout_proj)r0rrrr1ru seq_length query_states key_states value_statescossinattention_interface attn_output attn_weightss r3razMLCDAttention.forwards;"/!4!4Sb!9 J{{=199:zSWSaSacgcpcp:qr [[/77ZQUQ_Q_aeanan8op {{=199:zSWSaSacgcpcp:qr "!$..q1779!!$..q1779#>|ZY\^a#b j$++Aq!Q7BBD ''1a3>>@ #++Aq!Q7BBD (?(M(M KK , ,.E) %8      % $}}C$,,JJnn %  % ! \"))!Q15@@B !&&z:rB mmK0 !))!Q2==? L((r4rh) r<r=r>r?rr"rQrctupler r rarBrCs@r3r|r|s//3 ,)||,)#5<<#=>,) t+ ,) +, ,) u||U\\D00 1 ,)r4r|c eZdZdeffd Z d dej deej ej fdej dzdee deejf d Z xZ S) MLCDEncoderLayerrfcDt||t||_yrh)r!r"r| self_attnrjs r3r"zMLCDEncoderLayer.__init__s  &v.r4Nrrrr1rJc |}|j|}|jd|||d|\}}||z}|}|j|}|j|}||z}|S)a Args: hidden_states (`torch.FloatTensor`): Input to the layer of shape `(batch, seq_len, embed_dim)`. Represents the hidden states from the previous layer or the input embeddings. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.FloatTensor`): Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values. )rrrr ) layer_norm1r layer_norm2mlp)r0rrrr1residual_s r3razMLCDEncoderLayer.forwards$!((7 )4>> ' 3)    q !=0  ((7 /  =0 r4rh) r<r=r>rr"rQrcrr r rzrarBrCs@r3rrsz////3 "||"#5<<#=>" t+ " +, " u  ! "r4rc eZdZdZdeffd Z d dejdeejejfdejdzde e d ee zf d Z xZS) MLCDEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`MLCDEncoderLayer`]. Args: config: MLCDVisionConfig rfc$t||y)z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.N)r!r"rjs r3r"zMLCDEncoder.__init__-s  r4N inputs_embedsrrr1rJc V|}|jD] }||||fi|}t|S)a= Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) )last_hidden_state)layersr)r0rrrr1r encoder_layers r3razMLCDEncoder.forward1sK,& ![[ M)# M +  r4rh)r<r=r>r?rr"rQrzrrcr r rrarBrCs@r3rr$s{!/!/3 ! ((! #5<<#=>!  t+ ! +, !  ! r4rcleZdZUeed<dZdZdZdZdZ dZ dZ e e dZej dZy)MLCDPreTrainedModelrfmlcdTF)r attentionsc& |jj}t|tr|jj}t j |j d|jdz|zt j |jj|jj|zt j|jtj|jjdj!dyt|t"r|jj}|jdzd|jj$zdzz|z}|jdz|z}t j |j&j|t j |j(j|t j |j*j|t j |j,j|yt|t.r|jj}|jj0dzd|jj$zdzz|z}d|jj0zdz|z}t j |j2j|t j |j4j|yt|t6ro|jj}|jj0|jj8zdzdz|z}t j |j:d|yt|t<j>r?t j@|jBt jD|jyt|t<jFr,|jB t j@|jByt|tHrod |jJtjd |jLdtjN |jLz zz }t j|jP|yy) zInitialize the weightsr:g)meanstd)rrM)r7rMrNr;rrm))rfr, isinstancereinitnormal_rs embed_dimrorpr+copy_ position_idsrQrRrnrUr|r%rrrrrEr#fc1fc2MLCDVisionTransformerr& class_pos_embnn LayerNormzeros_biasones_LinearrGthetarOrrS)r0modulefactor in_proj_std out_proj_stdfc_std pos_emb_stdrSs r3 _init_weightsz!MLCDPreTrainedModel._init_weightsds#// f2 3[[33F LL//cv?O?OQU?UX^?^ _ LL//66FMM<[<[^dr__annotations__base_model_prefixsupports_gradient_checkpointingaccepts_loss_kwargs_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrr|_can_record_outputsrQno_gradrr r4r3rrUs[ &*#N"&)# U]]_!2!2r4rc leZdZdeffd Ze ddejdzdee de e zfdZ xZ S) rrfct||t|j|jzdz|_t jtjd|j|jzdz|_ y)Nrr7) r!r"rGr#r&vision_rotary_embeddingr ParameterrQrandnrrjs r3r"zMLCDVisionTransformer.__init__sh  ':6;M;MQWQkQk;kop;p'q$\\%++a9K9KvOiOi9imn9n*opr4Nrkr1rJc | td|jd|jjz}|jd|jjz}|j ||}|j |j j}tj|j |gd}tj||fd}|j|jf}|j|}|j|}|jd||d|} | d} | dddddf} |j| } t!| | S) Nz You have to specify pixel_valuesrMrrN)rr)r pooler_outputr ) ValueErrorrnrfr)rrqrrLrQrtrrry pre_layrnormencoderpost_layernormr) r0rkr1rHrIr`embrrencoder_outputsr pooled_outputs r3razMLCDVisionTransformer.forwardsP  ?@ @)//3t{{7M7MM(..r2dkk6L6LL556HJ[\'**4+=+=+D+DED$6$6#GQOii8bA"wwy#'')4 5 ))-8 &$,, ' 3   ,A.)!Q'2 ++M: )/'  r4rh)r<r=r>rr"r rQrzr r rrrarBrCs@r3rrs^q/q 26  ''$.  +,   + +    r4rc leZdZede ddej dzdeede e zfdZ y) MLCDVisionModelF)tie_last_hidden_statesNrkr1rJc *|jdd|i|S)a Example: ```python >>> import httpx >>> from io import BytesIO >>> from PIL import Image >>> from transformers import AutoProcessor, MLCDVisionModel >>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs, output_attentions=True) >>> features = outputs.last_hidden_state >>> print(f"Extracted features shape: {features.shape}") >>> print(f"Number of attention layers: {len(outputs.attentions)}") >>> print(f"Attention shape: {outputs.attentions[0].shape}") ```rkr ) vision_model)r0rkr1s r3razMLCDVisionModel.forwards)>!t   %   r4rh) r<r=r>rr rQrzr r rrrar r4r3rrsYu526  ''$.  +,   + +  6  r4r)rrr)3collections.abcrrQtorch.nnrrrconfiguration_utilsrmodeling_outputsrrmodeling_utilsr r processing_utilsr utilsr r r utils.genericrclip.modeling_cliprrrrrrrllama.modeling_llamarqwen2_vl.modeling_qwen2_vlrr get_loggerr<loggerrrErGrer|rrrrr__all__r r4r3rs% &3KF&@@/;[   H %Y%'Y%x g /D/$9)M9)x'''T. +. b02/0202f' 1' T# o# L r4