
    i                        d Z ddlmZ ddlmZ ddlmZ ddlZddlm	c m
Z ddlm	Z	 ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0  e&jb                  e2      Z3e e$d       G d de                    Z4e e$d       G d de                    Z5	 	 	 	 dAdZ6g fdZ7 G d  d!e	jp                        Z9 G d" d#e	jt                        Z; G d$ d%e	jx                        Z= G d& d'ej                  jx                        Z>d( Z?dBd)Z@ G d* d+e	jx                        ZA	 dCd,e	jx                  d-ej                  d.ej                  d/ej                  d0ej                  dz  d1eCd2eCfd3ZD G d4 d5e	jx                        ZE G d6 d7e      ZF G d8 d9e      ZGe$ G d: d;e             ZHe$ G d< d=eH             ZI G d> d?eHe      ZJg d@ZKy)DzPyTorch Idefics model.    )Callable)	dataclass)AnyN)nn   )initialization)ACT2FN)CacheDynamicCache)GenerationMixin)create_causal_mask)GradientCheckpointingLayer)ModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedConfigPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)OutputRecordercheck_model_inputs   )IdeficsConfig)IdeficsPerceiverResampler)IdeficsVisionEmbeddingsIdeficsVisionTransformerz{
    Base class for Idefics model's outputs that may also contain a past key/values (to speed up sequential decoding).
    )custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZe	dz  ed<   dZ
eej                     dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   y)IdeficsBaseModelOutputWithPasta  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
        `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
        input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    Nlast_hidden_statepast_key_valueshidden_states
attentionsimage_hidden_states)__name__
__module____qualname____doc__r"   torchFloatTensor__annotations__r#   r
   r$   tupler%   r&        v/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/transformers/models/idefics/modeling_idefics.pyr!   r!   0   s|    & 37u((4/6$(OUT\(59M5**+d2926Je''(4/6;?u001D8?r0   r!   zS
    Base class for Idefics causal language model (or autoregressive) outputs.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   y)	IdeficsCausalLMOutputWithPastae  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
        sequence_length, hidden_size)`.

        image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    Nlosslogitsr#   r$   r%   r&   )r'   r(   r)   r*   r4   r+   r,   r-   r5   r#   r
   r$   r.   r%   r&   r/   r0   r1   r3   r3   Q   s    " &*D%

d
")'+FE$+$(OUT\(59M5**+d2926Je''(4/6;?u001D8?r0   r3   c                    t        j                  | j                  d         j                  dd      j	                  d|      j                  d      j                  | j                        }| j                  d|      } |j                  d      |d<   |j                  d      |d<   |j                  d      |d<   |j                  d      |d<   d|v r|d   }|j                  d|      |d<   ||j                  d|      |d	<   |d   |d   j                  d|      |d<   |d   |d   j                  d|      |d<   | |fS |d   |d   j                  d|      |d<   | |fS |d   |d   j                  d|      |d<   | |fS )
Nr   r   pixel_valuesimage_encoder_embeddingsperceiver_embeddingsimage_attention_masktoken_type_idsattention_mask)	r+   arangeshapeviewrepeattodeviceindex_selectget)	input_idsexpand_sizeis_encoder_decoderr=   encoder_outputsmodel_kwargsexpanded_return_idxr<   s           r1   expand_inputs_for_generationrL   q   s    	Y__Q'(--b!4;;A{KPPQSTWWXaXhXhi  &&q*=>I#/#3#3N#CL /;/?/?@Z/[L+,+7+;+;<R+SL'(+7+;+;<R+SL'(<'%&67)7)D)DQH[)\%&!)7)D)DQH[)\%&*+7/;<R/S/`/`"0
+, N#/'3N'C'P'PQRTg'h^$ l"" 
0	1	=3?@Z3[3h3h"4
/0 l"" 
,	-	9/;<R/S/`/`"0
+, l""r0   c                 2   t         j                  t         j                  t         j                  d}|D cg c]  }||   	 }}| j	                         D ];  |r&t        fd|D              rj                  d       +j                  d       = | S c c}w )N)	LayerNormLinear	Embeddingc              3   6   K   | ]  }t        |        y wN)
isinstance).0tmodules     r1   	<genexpr>zfreeze_model.<locals>.<genexpr>   s     $]qZ%:$]s   TF)r   rN   rO   rP   modulesanyrequires_grad_)modelmodule_exceptionsmappingmmodule_exceptions_mappedrV   s        @r1   freeze_modelr`      s    \\))\\G
 5FFq
FF--/ )$]D\$]!]!!$'!!%(	)
 L  Gs   Bc                   N     e Zd ZdZ	 	 	 	 ddedz  ddf fdZd ZdefdZ xZ	S )	IdeficsDecoupledEmbeddinga  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the embeddings. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `num_additional_embeddings` > 0,
    then it will create `num_additional_embeddings` additional parameters that are always trained. If
    `num_additional_embeddings=0`, then the module defaults back to the regular behavior of `nn.Embedding`.
    Npartially_freezereturnc           	      B   |||kD  rt        d| d|       t        	|   d|||||d| || _        || _        || _        || _        |r| j                  j                  d       | j
                  dkD  r)t        j                  | j
                  |||      | _        yy)	a)  
        Args:
            num_embeddings (`int`):
                Size of the dictionary of embeddings
            num_additional_embeddings (`int`):
                Number of additional embeddings. Only useful when you `partially_freeze=True`.
            embedding_dim (`int`):
                The size of each embedding vector
            partially_freeze: (`bool`, *optional*, defaults to `False`):
                If `True`, the regular `weight` will be frozen. `additional_weight` is never frozen.
            padding_idx (`int`, *optional*):
                The padding index (needs to be less than num_embeddings)

        Note: there are a lot of other parameters to initialize a standard `nn.Embedding` such as `padding_idx`,
        `max_norm` or `norm_type`. We are not supporting these.
        Nz/padding_idx must be within num_embeddings. Got z and )num_embeddingsembedding_dimrC   dtypepadding_idxFr   )rf   rg   rC   rh   r/   )
ValueErrorsuper__init__rf   ri   num_additional_embeddingsrc   weightrZ   r   rP   additional_embedding)
selfrf   rm   rg   rc   rC   rh   ri   kwargs	__class__s
            r1   rl   z"IdeficsDecoupledEmbedding.__init__   s    6 "{^'CN{m[`ao`pqrr 	
)'#	
 	
 -&)B& 0KK&&u-))A-(*#==+	)D% .r0   c                 b   | j                   dk(  r t        j                  || j                        S |j	                         }t        j                  || j                  k\        }||   }| j                  || j                  z
        }d||<   t        j                  || j                        }|||<   |S )a  
        we have 2 embeddings, with different indices - one pretrained self.weight and another
        self.additional_embedding.weight that is being trained.

        in order to make a lookup of the input ids, we:
        1. find out the indices of the entries belonging to the 2nd embedding
        2. extract those values while subtracting the size of the first embedding (num_embeddings), since the 2nd
           embedding starts from 0 and not num_embeddings
        3. perform the 2nd embedding lookup
        4. now we handle the 1st embedding, we overwrite indices belonging to the 2nd embedding with a padding index
        5. perform the 1st embedding lookup
        6. now we overwrite the values in the 1st embedding lookup with the values of the 2nd embedding lookup

        note: for the 1st embedding lookup we could have looked up only the low indices and not do the padding, but
        then we have to create a new tensor and populate it with 2 tensors that are spread out across various indices -
        i.e. not a simple concat - I haven't benchmarked the complex case if it's any faster, given that seqlens are
        usually relatively short it's probably not faster or if faster not by much - but might be a good idea to
        measure.

        r   )	rm   F	embeddingrn   cloner+   whererf   ro   )rp   rF   additional_vocab_indicesinput_ids_additional_vocabadditional_embeddingsfull_vectors         r1   forwardz!IdeficsDecoupledEmbedding.forward   s    * ))Q.;;y$++66 OO%	#(;;yD<O<O/O#P %./G%H" $ 9 9:TW[WjWj:j k /0	*+kk)T[[9 1F,-r0   c                 n    d| j                    d| j                   d| j                   d| j                   S )Nznum_embeddings=z, num_additional_embeddings=z, embedding_dim=, partially_freeze=)rf   rm   rg   rc   rp   s    r1   
extra_reprz$IdeficsDecoupledEmbedding.extra_repr  sq     !4!4 55QRVRpRpQq  rB  CG  CU  CU  BV  Vi  jn  j  j  i@  A  	Ar0   )FNNN)
r'   r(   r)   r*   boolrl   r|   strr   __classcell__rr   s   @r1   rb   rb      sH     ).3
 +3 
3j%NAC Ar0   rb   c                        e Zd ZdZ	 	 	 	 	 ddedededededdf fd	Zd
ej                  dej                  fdZ	de
fdZ xZS )IdeficsDecoupledLineara  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the parameters. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `out_additional_features` > 0,
    then it will create `out_additional_features * in_features` additional parameters that are always trained. If
    `out_additional_features=0`, then the module defaults back to the regular behavior of `nn.Linear`.
    Nin_featuresout_featuresout_additional_featuresbiasrc   rd   c                 "   t         |   |||||       || _        || _        || _        || _        |r8| j                  j                  d       |r| j                  j                  d       |dkD  r t        j                  |||||      | _        yy)aG  
        out_additional_features: int. Number of additional trainable dimensions. Only makes sense when
        `partially_freeze=True`. partially_freeze: bool. If True, the regular `weight` will be frozen and extra
        parameters (if any) will be trainable. If False, default to the regular behavior of nn.Linear.
        Fr   )r   r   r   rC   rh   N)rk   rl   r   rc   r   r   rn   rZ   r   r   rO   additional_fc)	rp   r   r   r   r   rc   rC   rh   rr   s	           r1   rl   zIdeficsDecoupledLinear.__init__  s     	lD&%H'>$ 0&(KK&&u-		((/"Q&!#'4"D 'r0   inputc                     t        j                  || j                  | j                        }| j                  dkD  r)| j                  |      }t        j                  ||fd      }|S )Nr   r7   )rt   linearrn   r   r   r   r+   cat)rp   r   outputadditional_featuress       r1   r|   zIdeficsDecoupledLinear.forwardC  sV    %dii8''!+"&"4"4U";YY(;<bAFr0   c           
          d| j                    d| j                   d| j                   d| j                  du d| j                   
S )z=Overwriting `nn.Linear.extra_repr` to include new parameters.zin_features=z, out_features=z, out_additional_features=z, bias=Nr~   r   r   r   r   rc   r   s    r1   r   z!IdeficsDecoupledLinear.extra_reprL  s    d../t?P?P>QQklp  mI  mI  lJ  JQ  RV  R[  R[  cg  Rg  Qh  h{  |@  |Q  |Q  {R  S  	Sr0   )r   TTNN)r'   r(   r)   r*   intr   rl   r+   Tensorr|   r   r   r   r   s   @r1   r   r     s     ()!%"" " "%	"
 " " 
"HU\\ ell SC Sr0   r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )IdeficsRMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z=
        IdeficsRMSNorm is equivalent to T5LayerNorm
        N)rk   rl   r   	Parameterr+   onesrn   variance_epsilon)rp   hidden_sizeepsrr   s      r1   rl   zIdeficsRMSNorm.__init__S  s1     	ll5::k#:; #r0   c                    |j                  t        j                        j                  d      j	                  dd      }|t        j
                  || j                  z         z  }| j                  j                  t        j                  t        j                  fv r%|j                  | j                  j                        }| j                  |z  S )N   r7   T)keepdim)rB   r+   float32powmeanrsqrtr   rn   rh   float16bfloat16)rp   r$   variances      r1   r|   zIdeficsRMSNorm.forward[  s     ##EMM266q9>>r4>P%Ht?T?T4T(UU ;; ??),,T[[->->?M{{]**r0   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)r.   rn   r?   r   r   s    r1   r   zIdeficsRMSNorm.extra_repre  s*    ))*+6$2G2G1HIIr0   )gư>)r'   r(   r)   rl   r|   r   r   r   s   @r1   r   r   R  s    $+Jr0   r   c                   .     e Zd Zd fd	Zd ZddZ xZS )IdeficsEmbeddingc                    t         |           || _        || _        || _        d| j                  t        j                  d| j                  dt
        j                        j                  |t
        j                        | j                  z  z  z  }| j                  d|d       | j                  || j                  j                  t        j                         	       y )
N      ?r   r   rh   rC   rh   inv_freqF
persistentseq_lenrC   rh   )rk   rl   dimmax_position_embeddingsbaser+   r>   int64rB   floatregister_buffer_set_cos_sin_cacher   rC   get_default_dtype)rp   r   r   r   rC   r   rr   s         r1   rl   zIdeficsEmbedding.__init__k  s    '>$	IIQ!5;;?BB&X]XcXcBdgkgogooq
 	ZeD 	+DMM4H4HPUPgPgPi 	  	
r0   c                    || _         t        j                  | j                   |t        j                        j	                  | j
                        }t        j                  d|| j
                        }t        j                  ||fd      }| j                  d|j                         j                  |      d       | j                  d|j                         j                  |      d       y )	Nr   i,j->ijr7   r   
cos_cachedFr   
sin_cached)max_seq_len_cachedr+   r>   r   type_asr   einsumr   r   cosrB   sin)rp   r   rC   rh   rU   freqsembs          r1   r   z#IdeficsEmbedding._set_cos_sin_cache|  s    ")LL00u{{S[[\`\i\ijY4==9iiB/\3779<<+>5Q\3779<<+>5Qr0   c                    || j                   kD  r(| j                  ||j                  |j                         | j                  d | j                  |j                        | j                  d | j                  |j                        fS )Nr   r   )r   r   rC   rh   r   rB   r   )rp   xr   s      r1   r|   zIdeficsEmbedding.forward  sy    T,,,##GAHHAGG#T OOHW%((qww(7OOHW%((qww(7
 	
r0   )i   i'  NrR   )r'   r(   r)   rl   r   r|   r   r   s   @r1   r   r   j  s    
"R
r0   r   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr7   r   r   )r?   r+   r   )r   x1x2s      r1   rotate_halfr     sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r0   c                     ||   j                  |      }||   j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )an  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`):
            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
            used to pass offsetted position ids when working with a KV-cache.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   position_idsunsqueeze_dimq_embedk_embeds           r1   apply_rotary_pos_embr     sg    * l

%
%m
4C
l

%
%m
4C3w;q>C/0G3w;q>C/0GGr0   c                   2     e Zd Zdededef fdZd Z xZS )
IdeficsMLPr   intermediate_size
hidden_actc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        t        |   | _        y )NFr   )	rk   rl   r   rO   	gate_proj	down_projup_projr	   act_fn)rp   r   r   r   rr   s       r1   rl   zIdeficsMLP.__init__  s[     	;0AN#4kNyy.?eLZ(r0   c                     | j                  | j                  | j                  |            | j                  |      z        S rR   )r   r   r   r   )rp   r   s     r1   r|   zIdeficsMLP.forward  s0    ~~dkk$..*;<t||ANOOr0   )r'   r(   r)   r   r   rl   r|   r   r   s   @r1   r   r     s*    
)
) 
) 	
)Pr0   r   rV   querykeyvaluer=   scalingdropoutc                    t        j                  ||j                  dd            |z  }|||z   }t        j                  j                  |dt         j                        j                  |j                        }t        j                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr7   )r   rh   ptrainingr   r   )r+   matmul	transposer   
functionalsoftmaxr   rB   rh   r   r   
contiguous)
rV   r   r   r   r=   r   r   rq   attn_weightsattn_outputs
             r1   eager_attention_forwardr     s     <<s}}R'<=GL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K$$r0   c                   n    e Zd ZdZ	 	 	 	 	 ddedededededz  ded	edz  f fd
Zde	j                  dedefdZ	 	 	 	 	 dde	j                  de	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  dee   dee	j                  e	j                  f   fdZ xZS )IdeficsAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNr   	num_headsr   is_cross_attentionconfigqk_layer_norms	layer_idxc                    t         	|           || _        || _        || _        ||z  | _        || _        d| _        | j
                  dz  | _        || _	        |-t        j                  d| j                  j                   d       | j
                  |z  | j                  k7  rt        d| j                   d| d      || _        t!        t"        j$                  d      st        d	      | j                  rt!        |j&                  d
      s| j                  n|j&                  j(                  }t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  ||| j
                  z  d      | _        t#        j*                  ||| j
                  z  d      | _        nt#        j*                  | j                  || j
                  z  d      | _        t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  | j                  || j
                  z  d      | _        t#        j*                  || j
                  z  |d      | _        t5        | j
                        | _        || _        | j8                  rMt;        | j
                  |j<                        | _        t;        | j
                  |j<                        | _         y y )NTg      zInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.z?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).scaled_dot_product_attentionz)this model requires pytorch 2.0 or higher	embed_dimFr   r   )!rk   rl   r   r   r   head_dimr   	is_causalr   r   loggerwarning_oncerr   r'   rj   r   hasattrr   r   vision_configr   rO   q_projk_projv_projo_projr   
rotary_embr   r   rms_norm_epsq_layer_normk_layer_norm)
rp   r   r   r   r   r   r   r   kv_input_dimrr   s
            r1   rl   zIdeficsAttention.__init__  s    	&"#y0}}d*" !8!8 9 :, , MMI%$*:*::QRVRbRbQc$YKr3 
 #5r}}&DEHII""(/0D0Dk(R  X^XlXlXvXv  ))  DMM)DK
 ))L)dmm2KRWXDK))DMM)DK ))  DMM)DK
 ))  DMM)DK
 ))  DMM)DK
 ii%

 +4==9, .t}}&BUBU VD .t}}&BUBU VD r0   tensorr   bszc                     |j                  ||| j                  | j                        j                  dd      j	                         S )Nr   r   )r@   r   r   r   r   )rp   r  r   r  s       r1   _shapezIdeficsAttention._shape3  s7    {{3GQQRSUVWbbddr0   r$   key_value_statesr=   r   r#   cache_positionrq   rd   c                    | j                   xs |d u}|j                         \  }	}
}| j                  |      j                  |	|
| j                  | j
                        j                  dd      }|s| j                  |      j                  |	|
| j                  | j
                        j                  dd      }| j                  |      j                  |	|
| j                  | j
                        j                  dd      }n|j                         \  }}}| j                  |      j                  |	|| j                  | j
                        j                  dd      }| j                  |      j                  |	|| j                  | j
                        j                  dd      }|j                  d   }|||d   z  }|s2| j                  |t        ||
            \  }}t        |||||      \  }}|%d|i}|j                  ||| j                  |      \  }}| j                  r"| j!                  |      }| j#                  |      }t%        j&                  | j(                  j*                  t,              } || ||||f| j.                  sdn| j0                  | j2                  d|\  }}|j5                  |	|
d	      j7                         }| j9                  |      }||fS )
Nr   r   r   r   )r   r          )r   r   r7   )r   sizer  r@   r   r   r   r  r  r?   r
  maxr   updater   r   r  r  r   get_interfacer   _attn_implementationr   r   r   r   reshaper   r	  )rp   r$   r  r=   r   r#   r  rq   r   r  q_len_query_states
key_statesvalue_stateskv_len
kv_seq_lenr   r   cache_kwargsattention_interfacer   r   s                          r1   r|   zIdeficsAttention.forward6  s    "44T8HPT8T%**,UA{{=166sE4>>SWS`S`akklmopq!]388eT^^UYUbUbcmmnoqrsJ;;}5::3t~~W[WdWdeoopqstuL+002LAvq%56;;CY]YfYfgqqrsuvwJ,-223PTP]P]^hhijlmn   %%b)
&.++J!|SU=STHC';L*VY[^`l'm$L* &,n=L'6'='=j,X\XfXfht'u$J,,\:L**:6J(?(M(MKK,,.E)
 %8	%
  $}}C$,,LL	%
 	%
!\ "))#ub9DDFkk+.L((r0   )r  FNFNNNNNN)r'   r(   r)   r*   r   r   r   r   rl   r+   r   r  
LongTensorr
   r   r   r.   r|   r   r   s   @r1   r   r     sH   G #(*.$ $OWOW OW 	OW
 !OW !4'OW OW :OWbeU\\ eC ec e 15.204(,26>)||>)  ,,->) t+	>)
 &&->) >) ((4/>) +,>) 
u||U\\)	*>)r0   r   c                        e Zd Zddededz  f fdZe	 	 	 	 ddej                  dej                  dz  dej                  dz  de
dz  d	ej                  dz  d
ee   dej                  fd       Z xZS )IdeficsDecoderLayerNr   r   c                    t         |           |j                  | _        t        | j                  |j                  |j
                  ||      | _        t        | j                  |j                  |j                        | _
        t        |j                  |j                        | _        t        |j                  |j                        | _        |j
                  | _        y )N)r   r   r   r   r   r   r   r   r   )rk   rl   r   r   num_attention_headsr   	self_attnr   r   r   mlpr   r  input_layernormpost_attention_layernormrp   r   r   rr   s      r1   rl   zIdeficsDecoderLayer.__init__y  s    !--)((00NN
 (($66((

  .f.@.@fFYFYZ(6v7I7IvObOb(c%~~r0   r$   r=   r   r#   r  rq   rd   c           	         |}| j                  |      } | j                  d|||||d|\  }}t        j                  j	                  || j                  | j
                        }||z   }|}| j                  |      }| j                  |      }t        j                  j	                  || j                  | j
                        }||z   }|S )N)r$   r=   r   r#   r  r   r/   )r/  r-  r   r   r   r   r0  r.  )	rp   r$   r=   r   r#   r  rq   residualr  s	            r1   r|   zIdeficsDecoderLayer.forward  s     !,,]; *4>> 
')%+)
 
q --mt||VZVcVc-d =0 !55mD/--mt||VZVcVc-d =0r0   rR   )NNNN)r'   r(   r)   r   r   rl   r   r+   r   r'  r
   r   r   r,   r|   r   r   s   @r1   r)  r)  x  s    &} &t &&  /304(,26 ||  t+  &&-	 
   ((4/  +,  
		   r0   r)  c                   
    e Zd Zddededz  f fdZe	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  de
e   dej                  fd       Z xZS )IdeficsGatedCrossAttentionLayerNr   r   c           	      	   t         |           |j                  | _        t        | j                  |j                  d|j
                  ||j                  |      | _        t        | j                  |j                  |j                        | _        t        |j                  |j                        | _        t        |j                  |j                        | _        |j
                  | _        t#        j$                         | _        t#        j$                         | _        |j*                  dk(  r|j,                  dk(  rtt#        j.                  t1        j2                  dd| j                              | _        t#        j.                  t1        j2                  dd| j                              | _        n|j,                  dk(  r\t#        j.                  t1        j2                  d            | _        t#        j.                  t1        j2                  d            | _        nt9        d	|j,                   d
      |j*                  dk(  r|j,                  dk(  rtt#        j.                  t1        j:                  dd| j                              | _        t#        j.                  t1        j:                  dd| j                              | _        n|j,                  dk(  r\t#        j.                  t1        j:                  d            | _        t#        j.                  t1        j:                  d            | _        n|t9        d	|j,                   d
      |j*                  dv r;|j,                  dk(  rt#        j.                  t1        j<                  d|j>                  dd| j                  f            | _        t#        j.                  t1        j<                  d|j>                  dd| j                  f            | _        n|j,                  dk(  rut#        j.                  t1        j<                  d|j>                  d            | _        t#        j.                  t1        j<                  d|j>                  d            | _        n2t9        d	|j,                   d
      tA        d|j*                   d      tC        | d      rtC        | d      st9        d      y )NT)r   r   r   r   r   r   r   r+  r   zerosvectorr   r   z Unknown value for `alpha_type` ()r   >   normalrandomgaussianr  )r   stdr  zAlpha initialization scheme z not yet implemented!alpha_cross_attnalpha_densez+Alpha parameters not initialized correctly!)"rk   rl   r   r   r,  r   r   
cross_attnr   r   r   r.  r   r  r/  r0  r   r   Tanhact_cross_attn	act_densealpha_initializer
alpha_typer   r+   r7  r>  r?  rj   r   r:  alphas_initializer_rangeNotImplementedErrorr  r1  s      r1   rl   z(IdeficsGatedCrossAttentionLayer.__init__  s   !--*((00#NN!00
 (($66((

  .f.@.@fFYFYZ(6v7I7IvObOb(c%nn ggi##w.  H,(*U[[AtGWGW5X(Y%#%<<Aq$BRBR0S#T ""g-(*U[[^(D%#%<<A#?  #CFDUDUCVVW!XYY%%/  H,(*UZZ1dFVFV5W(X%#%<<

1aAQAQ0R#S ""g-(*UZZ](C%#%<<

1#>  #CFDUDUCVVW!XYY%%)II  H,(*LLcv/N/NVWYZ\`\l\lUmn)% $&<<LLcv/N/NVWYZ\`\l\lUmn$  ""g-(*LLcv/N/NVWY)% $&<<#6KjKjrs0u#v  #CFDUDUCVVW!XYY &(DVE]E]D^^s&tuu01gdM6RJKK 7Sr0   r$   r=   r&   r;   cross_attention_gater#   rq   rd   c                    |t        d      |t        d      |t        d      |}| j                  |      } | j                  d	|||d|\  }}	t        j
                  j                  || j                  | j                        }|j                  |dk(  dddddf   d      }|| j                  | j                        |z  z   }|}| j                  |      }| j                  |      }t        j
                  j                  || j                  | j                        }|| j                  | j                        |z  z   }|S )
a  
        image_hidden_states (`torch.FloatTensor`):
            Input to the layer of shape `(batch, seq_len, embed_dim)`
        image_attention_mask (`torch.FloatTensor`, *optional*):
            image attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        cross_attention_gate (`torch.FloatTensor`, *optional*):
            gate of size `(batch, seq_len)` used to zero-out cross-attention output for tokens attending no images.
        Nzt`image_hidden_states` is required for Idefics cross attention module which are visual features to be conditioned on.z`cross_attention_gate` is required for Idefics cross attention module to zero-out the cross-attention hidden_states attending to no images.zMPast key value states are not implemented for Idefics cross attention module.)r$   r  r=   r   r   r  r/   )rj   rG  r/  r@  r   r   r   r   r   masked_fillrB  r>  r0  r.  rC  r?  )
rp   r$   r=   r&   r;   rH  r#   rq   r3  r  s
             r1   r|   z'IdeficsGatedCrossAttentionLayer.forward  sm   ( &# 
  ' ^  &%&uvv ,,]; +4?? 
'0/
 	
q --mt{{UYUbUb-c%113G13LaQRTXj2Y[^_ 4#6#6t7L7L#MP]#]] !55mD/--mt{{UYUbUb-c 4>>$2B2B#Cm#SSr0   rR   r&  )r'   r(   r)   r   r   rl   r   r+   r   r
   r   r   r,   r|   r   r   s   @r1   r5  r5    s    @L} @Lt @LD  /3374848(,8||8 t+8 #\\D0	8
 $llT18 $llT18 8 +,8 
		8 8r0   r5  c                        e Zd ZU eed<   dZdZdZddgZdZ	dZ
dZdZe eedd	
      dZ ej$                          fd       Z xZS )IdeficsPreTrainedModelr   r[   )imagetextTr)  r5  Fr   r-  )index
layer_name)r$   r%   c                 x   t         |   |       t        |t              ryt	        j
                  |j                         t	        j                  |j                  t        j                  |j                  j                  d         j                  d             y t        |t              r6| j                  j                  dk(  r?t	        j                   |j"                         t	        j                   |j$                         y | j                  j                  dk(  r?t	        j&                  |j"                         t	        j&                  |j$                         y | j                  j                  dv rmt	        j
                  |j"                  d| j                  j(                         t	        j
                  |j$                  d| j                  j(                         y y t        |t*              r t	        j
                  |j,                         y t        |t.              rd|j0                  t        j                  d	|j2                  d
      |j2                  z  z  z  }t	        j                  |j4                  |       t        j                  |j6                        j9                  |      }t        j:                  d||      }t        j<                  ||fd      }t	        j                  |j>                  |jA                                t	        j                  |jB                  |jE                                y y )Nr7   )r   r7   r7  r   >   r:  r;  r<  r  )r   r=  r   r   r   r   r   )#rk   _init_weightsrS   r   initnormal_class_embeddingcopy_r   r+   r>   r?   expandr5  r   rD  zeros_r>  r?  ones_rF  r   latentsr   r   r   r   r   r   r   r   r   r   r   r   )rp   rV   r   rU   r   r   rr   s         r1   rR  z$IdeficsPreTrainedModel._init_weightsA  s   
 	f%f56LL//0JJv**ELL9L9L9R9RSU9V,W,^,^_f,gh ?@{{,,7F334F../..&8

6223

6--...2RRV443DKKDhDhiV//ct{{?c?cd S  9:LL( 01fkkell1fjj!.Lvzz.YZ[HJJv1V;;<DDXNALLAx8E))UEN3CJJv((#'')4JJv((#'')4 2r0   )r'   r(   r)   r   r-   base_model_prefixinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_can_compile_fullgraph_supports_attention_backendr)  r   r   _can_record_outputsr+   no_gradrR  r   r   s   @r1   rL  rL  /  st    (&*#.0QRN ""& -$%5Q;W
 U]]_5 5r0   rL  c                        e Zd ZdZdef fdZddZg fdZg fdZe	e
	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dej                  dz  dee   deez  fd              Z xZS )IdeficsModelz
    Transformer decoder consisting of `config.num_hidden_layers` layers. Each layer is a [`IdeficsDecoderLayer`]

    Args:
        config: IdeficsConfig
    r   c           	         t         |   |       || _        |j                  | _        |j
                  | _        t        |j
                  |j                  |j                  |j                  | j                        | _
        |j                  j                  | _        |j                  | _        |j                  | j                  _        t        |j                        | _        |j                   r]|j"                  }t%        ||j                  j&                  |j(                  |j*                  |j,                  |j.                        | _        t3        j4                  t7        |j8                        D cg c]  }t;        ||       c}      | _        |j>                  | _        |j8                  | j>                  z  }t3        j4                  t7        |      D cg c]  }tA        ||       c}      | _!        d| _"        tG        |j                  |jH                        | _%        | jM                          | jO                  |       y c c}w c c}w )N)rf   rm   rg   rc   ri   )r   Fr   )(rk   rl   r   pad_token_idri   
vocab_sizerb   additional_vocab_sizer   freeze_text_layersembed_tokensr  
image_sizer  r   vision_modeluse_resamplerperceiver_configr   r   resampler_depthresampler_n_headsresampler_head_dimresampler_n_latentsperceiver_resamplerr   
ModuleListrangenum_hidden_layersr)  layerscross_layer_intervalr5  gated_cross_attn_layersgradient_checkpointingr   r  norm	post_initfreeze_relevant_params)rp   r   rp  inum_cross_layersrr   s        r1   rl   zIdeficsModel.__init__j  s    !.. ++5!,,&,&B&B ,,#66((
 !..99#11282M2M/4V5I5IJ %66'@$$.. 00 22 33 44(D$ mm?DVE]E]?^_! 15_
 %+$?$?!!33t7P7PP')}}KPQaKbca,VqAc(
$ ',#"6#5#56;N;NO	 	##F+ ` ds   1IINc                     || j                   }|j                  r| j                  |j                         |j                  r"t	        | j
                  |j                         y y N)r\   )r   rk  freeze_text_module_exceptionsfreeze_vision_layersr`   rn  freeze_vision_module_exceptions)rp   r   s     r1   r  z#IdeficsModel.freeze_relevant_params  sQ    >[[F$$##F$H$HI&&**f>d>de 'r0   c                 X    | j                   | j                  fD ]  }t        ||        y r  )ry  r}  r`   )rp   r\   rV   s      r1   rk  zIdeficsModel.freeze_text_layers  s+    {{DII. 	FF3DE	Fr0   c                 2    t        | j                  |       y r  )r`   rn  )rp   r\   s     r1   r  z!IdeficsModel.freeze_vision_layers  s    T&&:KLr0   rF   r=   r   r#   inputs_embedsr8   r9   r:   r;   	use_cacheinterpolate_pos_encodingr  rq   rd   c           	         ||j                   n|j                   }|du |duz  rt        d      || j                  |      }|
r|t        | j                        }|j
                  \  }}}||j                         nd}||z   }|2t        j                  |||j
                  d   z   |j                         }|F|D|j                         j                  d      dz
  }|j                  |dk(  d       |dd| df   }n||j                  d      }t        d |||fD              d	k7  rt        d
      |~|j                  | j                  |      }|j
                  dd	 \  }} |j!                         j"                  ||z  g|j
                  d	d  }| j%                  ||      j&                  }nJ|H|j)                         \  }}}}|j                  | j                  |      }|j#                  ||z  ||      }| j                  j*                  rN|4| j-                        }|j)                  d      |j)                  d	      }}n|j)                         \  }}}}|}n0|#j)                  d      |j)                  d	      }}nt        d      |j#                  ||z  |      }|	j)                  d      }|	j                  d      }	|	j/                  ddd|      }	|	j#                  ||||z        }	|C|j)                         \  }}}||f}|	t        j0                  ||      }	| j3                  |	      }	nd}	|	dk(  j5                  d      j                  | j                        j7                  d      j                  |      }|2t        j0                  ||ft        j8                  |j                         }t;        | j                  |||||      }|}t=        | j>                        D ]P  \  }} || j@                  z  dk(  r+| jB                  || j@                  z     }! |!|||f|	|dd|} | |f||||d|}R | jE                  |      }|j#                  ||||      }tG        |||      S )ab  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        Nz:You must specify exactly one of input_ids or inputs_embeds)r   r   r   )rC   r7   c              3   $   K   | ]  }|d u  
 y wrR   r/   )rT   r   s     r1   rW   z'IdeficsModel.forward.<locals>.<genexpr>  s     aQqDyas   r   z_Exactly 1 of pixel_values, image_encoder_embeddings or perceiver_embeddings has to be not-None.)rh   rC   )r8   r  zBIf `perceiver_embeddings` are passed, use_resampler should be Truer  r   r   )r   input_embedsr=   r  r#   r   )r;   rH  r#   )r=   r   r#   r  )r"   r&   r#   )$rC   rj   rl  r   r   r?   get_seq_lengthr+   r>   longcumsummasked_fill_r   sumrB   rh   r   r@   rn  r"   r  ro  ru  rA   r   invert_attention_maskrY   squeezer   r   	enumeratery  rz  r{  r}  r!   )"rp   rF   r=   r   r#   r  r8   r9   r:   r;   r  r  r  rq   rC   
batch_size
seq_lengthr  past_key_values_lengthseq_length_with_past
num_imagesr&   image_seq_lenimage_hidden_sizetext_seq_lenimage_batch_sizeimage_sequence_lengthimage_hidden_shaperH  causal_maskr$   idxdecoder_layercross_attn_blocks"                                     r1   r|   zIdeficsModel.forward  s3   4 &/%:!!@T@T-t";<YZZ  --i8M0*$++>O$1$7$7!
JETE`!?!?!Afg),BB!"\\&(>ATATUVAW(W`m`t`tN %,*>)..077;a?L%%n&91='J;<8L!)33A6La<1IK_"`aaeffq  %'??F?KL%1%7%7%;"J
9<22499*z:QkT`TfTfghgiTjkL #'"3"3)D\ #4 #   &1G_GdGdGfDJ
M3D":"="=DJJW]"="^"5":"::
;RTact"u;;$$#+'+'?'?@S'T$3G3L3LQ3OQeQjQjklQm0K_KdKdKfH
J7H"6!)/B/G/G/JL_LdLdefLg,Mabb166z:P]C]_pq ,0033==bA3::1aMR388\S]`mSmn*9L9Q9Q9S63Q"24I!J#+',zz2DV'T$#'#=#=>R#S #'  $83#>"C"C"C"K!O!OVZV`V`!O!a j jop j quu 

 !"ZZ12%**]MaMaN );;&))+%
 &"+DKK"8 	CT...!3#'#?#?tG`G`@`#a  0!'! *>)=$(! ! **) /- M	. 		-0166z:}^op-+ 3+
 	
r0   rR   )NNNNNNNNNNFN)r'   r(   r)   r*   r   rl   r  rk  r  r   r   r+   r'  r   r
   r,   r   r   r   r.   r!   r|   r   r   s   @r1   rf  rf  a  s   0,} 0,df 46 F 68 M  .2.204(,2615=A9=48!%0526^
##d*^
 t+^
 &&-	^

 ^
 ((4/^
 ''$.^
 #("3"3d":^
 $//$6^
 $llT1^
 $;^
 #'+^
 ((4/^
 +,^
 
/	/^
  ^
r0   rf  c            $       0    e Zd ZddiZd fd	Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dej                  dz  deej                  z  dee   deez  f d              Z	 	 	 	 	 	 	 	 	 d fd	Z	 ddedeeef   dedeeef   f fdZ xZS )IdeficsForVisionText2Textlm_head.weightmodel.embed_tokens.weightNc                    t         |   |       t        |      | _        t	        |j
                  |j                  |j                  d|j                        | _	        |j                  dkD  r
ddd| _
        | j                          y )NFr   r   r  z.model.embed_tokens.additional_embedding.weight)r  zlm_head.additional_fc.weight)rk   rl   rf  r[   r   r   ri  rj  freeze_lm_headlm_head_tied_weights_keysr~  )rp   r   rn  rr   s      r1   rl   z"IdeficsForVisionText2Text.__init__S  s{     !&)
-****$*$@$@#22
 ''!+"=0`'D# 	r0   rF   r=   r   r#   r  r8   r9   r:   r;   labelsr  r  r  logits_to_keeprq   rd   c                     | j                   d|||||||||	||d|d|}|j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }d}|
* | j                  d||
| j                  j                  d|}t        |||j                  |j                  |j                  |j                        S )aC  
        image_encoder_embeddings (`torch.FloatTensor`, *optional*):
            The output of the image encoder.
        perceiver_embeddings (`torch.FloatTensor`, *optional*):
            The output of the perceiver resampler.
        image_attention_mask (`torch.LongTensor`, *optional*):
            The attention mask for the image encoder.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoProcessor, IdeficsForVisionText2Text

        >>> model = IdeficsForVisionText2Text.from_pretrained("HuggingFaceM4/idefics-9b")
        >>> processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics-9b")

        >>> dogs_image_url_1 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image1.jpeg"
        >>> dogs_image_url_2 = "https://huggingface.co/datasets/hf-internal-testing/fixtures_nlvr2/raw/main/image2.jpeg"

        >>> prompts = [
        ...     [
        ...         "User:",
        ...         dogs_image_url_1,
        ...         "Describe this image.\nAssistant: An image of two dogs.\n",
        ...         "User:",
        ...         dogs_image_url_2,
        ...         "Describe this image.\nAssistant:",
        ...     ]
        ... ]
        >>> inputs = processor(prompts, return_tensors="pt")
        >>> generate_ids = model.generate(**inputs, max_new_tokens=6)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True)
        ```T)rF   r=   r   r#   r  r8   r9   r:   r;   r  r  return_dictr  N)r5   r  ri  )r4   r5   r#   r$   r%   r&   r/   )r[   r"   rS   r   slicer  loss_functionr   ri  r3   r#   r$   r%   r&   )rp   rF   r=   r   r#   r  r8   r9   r:   r;   r  r  r  r  r  rq   outputsr$   slice_indicesr5   r4   s                        r1   r|   z!IdeficsForVisionText2Text.forwardg  s    r 3=$** 3
)%+'%%=!5!5%=)3
 3
"  118B>SV8W~ot4]kmA}a,?@A%4%%pVFt{{OeOepiopD,#33!//)) ' ; ;
 	
r0   c                    i }|"| j                   j                  r||d<   n||d<   n||d<   |j                  dd      |d<   t        |   |f||||||
|	d||}|	#|!|d   j
                  d   }|	d d | d f   |d	<   |S )
Nr:   r9   r8   r  F)r#   r=   r  r  r   r  r;   rF   r   r;   )r   ro  poprk   prepare_inputs_for_generationr?   )rp   rF   r=   r   r  r#   r  r8   r&   r;   r  rq   images_kwargsmodel_inputsr  rr   s                  r1   r  z7IdeficsForVisionText2Text.prepare_inputs_for_generation  s      *{{((8K45<O89,8M.)4:JJ?Y[`4a01w<
+)')%!5
 
 
  +0E%k288;J3GJ;<3XL/0r0   r  rJ   rH   c                     t        |   |||fi |}d|v rT|d   }|d d dd d f   j                  d      }|j                  dd      r||d<   nt	        j
                  ||gd      |d<   |j                  |d<   |S )Nr;   r7   r   r  Tr   r&   )rk   #_update_model_kwargs_for_generationr   rE   r+   r   r&   )rp   r  rJ   rH   rq   r;   	last_maskrr   s          r1   r  z=IdeficsForVisionText2Text._update_model_kwargs_for_generation  s     wB
 	
 "\1#/0F#G ,QAX6@@CIT27@347<yyBVXaAbhi7j34 /6.I.I*+r0   rR   )NNNNNNNNNNNFNr   )	NNNNNNNNN)F)r'   r(   r)   r  rl   r   r   r+   r'  r   r
   r,   r   r   r   r   r.   r3   r|   r  r   dictr   r   r  r   r   s   @r1   r  r  P  s    *,GH(  .2.204(,2615=A9=48*.!%0526-.X
##d*X
 t+X
 &&-	X

 X
 ((4/X
 ''$.X
 #("3"3d":X
 $//$6X
 $llT1X
   4'X
 $;X
 #'+X
 ((4/X
 ell*X
  +,!X
" 
.	.#X
  X
z  !+b $)	 38n !	 
c3h r0   r  )r  rf  rL  )r   FNN)r   )r  )Lr*   collections.abcr   dataclassesr   typingr   r+   torch.nn.functionalr   r   rt    r   rS  activationsr	   cache_utilsr
   r   
generationr   masking_utilsr   modeling_layersr   modeling_outputsr   modeling_utilsr   r   r   processing_utilsr   utilsr   r   r   r   utils.genericr   r   configuration_ideficsr   	perceiverr   visionr   r   
get_loggerr'   r  r!   r3   rL   r`   rP   rb   rO   r   Moduler   r   r   r   r   r   r   r   r   r)  r5  rL  rf  r  __all__r/   r0   r1   <module>r     s8  &  $ !      & ! . ) / 9 + X X & R R ? 0 0 E 
		H	% 
@[ @ @6 
@K @ @8 *#Z +- fA fAR8SRYY 8SxJRYY J0$
uxx $
N(:P P2 %II%<<% 
% <<	%
 LL4'% % %0U)ryy U)r54 5p|&@ |~ .5_ .5 .5b k
) k
 k
\x 6 xv Rr0   