a ha[@svddlmZmZmZddlZddlmZddlmZddl m Z ddl m Z m Z ddlmZmZddlmZdd lmZmZd d lmZmZmZmZmZmZmZd d lmZd d l m!Z!m"Z"e#e$Z%GdddeZ&GdddeZ'Gddde!Z(GdddeZ)GdddeZ*GdddeZ+GdddeZ,GdddeZ-eGdddeZ.Gd d!d!eZ/gd"Z0dS)#)CallableOptionalUnionN)PretrainedConfig)FlashAttentionKwargs)BaseModelOutputBaseModelOutputWithPooling)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringlogging)CLIPMLP CLIPAttention CLIPEncoderCLIPEncoderLayerCLIPVisionEmbeddingsCLIPVisionModelCLIPVisionTransformer)eager_attention_forward)VisionRotaryEmbeddingapply_rotary_pos_emb_visionc s*eZdZdZdZdZdfdd ZZS)MLCDVisionConfiga This is the configuration class to store the configuration of a [`MLCDVisionModel`]. It is used to instantiate a MLCD vision encoder according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the vision encoder of the MLCD [DeepGlint-AI/mlcd-vit-bigG-patch14-336](https://huggingface.co/DeepGlint-AI/mlcd-vit-bigG-patch14-336) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. Args: hidden_size (`int`, *optional*, defaults to 1664): Dimensionality of the encoder layers and the pooler layer. intermediate_size (`int`, *optional*, defaults to 8192): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. projection_dim (`int`, *optional*, defaults to 1024): Dimensionality of text and vision projection layers. num_hidden_layers (`int`, *optional*, defaults to 48): Number of hidden layers in the Transformer encoder. num_attention_heads (`int`, *optional*, defaults to 16): Number of attention heads for each attention layer in the Transformer encoder. num_channels (`int`, *optional*, defaults to 3): The number of input channels. image_size (`int`, *optional*, defaults to 336): The size (resolution) of each image. patch_size (`int`, *optional*, defaults to 14): The size (resolution) of each patch. hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the layer normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. initializer_factor (`float`, *optional*, defaults to 1.0): A factor for initializing all weight matrices (should be kept to 1, used internally for initialization testing). Example: ```python >>> from transformers import MLCDVisionConfig, MLCDVisionModel >>> # Initializing a MLCDVisionConfig with DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> configuration = MLCDVisionConfig() >>> # Initializing a MLCDVisionModel (with random weights) from the DeepGlint-AI/mlcd-vit-bigG-patch14-336 style configuration >>> model = MLCDVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```Zmlcd_vision_modelZ vision_config 0rPgeluh㈵>{Gz??c sdtjfi|||_||_||_||_||_||_||_||_ | |_ | |_ | |_ | |_ | |_dSN)super__init__ hidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_groups num_channels patch_size image_sizeinitializer_rangeinitializer_factorattention_dropoutlayer_norm_eps hidden_act)selfr*r+r,r-r.r/r1r0r6r5r4r2r3kwargs __class__a/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/models/mlcd/modular_mlcd.pyr)dszMLCDVisionConfig.__init__) rrrrrrr r!r"r#r$r%r&)__name__ __module__ __qualname____doc__Z model_typeZbase_config_keyr) __classcell__r;r;r9r<r*s"6rc@s eZdZdS)MLCDMLPN)r=r>r?r;r;r;r<rBsrBc@s eZdZeeejdddZdS)MLCDRotaryEmbedding)num_patches_heightnum_patches_widthreturnc Cstj||jjddd|}tj||jjdd|d}tj||gdd}t||}tj||jj|jj d}t ||j}||d} | S)a} Calculate the Rotary Position Embedding (RoPE) for MLCDVisionModel based on the grid size. Args: num_patches_height (int): Number of patches in the height dimension. num_patches_width (int): Number of patches in the width dimension. Returns: torch.Tensor: Rotary positional embeddings for the given grid size. )devicerrdim)rGdtype) torchZarangeZinv_freqrG unsqueezeexpandstackflattenmaxrKouter) r7rDrEZhpos_idsZwpos_idsZpos_idsZ max_grid_sizeseqZrotary_pos_emb_fullrotary_pos_embr;r;r<forwards  zMLCDRotaryEmbedding.forwardN)r=r>r?intrLTensorrUr;r;r;r<rCsrCcs6eZdZedfdd ZejejdddZZ S)MLCDVisionEmbeddingsconfigcst||`dSr')r(r)Zposition_embeddingr7rZr9r;r<r)s zMLCDVisionEmbeddings.__init__) pixel_valuesrFcCs^|jd}|jjj}||j|d}|ddd}|j|dd}t j ||gdd}|S)Nr)rKrrrHrI) shapepatch_embeddingweightrKtorPZ transposeclass_embeddingrNrLcat)r7r\ batch_sizeZ target_dtypeZ patch_embedsZ class_embeds embeddingsr;r;r<rUs  zMLCDVisionEmbeddings.forward) r=r>r?rr)rL FloatTensorrWrUrAr;r;r9r<rXsrXc sfeZdZdZedfdd Zd ejeejejfe eje e eeje ejfdddZ Z S) MLCDAttentionzMulti-headed attention with RoPE. Refer to papers: - Attention is all you need: https://huggingface.co/papers/1706.03762 - RoFormer: Enhanced Transformer with Rotary Position Embedding: https://huggingface.co/papers/2104.09864 rYcst||j|_d|_dS)NF)r(r)r. is_causalr[r9r;r<r)s zMLCDAttention.__init__N) hidden_statesposition_embeddingsattention_maskr8rFcKsp|jdd\}}|||||j|jf}|||||j|jf}|||||j|jf} |dd} |dd} t ||| | \}}| dddd }| dddd }| dddd } t } |j jdkrt|j j} | |||| |f|jsdn|j|j|jd|\} }| dddd } | ||d} || } | ddd } | |fS) NrHrrrreagerr$)dropoutZscalingrg)r]q_projZreshapeZ num_headsZhead_dimk_projv_projrMfloatrZpermute contiguousrrZZ_attn_implementationr Ztrainingrlscalergviewout_proj)r7rhrirjr8rcZ seq_lengthZ query_statesZ key_statesZ value_statescossinZattention_interfaceZ attn_output attn_weightsr;r;r<rUs>     zMLCDAttention.forward)N)r=r>r?r@rr)rLrWtuplerr rrUrAr;r;r9r<rfs rfcsXeZdZedfdd Zd ejeejejfeejee eej dddZ Z S) MLCDEncoderLayerrYcst|t||_dSr')r(r)rf self_attnr[r9r;r<r)s zMLCDEncoderLayer.__init__NF)rhrirjoutput_attentionsrFcCsd|}||}|j||||d\}}||}|}||}||}||}|f}|r`||f7}|S)a Args: hidden_states (`torch.FloatTensor`): Input to the layer of shape `(batch, seq_len, embed_dim)`. Represents the hidden states from the previous layer or the input embeddings. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.FloatTensor`): Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values. output_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. rhrirjr{)Z layer_norm1rzZ layer_norm2Zmlp)r7rhrirjr{Zresidualrwoutputsr;r;r<rUs"     zMLCDEncoderLayer.forward)NF) r=r>r?rr)rLrWrxrboolrerUrAr;r;r9r<rysryc sjeZdZdZedfdd Zd ejeej ej fe ej e e e e e e e ee fdddZZS) MLCDEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`MLCDEncoderLayer`]. Args: config: MLCDVisionConfig rYcst|dS)z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.N)r(r)r[r9r;r<r)5szMLCDEncoder.__init__N) inputs_embedsrirjr{output_hidden_states return_dictrFc Cs|dur |n|jj}|dur |n|jj}|dur4|n|jj}|rDdnd}|rPdnd}|} t|jD]@\} } |rx|| f}| | |||d} | d} |rb|| df}qb|r|| f}|stdd| ||fDSt| ||dS) aj Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr;r|rrcss|]}|dur|VqdSr'r;).0vr;r;r< wz&MLCDEncoder.forward..)last_hidden_staterh attentions)rZruse_return_dictr{ enumerateZlayersrxr) r7rrirjr{rrZencoder_statesZall_attentionsrhidxZ encoder_layerZ layer_outputsr;r;r<rU9s8"    zMLCDEncoder.forward)NNNN)r=r>r?r@rr)rLrerxrWrr~rrrUrAr;r;r9r<r,s rc sXeZdZedfdd Zedeejee ee ee e e e fdddZ ZS) MLCDVisionTransformerrYcsFt|t|j|jd|_tt d|j|jd|_ dS)Nrr) r(r)rCr*r-vision_rotary_embeddingnn ParameterrLZrandn class_pos_embr[r9r;r<r)s zMLCDVisionTransformer.__init__Nr\r{rrrFcCs>|dur |n|jj}|dur |n|jj}|dur4|n|jj}|durLtd|jd|jj}|jd|jj}|||}||j j }t j |j |gdd}t j ||fdd}| |f} ||} || } |j| | |||d} | d} | dddddf} || } |s*| | f| ddSt| | | j| jdS) Nz You have to specify pixel_valuesrHrrI)rrir{rrr)rZ pooler_outputrhr)rZrrr{ ValueErrorr]r0rr`rrGrLrbrurvrdZ pre_layrnormencoderZpost_layernormr rhr)r7r\r{rrrDrErTZembrirhZencoder_outputsrZ pooled_outputr;r;r<rUsB     zMLCDVisionTransformer.forward)NNNN)r=r>r?rr)r rrLrer~rrxr rUrAr;r;r9r<rs rc@s.eZdZUeed<dZdZdZdZddZ dS)MLCDPreTrainedModelrZZmlcdTcCs|jj}t|trX|jj}tjj|jd|jd|dtjj|j j |jj |dnt|t r|jj}|jdd|jj d|}|jd|}tjj|jj |dtjj|jj |dtjj|jj |dtjj|jj |dnt|tr\|jj}|jjdd|jj d|}d|jjd|}tjj|jj |dtjj|jj |dnt|tr|jj}|jj|jjdd|}tjj|jd|dnPt|tjr|jj|j jdn&t|tjr|jdur|jjdS)zInitialize the weightsr$g)meanstd)rrr&N)rZr3 isinstancerXrinitZnormal_raZ embed_dimr^r_r2rfr,rmrnrortrBr*Zfc1Zfc2rr-rZ LayerNormZbiasdataZzero_Zfill_ZLinear)r7modulefactorZ in_proj_stdZ out_proj_stdZfc_stdZ pos_emb_stdr;r;r< _init_weightss8       z!MLCDPreTrainedModel._init_weightsN) r=r>r?r__annotations__Zbase_model_prefixZsupports_gradient_checkpointingZ_supports_flash_attnZ_supports_sdparr;r;r;r<rs rc @sBeZdZedeejeeeeeeee e fdddZ dS)MLCDVisionModelNrcCsN|dur |n|jj}|dur |n|jj}|dur4|n|jj}|j||||dS)a Example: ```python >>> import requests >>> from PIL import Image >>> from transformers import AutoProcessor, MLCDVisionModel >>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs, output_attentions=True) >>> features = outputs.last_hidden_state >>> print(f"Extracted features shape: {features.shape}") >>> print(f"Number of attention layers: {len(outputs.attentions)}") >>> print(f"Attention shape: {outputs.attentions[0].shape}") ```N)r\r{rr)rZrrr{Z vision_model)r7r\r{rrr;r;r<rUszMLCDVisionModel.forward)NNNN) r=r>r?r rrLrer~rrxr rUr;r;r;r<rs r)rrr)1typingrrrrLZtorch.nnrZconfiguration_utilsrZmodeling_flash_attention_utilsrZmodeling_outputsrr Zmodeling_utilsr r Zprocessing_utilsr utilsr rZclip.modeling_cliprrrrrrrZllama.modeling_llamarZqwen2_vl.modeling_qwen2_vlrrZ get_loggerr=loggerrrBrCrXrfryrrrr__all__r;r;r;r<s0    $  \"<2S9'-