a hr@sdZddlZddlmZddlZddlmmZddlmZddl m Z ddl m Z ddl mZdd lmZdd lmZmZd d lmZmZmZd d lmZmZd dlmZmZmZd dlm Z m!Z!m"Z"GdddeZ#GdddeZ$GdddeZ%Gddde"Z&GdddeZ'GdddeZ(Gdddej)Z*GdddeZ+Gd d!d!e Z,Gd"d#d#e Z-Gd$d%d%e!Z.Gd&d'd'ej)Z/eGd(d)d)eZ0ed*d+Gd,d-d-e0Z1ed.d+Gd/d0d0e0Z2eGd1d2d2eZ3gd3Z4dS)4z%Pytorch implementation of AIMv2 ModelN)Optional)nn)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputWithPooling)PreTrainedModel)auto_docstringcan_return_tuple) CLIPModelCLIPTextEmbeddings_get_vector_norm)LlamaMLP LlamaRMSNorm) SiglipConfigSiglipTextConfigSiglipVisionConfig)SiglipAttention SiglipEncoder SiglipOutputcsDeZdZdZdeeeeeeeeeeeeeeedfdd ZZ S)Aimv2VisionConfiga This is the configuration class to store the configuration of a [`Aimv2VisionModel`]. It is used to instantiate a AIMv2 vision encoder according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the vision encoder of the AIMv2 [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. Args: hidden_size (`int`, *optional*, defaults to 1024): Dimensionality of the encoder layers and the pooler layer. intermediate_size (`int`, *optional*, defaults to 2816): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. num_hidden_layers (`int`, *optional*, defaults to 24): Number of hidden layers in the Transformer encoder. num_attention_heads (`int`, *optional*, defaults to 8): Number of attention heads for each attention layer in the Transformer encoder. num_channels (`int`, *optional*, defaults to 3): Number of channels in the input images. image_size (`int`, *optional*, defaults to 224): The size (resolution) of each image. patch_size (`int`, *optional*, defaults to 14): The size (resolution) of each patch. rms_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the rms normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. qkv_bias (`bool`, *optional*, defaults to `False`): Whether to add a bias to the queries, keys and values. mlp_bias (`bool`, *optional*, defaults to `False`): Whether to add a bias to the Linear layers or Not. hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported. initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the for initializing all weight matrices. use_head (`str`, *optional*, defaults to `True`): Whether to use Attention Pooling Head or Not. is_native (`str`, *optional*, defaults to `False`): Whether to use ckpt trained for image native resolution or not. Example: ```python >>> from transformers import SiglipVisionConfig, SiglipVisionModel >>> # Initializing a Aimv2VisionConfig with apple/aimv2-large-patch14-224 style configuration >>> configuration = Aimv2VisionConfig() >>> # Initializing a Aimv2VisionModel (with random weights) from the apple/aimv2-large-patch14-224 style configuration >>> model = Aimv2VisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ``` rh㈵>Fsilu{Gz?T) hidden_sizeintermediate_sizenum_hidden_layersnum_attention_heads num_channels image_size patch_size rms_norm_epsattention_dropoutqkv_biasmlp_bias hidden_actinitializer_rangeuse_head is_nativec sXtjf||||| |||| d |||_| |_| |_| |_| |_||_||_|` dS)N) r"r#r$r%r-r&r'r(r+) super__init__r/r.r*r,r+r)r0layer_norm_eps)selfr"r#r$r%r&r'r(r)r*r+r,r-r.r/r0kwargs __class__c/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/models/aimv2/modular_aimv2.pyr2`s*  zAimv2VisionConfig.__init__)rrrrrrrrrFFr r!TF) __name__ __module__ __qualname____doc__intfloatboolstrr2 __classcell__r8r8r6r9r'sB:rcsLeZdZdZdeeeeeeeeeeeeeeeeedfdd Z Z S)Aimv2TextConfiga This is the configuration class to store the configuration of a [`Aimv2TextModel`]. It is used to instantiate a AIMv2 text encoder according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the text encoder of the AIMv2 [apple/aimv2-large-patch14-224-lit](https://huggingface.co/apple/aimv2-large-patch14-224-lit) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. Args: vocab_size (`int`, *optional*, defaults to 49408): Vocabulary size of the AIMv2 text model. Defines the number of different tokens that can be represented by the `inputs_ids` passed when calling [`Aimv2Model`]. hidden_size (`int`, *optional*, defaults to 768): Dimensionality of the encoder layers and the pooler layer. intermediate_size (`int`, *optional*, defaults to 2048): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. num_hidden_layers (`int`, *optional*, defaults to 12): Number of hidden layers in the Transformer encoder. num_attention_heads (`int`, *optional*, defaults to 6): Number of attention heads for each attention layer in the Transformer encoder. rms_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the rms normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. qkv_bias (`bool`, *optional*, defaults to `False`): Whether to add a bias to the queries, keys and values. mlp_bias (`bool`, *optional*, defaults to `False`): Whether to add a bias to the Linear layers or Not. hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported. pad_token_id (`int`, *optional*, defaults to 1): The id of the padding token in the vocabulary. bos_token_id (`int`, *optional*, defaults to 49406): The id of the beginning-of-sequence token in the vocabulary. eos_token_id (`int`, *optional*, defaults to 49407): The id of the end-of-sequence token in the vocabulary. max_position_embeddings (`int`, *optional*, defaults to 77): The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the for initializing all weight matrices.  rrFr NMr!) vocab_sizer"r#r$r%r)r*r+r,r- pad_token_id bos_token_id eos_token_idmax_position_embeddingsr.c sZtjf|||||| || | | d |||_||_| |_||_||_|`|`|` |` dS)N) rKr"r#r$r%r-rOrLrMrN) r1r2r.r*r,r+r)rMrLZprojection_sizer3)r4rKr"r#r$r%r)r*r+r,r-rLrMrNrOr.r5r6r8r9r2s. zAimv2TextConfig.__init__)rDrErFrGrHrrFFr NNrIrJr!) r:r;r<r=r>r?r@rArr2rBr8r8r6r9rCsB/rCcs"eZdZdZdfdd ZZS) Aimv2Configa@ [`Aimv2Config`] is the configuration class to store the configuration of a [`Aimv2Model`]. It is used to instantiate a AIMv2 model according to the specified arguments, defining the text model and vision model configs. Instantiating a configuration with the defaults will yield a similar configuration to that of the AIMv2 [apple/aimv2-large-patch14-224-lit](https://huggingface.co/apple/aimv2-large-patch14-224-lit) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. Args: text_config (`dict`, *optional*): Dictionary of configuration options used to initialize [`Aimv2TextConfig`]. vision_config (`dict`, *optional*): Dictionary of configuration options used to initialize [`Aimv2VisionConfig`]. projection_dim (`int`, *optional*, defaults to 512): Dimensionality of text and vision projection layers. logit_scale_init_value (`float`, *optional*, defaults to 2.6592): The initial value of the *logit_scale* parameter. kwargs (*optional*): Dictionary of keyword arguments. Example: ```python >>> from transformers import Aimv2Config, Aimv2Model >>> # Initializing a Aimv2Config with apple/aimv2-large-patch14-224-lit style configuration >>> configuration = Aimv2Config() >>> # Initializing a Aimv2Model (with random weights) from the apple/aimv2-large-patch14-224-lit style configuration >>> model = Aimv2Model(configuration) >>> # Accessing the model configuration >>> configuration = model.config >>> # We can also initialize a Aimv2Config from a Aimv2TextConfig and a Aimv2VisionConfig >>> from transformers import Aimv2TextConfig, Aimv2VisionConfig >>> # Initializing a AIMv2Text and AIMv2Vision configuration >>> config_text = Aimv2TextConfig() >>> config_vision = Aimv2VisionConfig() >>> config = Aimv2Config(text_config=config_text, vision_config=config_vision) ```N/L F@c s0tj||fi|||_||_d|_|`dS)NgY@)r1r2projection_dimlogit_scale_init_valuemax_logit_scaleZinitializer_factor)r4 text_config vision_configrSrTr5r6r8r9r2s zAimv2Config.__init__)NNrQrR)r:r;r<r=r2rBr8r8r6r9rPs.rPc@s eZdZdS) Aimv2OutputNr:r;r<r8r8r8r9rXsrXc@s eZdZdS) Aimv2RMSNormNrYr8r8r8r9rZ#srZc@s eZdZdS)Aimv2MLPNrYr8r8r8r9r['sr[csVeZdZedfdd Zedddejfejddd Z ejejd d d Z Z S) Aimv2VisionEmbeddingsconfigcst||_|j|_tj|j|j|j|jd|_t |j|j |_ |j |jd}|jj slt||j|_|jdt|ddddS)N)Z kernel_sizeZstrider position_ids)F) persistent)r1r2r^r(rZConv2dr&r" patch_embedrZr)rms_normr'r0Z Embeddingposition_embeddingZregister_buffertorcharangeexpand)r4r^Z num_patchesr6r8r9r2,s zAimv2VisionEmbeddings.__init__g@cpureturnc Cstjt|||d}tjt|||d}tj||dd\}}|d}tj|||d|} d|| } |d| dddf} |d| dddf} tj| | | | gdddddddfS) NdtypedeviceZxy)Zindexingg?).Nr`dim)rfrgr>Zmeshgridflattenconcatsincos) heightwidth embed_dimZ temperaturerornZgrid_wZgrid_hZpos_dimomegaZout_hZout_wr8r8r9"build_2d_sincos_position_embedding:s z8Aimv2VisionEmbeddings.build_2d_sincos_position_embedding) pixel_valuesrlcCs||\}}}}||ddd}||}|jjrd|j||j||j|jj |j |j d}n | |j }||}|S)Nr r`)ryrorn)sizercrs transposerdr^r0r{r(r"rornrer_)r4r|_rwrx hidden_statesZ pos_embedr8r8r9forwardKs  zAimv2VisionEmbeddings.forward) r:r;r<rr2 staticmethodrfZfloat32Tensorr{rrBr8r8r6r9r\+s r\c@s eZdZdS)Aimv2TextEmbeddingsNrYr8r8r8r9r_srcseZdZfddZZS)Aimv2Attentioncspt|tj|j|j|jd|_tj|j|j|jd|_tj|j|j|jd|_tj|j|j|jd|_ dS)NZbias) r1r2rLinearryr+k_projv_projZq_projZout_projr4r^r6r8r9r2ds  zAimv2Attention.__init__)r:r;r<r2rBr8r8r6r9rcsrcsPeZdZedfdd Zd ejeejeee ejejfdddZ Z S) Aimv2EncoderLayerr]csBtt||_t||_t|j|j|_ t|j|j|_ dSN) r1r2r attentionr[ffnrZr"r) rms_norm1 rms_norm2rr6r8r9r2ms    zAimv2EncoderLayer.__init__NF)rattention_maskoutput_attentionsrlcCsT||}|j||d\}}||}||}||}||}|rL||fS|dfS)N)rr)rrrr)r4rrrZnorm_hidden_states attn_outputZ attn_weightsZ mlp_outputr8r8r9rts   zAimv2EncoderLayer.forward)NF) r:r;r<rr2rfrrr@tuplerrBr8r8r6r9rls rc@s eZdZdS) Aimv2EncoderNrYr8r8r8r9rsrcs6eZdZedfdd ZejejdddZZS)Aimv2AttentionPoolingHeadr]cs|t|j|_|j|_tj|j|j|jd|_tj|j|j|jd|_ t t dd|j|_ tj|j|jdd|_dS)Nrr`T)r1r2r"r% num_headsrrr+rr ParameterrfZzeros cls_token output_projrr6r8r9r2s z"Aimv2AttentionPoolingHead.__init__)rrlc Cs|j\}}}|j|dd}|||||j||j}|||||j||j}||d|j||j}|dddd}|dddd}|dddd}t |||} | dd|d|} | j dd} | | } | S)Nrar`rr rrq) shaperrhrZreshaperrZpermuteFZscaled_dot_product_attentionr~meanr) r4r batch_sizeseq_lenZ hidden_dimrkeyvaluequeryroutputr8r8r9rs   z!Aimv2AttentionPoolingHead.forward) r:r;r<rr2rfrrrBr8r8r6r9rs rcsFeZdZUdZeed<dZdZgdZdZ dZ dZ fddZ Z S)Aimv2PreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. The model is only intended for inference and doesn't support finetuning. r^Zaimv2T)rrr\rcs^t|t|dr:t|jtjrZ|jjt dn t|t rZ|j jj d|jjddS)N logit_scaleg$I$I,@r)rZstd)r1 _init_weightshasattr isinstancerrrdataZfill_mathlogrrZnormal_r^r.)r4moduler6r8r9rs    z"Aimv2PreTrainedModel._init_weights)r:r;r<r=rP__annotations__Zbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesZ_supports_sdpa_supports_flash_attnZ_supports_flex_attnrrBr8r8r6r9rs rzL The Vision model from AIMv2 without any head or projection on top. )Z custom_introcsleZdZUeed<dZedfdd ZejdddZ e e d e e je ee eed d d ZZS)Aimv2VisionModelr^r|r]csZt|||_t||_t||_t|j|j |_ |j |_ |j rNt ||_ |dSr)r1r2r^r\ embeddingsrencoderrZr"r)rdr/rhead post_initrr6r8r9r2s    zAimv2VisionModel.__init__rkcCs|jjSr)rrcr4r8r8r9get_input_embeddingssz%Aimv2VisionModel.get_input_embeddingsNrroutput_hidden_statesrlc Cs||dur |n|jj}|dur |n|jj}||}|j|||d}|d}||}|jrd||nd}t|||j |j dS)a Examples: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, Siglip2VisionModel >>> model = Aimv2VisionModel.from_pretrained("apple/aimv2-large-patch14-native") >>> processor = AutoProcessor.from_pretrained("apple/aimv2-large-patch14-native") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled features ```N) inputs_embedsrrrlast_hidden_state pooler_outputr attentions) r^rrrrrdr/rrrr) r4r|rrrrencoder_outputsrrr8r8r9rs$  zAimv2VisionModel.forward)NNN)r:r;r<rrmain_input_namer2rModulerr r rrfrr@rrrBr8r8r6r9rs rzJ The text model from AIMv2 without any head or projection on top. csjeZdZdZedfdd ZejdddZdd Z e e de e je ee eed d d ZZS)Aimv2TextModel input_idsr]csJt|||_t||_t||_t|j|j |_ |j |_ | dSr) r1r2r^rrrrrZr"r)rdrNrrr6r8r9r2!s   zAimv2TextModel.__init__rkcCs|jjSrrZtoken_embeddingrr8r8r9r,sz#Aimv2TextModel.get_input_embeddingscCs ||j_dSrr)r4rr8r8r9set_input_embeddings/sz#Aimv2TextModel.set_input_embeddingsNrcCs|dur |n|jj}|dur |n|jj}||}|j\}}}tj|tj|jd} | d |d} |durt |j|| || dd}|j ||||d} | d} | | } | tj| jd| jd|jtj| jd|jkjddf} t| | | j| jdS) Nrmrra)r^Z input_embedsr_rcache_positionZpast_key_values)rrrr)rorqr)r^rrrrrfrglongroZ unsqueezerhrrrdtor>rNZargmaxrrr)r4rrrrrrrrrr_rrZ pooled_outputr8r8r9r2sF     "zAimv2TextModel.forward)NNN)r:r;r<rrCr2rrrrr r rrfrr@rrrBr8r8r6r9rs rc @sZeZdZdZedddZeed ee j ee j ee j ee ee edddZdS) Aimv2ModelTr]cCst|||j|_|jj|_|jj|_t |j|_ t |j|_ t j|j|jdd|_t j|j|jdd|_t t|jj|_t|j|_|dS)NFr)rr2rSrWr"Zvision_embed_dimrVZtext_embed_dimr _from_config vision_modelr text_modelrrvisual_projectiontext_projectionrrfZtensorr^rTrrrrUmax_log_logit_scalerrr8r8r9r2ks   zAimv2Model.__init__N)rr|rrrrlc Cs|dur |n|jj}|dur |n|jj}|j|||d}|j||||d}|j}||}|j} || } |t|}| t| } |j d|j  | j} | | |} | } t| | | |||dS)a Examples: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, Aimv2Model >>> model = Aimv2Model.from_pretrained("apple/aimv2-large-patch14-224-lit") >>> processor = AutoProcessor.from_pretrained("apple/aimv2-large-patch14-224-lit") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor( ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True ... ) >>> outputs = model(**inputs) >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities ```N)r|rr)rrrrr)logits_per_imagelogits_per_text text_embeds image_embedsZtext_model_outputZvision_model_output)r^rrrrrrrrrclamprexprrotrX) r4rr|rrrZvision_outputsZ text_outputsrrrrrr8r8r9r}s>!    zAimv2Model.forward)NNNNN)r:r;r<rrPr2r r rrfZ LongTensorZ FloatTensorrr@rXrr8r8r8r9rgs"r)rPrrCrrrr)5r=rtypingrrfZtorch.nn.functionalrZ functionalrZ masking_utilsrZmodeling_layersrZmodeling_outputsrZmodeling_utilsrutilsr r Zclip.modeling_clipr r rZllama.modeling_llamarrZsiglip.configuration_sigliprrrZsiglip.modeling_sigliprrrrrCrPrXrZr[rr\rrrrrrrrr__all__r8r8r8r9sN      d[94 "LI`