a hj @sddlmZmZmZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZddlmZmZdd lmZdd lmZmZmZd d lmZGd ddejZGdddejZGdddejZd-ejejejejeeje e eedddZ!ddZ"eje#ejdddZ$ejejejeje%ejejfdddZ&Gdd d ejZ'Gd!d"d"e Z(Gd#d$d$ejZ)Gd%d&d&ejZ*eGd'd(d(eZ+ed)d*Gd+d,d,e+Z,d(d,gZ-dS).)CallableOptionalUnionN)ACT2FN)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPooling)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstring torch_int)MLCDVisionConfigcs0eZdZfddZejejdddZZS)MLCDMLPcsDt||_t|j|_t|j|j |_ t|j |j|_ dSN) super__init__configrZ hidden_act activation_fnnnLinear hidden_sizeZintermediate_sizefc1fc2selfr __class__b/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/models/mlcd/modeling_mlcd.pyr%s   zMLCDMLP.__init__) hidden_statesreturncCs"||}||}||}|Sr)rrr)rr$r"r"r#forward,s   zMLCDMLP.forward)__name__ __module__ __qualname__rtorchTensorr& __classcell__r"r"r r#r$s rcsHeZdZUejed<d eeddfdd Zeeejddd Z Z S) MLCDRotaryEmbeddinginv_freq@N)dimthetar%cs>td|tjd|dtjd|}|jd|dddS)N?rdtyper.F persistent)rrr*arangefloatregister_buffer)rr0r1r.r r"r#r6s  zMLCDRotaryEmbedding.__init__)num_patches_heightnum_patches_widthr%c Cstj||jjddd|}tj||jjdd|d}tj||gdd}t||}tj||jj|jj d}t ||j}||d} | S)a} Calculate the Rotary Position Embedding (RoPE) for MLCDVisionModel based on the grid size. Args: num_patches_height (int): Number of patches in the height dimension. num_patches_width (int): Number of patches in the width dimension. Returns: torch.Tensor: Rotary positional embeddings for the given grid size. )devicerrr0)r=r5) r*r8r.r= unsqueezeexpandstackflattenmaxr5outer) rr;r<Zhpos_idsZwpos_idsZpos_idsZ max_grid_sizeseqZrotary_pos_emb_fullrotary_pos_embr"r"r#r&;s  zMLCDRotaryEmbedding.forward)r/) r'r(r)r*r+__annotations__intr9rr&r,r"r"r r#r-3s  r-csNeZdZedfdd ZejeeejdddZej ejddd Z Z S) MLCDVisionEmbeddingsrcst||_|j|_|j|_|j|_tt |j|_ tj |j |j|j|jdd|_|j|jd|_|jd|_|jdt |jddddS)NF)Z in_channelsZ out_channelsZ kernel_sizeZstridebiasr3r position_ids)rr>r6)rrrr embed_dimZ image_size patch_sizer Parameterr*randnclass_embeddingZConv2dZ num_channelspatch_embedding num_patches num_positionsr:r8rArr r"r#r]s   zMLCDVisionEmbeddings.__init__) embeddingsheightwidthr%c Cs|jdd}|jjd}|jdd}tjsP||krP||krP||jS|ddddf}|ddddf}|jd} ||j} ||j} t |d} | d| | | }| dddd}t j j|| | fdd d }| dddddd| }tj||fdd S) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 rrNr>g?rr3ZbicubicF)sizemodeZ align_cornersr?)shapeposition_embeddingweightr@r*Zjit is_tracingrMrOrreshapepermuter functionalZ interpolateviewcat) rrVrWrXrTr\rUZclass_pos_embedZpatch_pos_embedr0Z new_heightZ new_widthZsqrt_num_positionsr"r"r#interpolate_pos_encodingrs*      z-MLCDVisionEmbeddings.interpolate_pos_encoding) pixel_valuesr%cCs^|jd}|jjj}||j|d}|ddd}|j|dd}t j ||gdd}|S)Nrr4r3rr>r?) r[rSr]r5torC transposerRrAr*rc)rre batch_sizeZ target_dtypeZ patch_embedsZ class_embedsrVr"r"r#r&s  zMLCDVisionEmbeddings.forward) r'r(r)rrr*r+rIrd FloatTensorr&r,r"r"r r#rJ\s)rJ)modulequerykeyvalueattention_maskscalingdropoutkwargsc Kst||j}t||j} t||dd|} |durf|ddddddd|jdf} | | } tjj| dtj d |j } tjj | ||j d} t| | } | dd} | | fS)Nr3rr>)r0r5)ptrainingr) repeat_kvnum_key_value_groupsr*matmulrgr[rraZsoftmaxZfloat32rfr5rqru contiguous) rkrlrmrnrorprqrr key_states value_states attn_weightsZ causal_mask attn_outputr"r"r#eager_attention_forwards  & r~cCsH|dd|jddf}|d|jdddf}tj| |fddS)z*Rotates half the hidden dims of the input..Nr>r3r?)r[r*rc)xx1Zx2r"r"r# rotate_halfsr)r$n_repr%cCs^|j\}}}}|dkr|S|dddddddddf|||||}||||||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)r[rAr_)r$rbatchZnum_key_value_headsslenhead_dimr"r"r#rvs 0rv)qkcossinr%cCs|j}|j}||}}|d|d}}||t||}||t||}||}||}||fS)Nrs)r5r9r@rrf)rrrrZ orig_q_dtypeZ orig_k_dtypeZq_embedZk_embedr"r"r#apply_rotary_pos_emb_visions  rc sfeZdZdZedfdd Zd ejeejejfe eje e eeje ejfdddZ Z S) MLCDAttentionzMulti-headed attention with RoPE. Refer to papers: - Attention is all you need: https://huggingface.co/papers/1706.03762 - RoFormer: Enhanced Transformer with Rotary Position Embedding: https://huggingface.co/papers/2104.09864 rKcst||_|j|_|j|_|j|j|_|j|j|jkrZtd|jd|jd|jd|_ |j |_ d|_ t |j|j|_t |j|j|_t |j|j|_t |j|j|_|j|_dS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).F)rrrrrNnum_attention_heads num_headsr ValueErrorscaleZattention_dropoutrq is_causalrrk_projv_projq_projout_projrwrr r"r#rs&   zMLCDAttention.__init__N)r$position_embeddingsrorrr%cKsp|jdd\}}|||||j|jf}|||||j|jf}|||||j|jf} |dd} |dd} t ||| | \}}| dddd }| dddd }| dddd } t } |j jdkrt|j j} | |||| |f|jsdn|j|j|jd |\} }| dddd } | ||d} || } | ddd } | |fS) z#Input shape: Batch x Time x ChannelNr>rrr3reagerrj)rqrpr)r[rr_rrrrr@r9rr`ryr~rZ_attn_implementationr rurqrrrbr)rr$rrorrrhZ seq_lengthZ query_statesrzr{rrZattention_interfacer}r|r"r"r#r&s>     zMLCDAttention.forward)N)r'r(r)__doc__rrr*r+tuplerr rr&r,r"r"r r#rsrcsXeZdZedfdd Zd ejeejejfeejee eej dddZ Z S) MLCDEncoderLayerrKcsRt|j|_t||_tj|j|jd|_ t ||_ tj|j|jd|_ dS)Neps) rrrrNr self_attnr LayerNormlayer_norm_eps layer_norm1rmlp layer_norm2rr r"r#r1s    zMLCDEncoderLayer.__init__NF)r$rrooutput_attentionsr%cCsd|}||}|j||||d\}}||}|}||}||}||}|f}|r`||f7}|S)a Args: hidden_states (`torch.FloatTensor`): Input to the layer of shape `(batch, seq_len, embed_dim)`. Represents the hidden states from the previous layer or the input embeddings. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.FloatTensor`): Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values. output_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. r$rror)rrrr)rr$rrorZresidualr|outputsr"r"r#r&9s"     zMLCDEncoderLayer.forward)NF) r'r(r)rrr*r+rrboolrir&r,r"r"r r#r0s rc sjeZdZdZedfdd Zd ejeej ej fe ej e e e e e e e ee fdddZZS) MLCDEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`MLCDEncoderLayer`]. Args: config: MLCDVisionConfig rKcs:t|_tfddtjD|_d|_dS)z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.csg|] }tqSr")r).0_rKr"r# sz(MLCDEncoder.__init__..FN) rrrrZ ModuleListrangenum_hidden_layerslayersZgradient_checkpointingrr rKr#ros  zMLCDEncoder.__init__N) inputs_embedsrroroutput_hidden_states return_dictr%c Cs|dur |n|jj}|dur |n|jj}|dur4|n|jj}|rDdnd}|rPdnd}|} t|jD]@\} } |rx|| f}| | |||d} | d} |rb|| df}qb|r|| f}|stdd| ||fDSt| ||dS) aj Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. position_embeddings (`tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr"rrrcss|]}|dur|VqdSrr")rvr"r"r# rz&MLCDEncoder.forward..)last_hidden_stater$ attentions)rruse_return_dictr enumeraterrr ) rrrrorrrZencoder_statesZall_attentionsr$idxZ encoder_layerZ layer_outputsr"r"r#r&vs8"    zMLCDEncoder.forward)NNNN)r'r(r)rrrr*rirr+rrrr r&r,r"r"r r#rfs  rc sXeZdZedfdd Zedeejee ee ee e e e fdddZ ZS) MLCDVisionTransformerrKcst||_|j}t||_tj||jd|_ t ||_ tj||jd|_ t |j|jd|_ttd|j|jd|_dS)Nrr3r)rrrrrJrVrrr pre_layrnormrencoderpost_layernormr-rvision_rotary_embeddingrPr*rQ class_pos_emb)rrrNr r"r#rs   zMLCDVisionTransformer.__init__Nrerrrr%cCs>|dur |n|jj}|dur |n|jj}|dur4|n|jj}|durLtd|jd|jj}|jd|jj}|||}||j j }t j |j |gdd}t j ||fdd}| |f} ||} || } |j| | |||d} | d} | dddddf} || } |s*| | f| ddSt| | | j| jdS) Nz You have to specify pixel_valuesrsr>rr?)rrrrrr)rZ pooler_outputr$r)rrrrrr[rOrrfrr=r*rcrrrVrrrr r$r)rrerrrr;r<rGZembrr$Zencoder_outputsrZ pooled_outputr"r"r#r&sB     zMLCDVisionTransformer.forward)NNNN)r'r(r)rrrrr*rirrrr r&r,r"r"r r#rs  rc@s.eZdZUeed<dZdZdZdZddZ dS)MLCDPreTrainedModelrZmlcdTcCs|jj}t|trX|jj}tjj|jd|jd|dtjj|j j |jj |dnt|t r|jj}|jdd|jj d|}|jd|}tjj|jj |dtjj|jj |dtjj|jj |dtjj|jj |dnt|tr\|jj}|jjdd|jj d|}d|jjd|}tjj|jj |dtjj|jj |dnt|tr|jj}|jj|jjdd|}tjj|jd|dnPt|tjr|jj|j jdn&t|tjr|jdur|jjdS)zInitialize the weightsrjr)meanstd)rr3r2N)rZinitializer_factor isinstancerJrinitZnormal_rRrNrSr]Zinitializer_rangerrrrrrrrrrrrrrrLdataZzero_Zfill_r)rrkfactorZ in_proj_stdZ out_proj_stdZfc_stdZ pos_emb_stdr"r"r# _init_weightss8       z!MLCDPreTrainedModel._init_weightsN) r'r(r)rrHZbase_model_prefixZsupports_gradient_checkpointingZ_supports_flash_attnZ_supports_sdparr"r"r"r#rs rzN The vision model from M_L_C_D without any head or projection on top. )Z custom_introc s|eZdZUeed<dZdgZedfdd Zej ddd Z e de e je ee ee eeeefd d d ZZS)MLCDVisionModelrrerrKcs"t|t||_|dSr)rrr vision_modelZ post_initrr r"r#r.s  zMLCDVisionModel.__init__)r%cCs |jjjSr)rrVrS)rr"r"r#get_input_embeddings4sz$MLCDVisionModel.get_input_embeddingsNrcCsN|dur |n|jj}|dur |n|jj}|dur4|n|jj}|j||||dS)a Example: ```python >>> import requests >>> from PIL import Image >>> from transformers import AutoProcessor, MLCDVisionModel >>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs, output_attentions=True) >>> features = outputs.last_hidden_state >>> print(f"Extracted features shape: {features.shape}") >>> print(f"Number of attention layers: {len(outputs.attentions)}") >>> print(f"Attention shape: {outputs.attentions[0].shape}") ```N)rerrr)rrrrr)rrerrrr"r"r#r&7szMLCDVisionModel.forward)NNNN)r'r(r)rrHZmain_input_nameZ_no_split_modulesrrModulerrrr*rirrrr r&r,r"r"r r#r$s"  r)rj).typingrrrr*Ztorch.nnrZ activationsrZmodeling_flash_attention_utilsrZmodeling_layersrZmodeling_outputsr r Zmodeling_utilsr r Zprocessing_utilsr utilsrrrZconfiguration_mlcdrrrr-rJr+r9r~rrIrvrrrrrrrr__all__r"r"r"r#sP      )S   M6V@':