a
    hO8                     @   s2  d dl mZmZ d dlZd dlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ d
dlmZmZmZmZmZmZ d
dlmZ ddlmZ eeZG dd deZG dd dejZG dd dejZ G dd deZ!G dd deZ"G dd deZ#G dd deZ$G dd deZ%g dZ&dS )     )OptionalUnionN)nn   )ACT2FN)Cache)FlashAttentionKwargs)Unpack)logging   )LlavaCausalLMOutputWithPastLlavaForConditionalGeneration
LlavaModelLlavaModelOutputWithPastLlavaPreTrainedModelTransformersKwargs)MistralRMSNorm   )Mistral3Configc                   @   s   e Zd ZdS )Mistral3RMSNormN__name__
__module____qualname__ r   r   i/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/models/mistral3/modular_mistral3.pyr   )   s   r   c                       s>   e Zd ZdZed fddZejejejdddZ  Z	S )Mistral3PatchMergerz<
    Learned merging of spatial_merge_size ** 2 patches
    configc                    sL   t    || _|jj}|j| _| jjj| _tj|| jd  |dd| _	d S )Nr   FZbias)
super__init__r   vision_confighidden_sizespatial_merge_size
patch_sizer   Linearmerging_layer)selfr   r#   	__class__r   r   r!   2   s    
zMistral3PatchMerger.__init__)image_featuresimage_sizesreturnc                    s    fdd|D }dd |D }|j d }g }t||D ]n\}}|| \}}	|||	|dddd}
tjjj	|
 j
 j
d}|| j
d  d }|| q<tj|dd	} |}|S )
Nc                    s(   g | ] }|d   j  |d  j  fqS )r   r   )r%   ).0Z
image_sizer(   r   r   
<listcomp><   s   z/Mistral3PatchMerger.forward.<locals>.<listcomp>c                 S   s   g | ]\}}|| qS r   r   )r.   hwr   r   r   r0   @       r   r   r   )Zkernel_sizeZstridedim)shape	enumeratesplitviewZpermuteZ	unsqueezetorchr   Z
functionalZunfoldr$   tappendcatr'   )r(   r+   r,   Ztokens_per_imagedZpermuted_tensorZimage_indexZimage_tokensr1   r2   Z
image_gridgridr   r/   r   forward;   s"    



zMistral3PatchMerger.forward)
r   r   r   __doc__r   r!   r;   TensorrA   __classcell__r   r   r)   r   r   -   s   	r   c                       s6   e Zd Zed fddZejejdddZ  ZS )Mistral3MultiModalProjectorr   c                    s   t    t|jj|jjd| _t|| _	t
|jtr:dnt|j}tj|jj| |jj|jd| _t|j | _tj|jj|jj|jd| _d S )N)epsr   r   )r    r!   r   r"   r#   text_configZrms_norm_epsnormr   patch_merger
isinstancevision_feature_layerintlenr   r&   Zmultimodal_projector_biaslinear_1r   Zprojector_hidden_actactlinear_2)r(   r   Znum_feature_layersr)   r   r   r!   T   s    


z$Mistral3MultiModalProjector.__init__)r+   r,   c                 C   s8   |  |}| ||}| |}| |}| |}|S N)rH   rI   rN   rO   rP   )r(   r+   r,   hidden_statesr   r   r   rA   d   s    



z#Mistral3MultiModalProjector.forward)	r   r   r   r   r!   r;   rC   rA   rD   r   r   r)   r   rE   S   s   rE   c                   @   s   e Zd ZdS )Mistral3CausalLMOutputWithPastNr   r   r   r   r   rS   m   s   rS   c                   @   s   e Zd ZdS )Mistral3ModelOutputWithPastNr   r   r   r   r   rT   q   s   rT   c                   @   s   e Zd ZdS )Mistral3PreTrainedModelNr   r   r   r   r   rU   u   s   rU   c                   @   s   e Zd Zdejejeeee	e f  dddZ
d	ejejeej eej ee eej eeee	e f  ee ee ee ee eej ejee eeef dddZdS )
Mistral3ModelNpixel_valuesr,   rK   c           	         s   |dur|n| j j}dd | D }| j|f|dd|t|trTj| }n fdd|D }tj|dd	}| 	|
d
|}| jj| j j   fdd|D }t|
d
|}|S )aU  
        Obtains image last hidden states from the vision tower and apply multimodal projection.

        Args:
            pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`):
               The tensors corresponding to the input images.
            vision_feature_layer (`Union[int, list[int]]`, *optional*):
                The index of the layer to select the vision feature. If multiple indices are provided,
                the vision feature of the corresponding indices will be concatenated to form the
                vision features.
            image_sizes (`torch.Tensor`, *optional*):
                Tensor containing the image sizes as returned by the processor.
        Returns:
            image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
        Nc                 S   s   i | ]\}}|d ur||qS rQ   r   )r.   kvr   r   r   
<dictcomp>   r3   z4Mistral3Model.get_image_features.<locals>.<dictcomp>T)r,   output_hidden_statesc                    s   g | ]} j | qS r   )rR   )r.   Z	layer_idx)image_outputsr   r   r0      r3   z4Mistral3Model.get_image_features.<locals>.<listcomp>r4   r5   r   c                    s    g | ]\}}|  |   qS r   r   )r.   heightwidth)downsample_ratior   r   r0      r3   )r   rK   itemsZvision_towerrJ   rL   rR   r;   r>   Zmulti_modal_projectorZsqueezer%   r$   r9   )	r(   rX   r,   rK   kwargsZselected_image_featureZhs_poolr+   Zsplit_sizesr   )r`   r]   r   get_image_featuresz   s    
z Mistral3Model.get_image_features)	input_idsrX   attention_maskposition_idspast_key_valuesinputs_embedsrK   	use_cacheoutput_attentionsr\   return_dictcache_positionr,   rb   r-   c                 K   s  |	d ur|	n| j j}	|
d ur |
n| j j}
|d ur4|n| j j}|d urH|n| j j}|d u |d uA rhtd|d u r||  |}|d ur| j|||d}tj	|dd
|j|j}| j|||d}|||}| jf ||||||	|
d|d	|}t|j|j|j|j|d ur|nd dS )	Nz:You must specify exactly one of input_ids or inputs_embeds)rX   rK   r,   r   r5   )rh   r+   T)	re   rf   rg   rh   ri   rj   r\   rk   rl   )last_hidden_staterg   rR   
attentionsimage_hidden_states)r   rj   r\   use_return_dictrK   
ValueErrorZget_input_embeddingsrc   r;   r>   toZdeviceZdtypeZget_placeholder_maskZmasked_scatterZlanguage_modelrT   rm   rg   rR   rn   )r(   rd   rX   re   rf   rg   rh   rK   ri   rj   r\   rk   rl   r,   rb   r+   Zspecial_image_maskoutputsr   r   r   rA      sR    
zMistral3Model.forward)N)NNNNNNNNNNNNN)r   r   r   r;   FloatTensorrC   r   r   rL   listrc   
LongTensorr   boolr	   r   tuplerT   rA   r   r   r   r   rV   y   sH    -             
rV   c                   @   s   e Zd Zd	ejejeeee	e f  dddZ
d
ejejeej eej ee eej eej ee ee ee ee eej eeejf eej ee eeef dddZdS ) Mistral3ForConditionalGenerationNrW   c                 K   s   | j jf |||d|S )NrW   )modelrc   )r(   rX   r,   rK   rb   r   r   r   rc      s    z3Mistral3ForConditionalGeneration.get_image_featuresr   )rd   rX   re   rf   rg   rh   labelsri   rj   r\   rk   rl   logits_to_keepr,   rb   r-   c                 K   s   |	dur|	n| j j}	|
dur |
n| j j}
|dur4|n| j j}| jf ||||||||	|
d||d|}|d }t|trt| dn|}| |dd|ddf }d}|dur| j	f ||| j j
jd|}t|||j|j|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, Mistral3ForConditionalGeneration

        >>> model = Mistral3ForConditionalGeneration.from_pretrained("mistralai/Mistral-Small-3.1-24B-Instruct-2503")
        >>> processor = AutoProcessor.from_pretrained("mistralai/Mistral-Small-3.1-24B-Instruct-2503")

        >>> prompt = "<s>[INST][IMG]What is the image?[/INST]"
        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, text=prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs, max_new_tokens=15)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is the image?The image depicts two cats lying on a pink blanket."
        ```NT)rd   rX   re   rf   rg   rh   ri   rj   r\   rk   rl   r,   r   )logitsr{   
vocab_size)lossr}   rg   rR   rn   ro   )r   rj   r\   rp   rz   rJ   rL   sliceZlm_headZloss_functionrG   r~   rS   rg   rR   rn   ro   )r(   rd   rX   re   rf   rg   rh   r{   ri   rj   r\   rk   rl   r|   r,   rb   rs   rR   Zslice_indicesr}   r   r   r   r   rA      sL    -z(Mistral3ForConditionalGeneration.forward)N)NNNNNNNNNNNNr   N)r   r   r   r;   rt   rC   r   r   rL   ru   rc   rv   r   rw   r	   r   rx   rS   rA   r   r   r   r   ry      sL                  
ry   )rV   rU   ry   )'typingr   r   r;   r   Zactivationsr   Zcache_utilsr   Zmodeling_flash_attention_utilsr   Zprocessing_utilsr	   utilsr
   Zllava.modeling_llavar   r   r   r   r   r   Zmistral.modeling_mistralr   Zconfiguration_mistral3r   Z
get_loggerr   loggerr   Moduler   rE   rS   rT   rU   rV   ry   __all__r   r   r   r   <module>   s(    
&ng