a
h' @ s d dl Z d dlmZ ddlmZ ddlmZ ddlmZ ddl m
Z
m Z ddl m
Z
mZ dd lmZmZ d
d lmZ e rd dlZd dlmZ d Zd
ZdZdZdZdddZ G dd dZ dS ) N)Image )create_causal_mask)_get_model_class)
AutoConfig) MODEL_FOR_PRETRAINING_MAPPING
MODEL_MAPPING)PROCESSOR_MAPPING_NAMES
AutoProcessor)TOKENIZER_MAPPING_NAMES
AutoTokenizer )is_torch_availablez[92mz[93mz[0mu ■u ⬚![]()
c
s| jdkr(dddddf jdkrJddddddf ttdd D }d}g }tD ]l\ } | kr|s }d f< |dkrt| ks d krt d krȈ d7 d| | f< d}qtddurfd d
tD d fd dtD }
|dur||dk} t| d
d |
| d
dt d|d |} tj | dt
t t dt t t d}
|d |
d |d dt|
d }dur|d7 }|| g }tD ]h\}||f dkr6|dd
tt|ttD n |tt|tt qtttt| }|D ]>}|dur|d d d | d d | nd qptD ]\ t|}v rt | t n|}d fddtD }d}durDd fddtD }|| dt d d | d| qd|S )z
Generates an attention matrix from a given attention mask.
Optionally applies a sliding window mask (e.g., for Gemma2/3) and
marks regions where image tokens occur based on the specified `img_token`.
r N c s s | ]}t t|V qd S N)lenrepr).0word r c/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/utils/attention_visualizer.py 8 z6generate_attention_matrix_from_mask..r r
c s$ g | ] fd dt D qS )c s0 g | ](}d | kr k r(n ndnd qS )r r
r r j)isliding_windowr r
I r zBgenerate_attention_matrix_from_mask...)range)r )nr )r r r I r z7generate_attention_matrix_from_mask.. c 3 sV | ]N} d |f r"t t t n*d |kr:t t t n d |f rJtntV qdS r NYELLOW BLACK_SQUARERESETGREEN WHITE_SQUAREr )maskr r r K s
)
boundariesz: i == j (diagonal) z: token_type_idszAttention MatrixzSliding Window Maskc S s g | ]}t | t qS r )r% r' )r kr r r r i r z | c 3 sj | ]b}| v r6 |f r6v r6t t t n* |krNt t t n |f r^tntV qd S r r$ r )r img_tokenr* r wordsr r r x s
c 3 sv | ]n}| v rBv rBd f d |f krBt t t n* |krZt t t n | rjtntV qdS r# r$ r )r r0 sliding_window_masktoken_type_bucketsr r1 r r r s
z:
)intndimr max enumerater jointorchwhereZcumsumboolarangeZ bucketizer( r& r' r% appendljustliststrrjustmapzipr )r1 r* r0 r token_type_idsimage_seq_lengthZmax_word_lengthZ
first_img_idxoutputr. Z row_dummyZ
is_specialr- Zlegendf_stringZvertical_headeridxrowZ word_reprZ colored_wordZ row_displayZsliding_window_rowr ) r r0 r* r! r r2 r3 r r1 r #generate_attention_matrix_from_mask( s|
,$(
.rK c @ s: e Zd ZedddZd edddZd eddd Zd
S )
AttentionMaskVisualizer)
model_namec C s t |}d| _t| dr0t| dd | _zt|t}W n t yZ t|t
}Y n0 |d u rtt d| d|| _ G dd d|t
j}|||| _| j|j || _|| _d S )Nr r z Model name z- is not supported for attention visualizationc @ s e Zd Zdd ZdS )z7AttentionMaskVisualizer.__init__.._ModelWrapperc S s$ t j| t dd| _|| _d S )Nr
)nnModule__init__ZLinearZ dummy_moduleconfig)selfrQ rM r r r rP s z@AttentionMaskVisualizer.__init__.._ModelWrapper.__init__N)__name__
__module__ __qualname__rP r r r r
_ModelWrapper s rV )r from_pretrained image_tokenhasattrZget_text_configgetattrr r r Exceptionr
ValueError
mapped_clsrN rO modeltodtyperepo_idrQ )rR rM rQ r] rV r r r rP s
z AttentionMaskVisualizer.__init__r/ )input_sentencec C s | j ||d d S )N)suffix)visualize_attention_mask)rR rb rc r r r __call__ s z AttentionMaskVisualizer.__call__c C sZ | j }i }d }| jjtv rd}ttj|ddj}d}t j
| j |d}t |drZ|j
}n|j|jgd }|r~|d|}||||d d
} |j|jgd | _
| d }
d | v r| d |d < |j| d
d } nJ| jjtv r t
| j } | |} | |d dd }
ntd|jj dd|j_| |
j\}
}tj|
||jjf| j j d}t |}t |j||
|d d}|d ur| }
n |
d d!|
d||}
dt"d| jj d| j# d }d}t$d| t$dd| jj d| j d | j#j% &t"| d | t$| t'| |
| j
t(| jd d |d |d }t$| t$| d S ) Nzchttps://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg?download=trueT)streamr, )rF rX r r pt)Zimagestextrc return_tensorsattention_maskrE Z input_ids)ri z Model type z) does not support attention visualizationeager)r` )rQ input_embedsrj cache_positionZpast_key_valuesr
z##z Attention visualization for z | r r4 z" Attention visualization for [1m:z[0m z r )r0 r rE rF ))r^ rQ Z
model_typer r openrequestsgetrawr
rW ra rY rX tokenizerZconvert_ids_to_tokensZimage_token_idreplacer r tokenizer\ Z_attn_implementationtrainshaper: ZzerosZ hidden_sizer` r= r r< Z unsqueezeexpandr r] printrS centerrK rZ )rR rb rc r^ kwargsrF Zimg processorrX inputsrj tokensrs Z
batch_sizeZ
seq_lengthrl rm Z causal_maskZtop_bottom_borderZ side_borderrH r r r rd s
z0AttentionMaskVisualizer.visualize_attention_maskN)r/ )r/ )rS rT rU rA rP re rd r r r r rL s rL )r NNN) rp ZPILr Z
masking_utilsr Zmodels.auto.auto_factoryr Z models.auto.configuration_autor Zmodels.auto.modeling_autor r Zmodels.auto.processing_autor r
Z models.auto.tokenization_autor r Z import_utilsr r: Ztorch.nnrN r( r% r' r& r) rK rL r r r r s&
l