a
    h]                     @   s   d dl Z d dlZd dlZddlmZ e r6d dlmZ dZdZe	ej
dedZeeefvrdedi Zd	d
 Zdd ZdddZdddZdS )    N   )is_torch_npu_available)npu_fusion_attention   ZNPU_FA2_SPARSE_MODE)defaultzEnvironment variable `NPU_FA2_SPARSE_MODE` can only be set as 2 (top-left aligned causal mask) or 3 (down-right aligned causal mask).c                 C   s4   | t vr,tjtjddg| ddd t | < t |  S )z6Get or create attention mask for the specified device.i   device   )Zdiagonal)ATTN_MASK_NPU_CACHEtorchZtriuZonesboolr    r   i/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/integrations/npu_flash_attention.pyget_attn_mask_npu(   s    $r   c                   C   s   t  rttkS dS )NF)r   SPARSE_MODE!TOP_LEFT_ALIGNED_CAUSAL_MASK_MODEr   r   r   r   'is_npu_fa2_top_left_aligned_causal_mask/   s    r           Fc                 K   s   d| }|d u r$dt | jd  }|sN| jd }t| |||d||dd }	n2t| j}
| jd }t| |||d|||
td	d }	|	S )N      ?r   ZBSND)	keep_probscaler   )r   r   
atten_masksparse_mode)mathsqrtshaper   r   r   r   )qkv	dropout_psoftmax_scalecausalkwargsr   head_numoutputattn_mask_npur   r   r   npu_flash_attn_func3   s,    	



r'   c
                 K   s   d| }|d u r$dt | jd  }|	s| jd }t| |||d d ||dt|dd     t|dd     dd }njt| j	}| jd }t| |||d d |||dt|dd     t|dd     t
dd }|S )Nr   r   r	   ZTND)pser   r   r   input_layoutactual_seq_qlenactual_seq_kvlenr   )	r(   Zpadding_maskr   r   r   r)   r*   r+   r   )r   r   r   r   tuplecpunumpytolistr   r   r   )r   r   r   Zcu_seqlens_qZcu_seqlens_kZmax_seqlen_qZmax_seqlen_kr    r!   r"   r#   r   r$   r%   r&   r   r   r   npu_flash_attn_varlen_funcV   sP    


r0   )r   NF)NNr   NF)r   osr   Zutils.import_utilsr   Z	torch_npur   r   Z#DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODEintgetenvr   
ValueErrorr
   r   r   r'   r0   r   r   r   r   <module>   s2      
)     