a
    ½ÀhIä  ã                   @   s¤  d Z ddlZddlZddlmZmZ ddlZddlZddlmZ ddl	m
Z
mZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z, e( -e.¡Z/dd„ Z0G dd„ dej1ƒZ2G dd„ dej1ƒZ3G dd„ dej1ƒZ4G dd„ dej1ƒZ5G dd„ dej1ƒZ6G dd„ dej1ƒZ7G d d!„ d!ej1ƒZ8G d"d#„ d#eƒZ9G d$d%„ d%ej1ƒZ:G d&d'„ d'ej1ƒZ;G d(d)„ d)ej1ƒZ<G d*d+„ d+ej1ƒZ=e'G d,d-„ d-e!ƒƒZ>e'd.d/G d0d1„ d1e>ƒƒZ?e'G d2d3„ d3e>ƒƒZ@e'd4d/G d5d6„ d6e>eƒƒZAe'd7d/G d8d9„ d9e>ƒƒZBe'G d:d;„ d;e>ƒƒZCe'G d<d=„ d=e>ƒƒZDe'G d>d?„ d?e>ƒƒZEg d@¢ZFdS )AzPyTorch RemBERT model.é    N)ÚOptionalÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forwardÚ find_pruneable_heads_and_indicesÚprune_linear_layer)Úauto_docstringÚlogging)Údeprecate_kwargé   )ÚRemBertConfigc                    sþ  zddl }ddl}ddl}W n ty:   t d¡ ‚ Y n0 tj |¡}t 	d|› ¡ |j
 |¡}g }g }	|D ]X\‰ }
t‡ fdd„dD ƒƒrqpt 	dˆ › d	|
› ¡ |j
 |ˆ ¡}| ˆ ¡ |	 |¡ qpt||	ƒD ]"\‰ }ˆ  d
d¡‰ ˆ  d¡‰ tdd„ ˆ D ƒƒr t 	dd ˆ ¡› ¡ qÔ| }ˆ D ]}| d|¡rJ| d|¡}n|g}|d dksl|d dkrxt|dƒ}n |d dks”|d dkr t|dƒ}nx|d dkrºt|dƒ}n^|d dkrÔt|dƒ}nDzt||d ƒ}W n0 ty   t 	d d ˆ ¡¡¡ Y q(Y n0 t|ƒdkr(t|d ƒ}|| }q(|dd… dkr\t|dƒ}n|dkrp| |¡}z,|j|jkrštd|j› d |j› d!ƒ‚W n> tyÚ } z$| j|j|jf7  _‚ W Y d}~n
d}~0 0 t 	d"ˆ › ¡ t |¡|_ qÔ| S )#z'Load tf checkpoints in a pytorch model.r   Nz™Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from c                 3   s   | ]}|ˆ v V  qd S ©N© )Ú.0Zdeny©Únamer!   úh/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/models/rembert/modeling_rembert.pyÚ	<genexpr>H   ó    z-load_tf_weights_in_rembert.<locals>.<genexpr>)Úadam_vÚadam_mZoutput_embeddingÚclszLoading TF weight z with shape zbert/zrembert/ú/c                 s   s   | ]}|d v V  qdS ))r(   r)   ZAdamWeightDecayOptimizerZAdamWeightDecayOptimizer_1Zglobal_stepNr!   )r"   Únr!   r!   r%   r&   Y   s   ÿz	Skipping z[A-Za-z]+_\d+z_(\d+)ZkernelÚgammaÚweightZoutput_biasÚbetaÚbiasZoutput_weightsZsquadÚ
classifierzSkipping {}é   r   iõÿÿÿZ_embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )!ÚreÚnumpyZ
tensorflowÚImportErrorÚloggerÚerrorÚosÚpathÚabspathÚinfoÚtrainZlist_variablesÚanyZload_variableÚappendÚzipÚreplaceÚsplitÚjoinÚ	fullmatchÚgetattrÚAttributeErrorÚformatÚlenÚintÚ	transposeÚshapeÚ
ValueErrorÚAssertionErrorÚargsÚtorchZ
from_numpyÚdata)ÚmodelÚconfigZtf_checkpoint_pathr3   ÚnpÚtfZtf_pathZ	init_varsÚnamesZarraysrJ   ÚarrayZpointerZm_nameZscope_namesÚnumÚer!   r#   r%   Úload_tf_weights_in_rembert2   s~    ÿ

þ



rX   c                       sT   e Zd ZdZ‡ fdd„Zd	eej eej eej eej e	ej
dœdd„Z‡  ZS )
ÚRemBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    sŠ   t ƒ  ¡  tj|j|j|jd| _t |j|j¡| _	t |j
|j¡| _tj|j|jd| _t |j¡| _| jdt |j¡ d¡dd d S )N)Úpadding_idx©ÚepsÚposition_ids)r   éÿÿÿÿF)Ú
persistent)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚinput_embedding_sizeÚpad_token_idÚword_embeddingsZmax_position_embeddingsÚposition_embeddingsZtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutZregister_bufferrN   ZarangeÚexpand©ÚselfrQ   ©Ú	__class__r!   r%   ra   ˆ   s    
ÿÿzRemBertEmbeddings.__init__Nr   )Ú	input_idsÚtoken_type_idsr]   Úinputs_embedsÚpast_key_values_lengthÚreturnc                 C   s¸   |d ur|  ¡ }n|  ¡ d d… }|d }|d u rL| jd d …||| …f }|d u rjtj|tj| jjd}|d u r||  |¡}|  |¡}|| }	|  |¡}
|	|
7 }	|  	|	¡}	|  
|	¡}	|	S )Nr^   r   ©ÚdtypeÚdevice)Úsizer]   rN   ÚzerosÚlongrz   rf   rh   rg   ri   rm   )rp   rs   rt   r]   ru   rv   Úinput_shapeÚ
seq_lengthrh   Ú
embeddingsrg   r!   r!   r%   Úforwardš   s"    





zRemBertEmbeddings.forward)NNNNr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ra   r   rN   Ú
LongTensorÚFloatTensorrH   ÚTensorr   Ú__classcell__r!   r!   rq   r%   rY   …   s        úùrY   c                       s0   e Zd Z‡ fdd„Zejejdœdd„Z‡  ZS )ÚRemBertPoolerc                    s*   t ƒ  ¡  t |j|j¡| _t ¡ | _d S r    )r`   ra   r   ÚLinearÚhidden_sizeÚdenseZTanhÚ
activationro   rq   r!   r%   ra   ½   s    
zRemBertPooler.__init__©Úhidden_statesrw   c                 C   s(   |d d …df }|   |¡}|  |¡}|S )Nr   )r   rŽ   )rp   r   Zfirst_token_tensorÚpooled_outputr!   r!   r%   r   Â   s    

zRemBertPooler.forward©r‚   rƒ   r„   ra   rN   rˆ   r   r‰   r!   r!   rq   r%   rŠ   ¼   s   rŠ   c                       sh   e Zd Zd‡ fdd„	Zedddddejeej eej eej ee	 e
eej ed	œd
d„ƒZ‡  ZS )ÚRemBertSelfAttentionNc                    s¼   t ƒ  ¡  |j|j dkr>t|dƒs>td|j› d|j› dƒ‚|j| _t|j|j ƒ| _| j| j | _t	 
|j| j¡| _t	 
|j| j¡| _t	 
|j| j¡| _t	 |j¡| _|j| _|| _d S )Nr   Zembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú))r`   ra   rŒ   Únum_attention_headsÚhasattrrK   rH   Úattention_head_sizeÚall_head_sizer   r‹   ÚqueryÚkeyÚvaluerk   Zattention_probs_dropout_probrm   Ú
is_decoderÚ	layer_idx©rp   rQ   r   rq   r!   r%   ra   Ì   s     

ÿÿzRemBertSelfAttention.__init__Zpast_key_valueÚpast_key_valuesz4.58)Únew_nameÚversionF©r   Úattention_maskÚ	head_maskÚencoder_hidden_statesrŸ   Úoutput_attentionsÚcache_positionrw   c                 C   sÚ  |j \}}	}
|  |¡ |d| j| j¡ dd¡}|d u}|d urnt|tƒrj|j 	| j
¡}|rb|j}qn|j}n|}|rv|n|}|r¨|d ur¨|r¨|j| j
 j}|j| j
 j}n†|  |¡ |d| j| j¡ dd¡}|  |¡ |d| j| j¡ dd¡}|d ur.|sþ|nd }| ||| j
d|i¡\}}|r.d|j| j
< t || dd¡¡}|t | j¡ }|d urd|| }tjj|dd}|  |¡}|d ur|| }t ||¡}| dddd	¡ ¡ }| ¡ d d… | jf }|j|Ž }||fS )
Nr^   r   r2   r§   Téþÿÿÿ©Údimr   r   )rJ   r™   Úviewr•   r—   rI   Ú
isinstancer   Ú
is_updatedÚgetr   Zcross_attention_cacheZself_attention_cacheZlayersÚkeysÚvaluesrš   r›   ÚupdaterN   ÚmatmulÚmathÚsqrtr   Z
functionalZsoftmaxrm   ZpermuteÚ
contiguousr{   r˜   )rp   r   r£   r¤   r¥   rŸ   r¦   r§   Ú
batch_sizer   Ú_Zquery_layerZis_cross_attentionr­   Zcurr_past_key_valueZcurrent_statesZ	key_layerZvalue_layerZattention_scoresZattention_probsZcontext_layerZnew_context_layer_shaper!   r!   r%   r   á   sf    
ÿþÿ

ÿþÿ
ÿþÿ
ÿ



zRemBertSelfAttention.forward)N)NNNNFN)r‚   rƒ   r„   ra   r   rN   rˆ   r   r‡   r
   ÚboolÚtupler   r‰   r!   r!   rq   r%   r“   Ë   s$         ø÷r“   c                       s4   e Zd Z‡ fdd„Zejejejdœdd„Z‡  ZS )ÚRemBertSelfOutputc                    sB   t ƒ  ¡  t |j|j¡| _tj|j|jd| _t |j	¡| _
d S ©Nr[   )r`   ra   r   r‹   rŒ   r   ri   rj   rk   rl   rm   ro   rq   r!   r%   ra   8  s    
zRemBertSelfOutput.__init__©r   Úinput_tensorrw   c                 C   s&   |   |¡}|  |¡}|  || ¡}|S r    ©r   rm   ri   ©rp   r   r½   r!   r!   r%   r   >  s    

zRemBertSelfOutput.forwardr’   r!   r!   rq   r%   rº   7  s   rº   c                
       sl   e Zd Zd
‡ fdd„	Zdd„ Zdejeej eej eej ee	 ee
 eej eej dœdd	„Z‡  ZS )ÚRemBertAttentionNc                    s.   t ƒ  ¡  t||d| _t|ƒ| _tƒ | _d S )N©r   )r`   ra   r“   rp   rº   ÚoutputÚsetÚpruned_headsrž   rq   r!   r%   ra   F  s    

zRemBertAttention.__init__c                 C   s²   t |ƒdkrd S t|| jj| jj| jƒ\}}t| jj|ƒ| j_t| jj|ƒ| j_t| jj	|ƒ| j_	t| j
j|dd| j
_| jjt |ƒ | j_| jj| jj | j_| j |¡| _d S )Nr   r   r©   )rG   r   rp   r•   r—   rÄ   r   r™   rš   r›   rÂ   r   r˜   Úunion)rp   ÚheadsÚindexr!   r!   r%   Úprune_headsM  s    ÿzRemBertAttention.prune_headsFr¢   c              	   C   s>   | j |||||||d}|  |d |¡}	|	f|dd …  }
|
S )N©r£   r¤   r¥   rŸ   r¦   r§   r   r   )rp   rÂ   )rp   r   r£   r¤   r¥   rŸ   r¦   r§   Zself_outputsÚattention_outputÚoutputsr!   r!   r%   r   `  s    
ù	zRemBertAttention.forward)N)NNNNFN)r‚   rƒ   r„   ra   rÈ   rN   rˆ   r   r‡   r
   r¸   r¹   r   r‰   r!   r!   rq   r%   rÀ   E  s$         ø÷rÀ   c                       s0   e Zd Z‡ fdd„Zejejdœdd„Z‡  ZS )ÚRemBertIntermediatec                    sB   t ƒ  ¡  t |j|j¡| _t|jt	ƒr6t
|j | _n|j| _d S r    )r`   ra   r   r‹   rŒ   Úintermediate_sizer   r¬   Ú
hidden_actÚstrr	   Úintermediate_act_fnro   rq   r!   r%   ra   z  s
    
zRemBertIntermediate.__init__r   c                 C   s   |   |¡}|  |¡}|S r    )r   rÐ   ©rp   r   r!   r!   r%   r   ‚  s    

zRemBertIntermediate.forwardr’   r!   r!   rq   r%   rÌ   y  s   rÌ   c                       s4   e Zd Z‡ fdd„Zejejejdœdd„Z‡  ZS )ÚRemBertOutputc                    sB   t ƒ  ¡  t |j|j¡| _tj|j|jd| _t 	|j
¡| _d S r»   )r`   ra   r   r‹   rÍ   rŒ   r   ri   rj   rk   rl   rm   ro   rq   r!   r%   ra   Š  s    
zRemBertOutput.__init__r¼   c                 C   s&   |   |¡}|  |¡}|  || ¡}|S r    r¾   r¿   r!   r!   r%   r     s    

zRemBertOutput.forwardr’   r!   r!   rq   r%   rÒ   ‰  s   rÒ   c                       st   e Zd Zd
‡ fdd„	Zdejeej eej eej eej ee ee	 eej e
ej dœ	dd„Zdd	„ Z‡  ZS )ÚRemBertLayerNc                    st   t ƒ  ¡  |j| _d| _t||ƒ| _|j| _|j| _| jr\| jsNt| › dƒ‚t||d| _	t
|ƒ| _t|ƒ| _d S )Nr   z> should be used as a decoder model if cross attention is addedrÁ   )r`   ra   Úchunk_size_feed_forwardÚseq_len_dimrÀ   Ú	attentionrœ   Úadd_cross_attentionrK   ÚcrossattentionrÌ   ÚintermediaterÒ   rÂ   rž   rq   r!   r%   ra   ˜  s    

zRemBertLayer.__init__F)	r   r£   r¤   r¥   Úencoder_attention_maskrŸ   r¦   r§   rw   c	              	   C   s¤   | j ||||||d}	|	d }
|	dd … }| jr‚|d ur‚t| dƒsRtd| › dƒ‚| j|
||||||d}|d }
||dd …  }t| j| j| j|
ƒ}|f| }|S )N)r£   r¤   r¦   rŸ   r§   r   r   rØ   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`rÉ   )	rÖ   rœ   r–   rK   rØ   r   Úfeed_forward_chunkrÔ   rÕ   )rp   r   r£   r¤   r¥   rÚ   rŸ   r¦   r§   Zself_attention_outputsrÊ   rË   Zcross_attention_outputsÚlayer_outputr!   r!   r%   r   §  s>    ú

ÿù	ÿ
zRemBertLayer.forwardc                 C   s   |   |¡}|  ||¡}|S r    )rÙ   rÂ   )rp   rÊ   Zintermediate_outputrÜ   r!   r!   r%   rÛ   Ø  s    
zRemBertLayer.feed_forward_chunk)N)NNNNNFN)r‚   rƒ   r„   ra   rN   rˆ   r   r‡   r
   r¸   r¹   r   rÛ   r‰   r!   r!   rq   r%   rÓ   —  s(          ÷ö1rÓ   c                       s|   e Zd Z‡ fdd„Zd	ejeej eej eej eej eeeej   ee	 e	e	e	eej e
eef dœdd„Z‡  ZS )
ÚRemBertEncoderc                    sL   t ƒ  ¡  ˆ | _t ˆ jˆ j¡| _t ‡ fdd„t	ˆ j
ƒD ƒ¡| _d| _d S )Nc                    s   g | ]}t ˆ |d ‘qS )rÁ   )rÓ   )r"   Úi©rQ   r!   r%   Ú
<listcomp>ä  r'   z+RemBertEncoder.__init__.<locals>.<listcomp>F)r`   ra   rQ   r   r‹   rd   rŒ   Úembedding_hidden_mapping_inZ
ModuleListÚrangeÚnum_hidden_layersÚlayerÚgradient_checkpointingro   rq   rß   r%   ra   ß  s
    
 zRemBertEncoder.__init__NFT)r   r£   r¤   r¥   rÚ   rŸ   Ú	use_cacher¦   Úoutput_hidden_statesÚreturn_dictr§   rw   c              	   C   sZ  | j r| jr|rt d¡ d}|rD|d u rDtt| jdt| jdƒ}|rft|tƒrft d¡ t 	|¡}|  
|¡}|	rxdnd }|r„dnd }|r˜| jjr˜dnd }t| jƒD ]n\}}|	r¼||f }|d urÌ|| nd }||||||||ƒ}|d }|r¦||d f }| jjr¦||d f }q¦|	r&||f }|
sHtd	d
„ |||||fD ƒƒS t|||||dS )NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Frß   zìPassing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. You should pass an instance of `EncoderDecoderCache` instead, e.g. `past_key_values=EncoderDecoderCache.from_legacy_cache(past_key_values)`.r!   r   r   r2   c                 s   s   | ]}|d ur|V  qd S r    r!   )r"   Úvr!   r!   r%   r&   %  s   øz)RemBertEncoder.forward.<locals>.<genexpr>)Úlast_hidden_staterŸ   r   Ú
attentionsÚcross_attentions)rå   Ztrainingr6   Zwarning_oncer   r   rQ   r¬   r¹   Zfrom_legacy_cacherá   r×   Ú	enumeraterä   r   )rp   r   r£   r¤   r¥   rÚ   rŸ   ræ   r¦   rç   rè   r§   Zall_hidden_statesZall_self_attentionsZall_cross_attentionsrÞ   Zlayer_moduleZlayer_head_maskZlayer_outputsr!   r!   r%   r   ç  sj    ÿÿ


ù

ûþûzRemBertEncoder.forward)
NNNNNNFFTN)r‚   rƒ   r„   ra   rN   rˆ   r   r‡   r¹   r¸   r   r   r   r‰   r!   r!   rq   r%   rÝ   Þ  s2             ô
órÝ   c                       s0   e Zd Z‡ fdd„Zejejdœdd„Z‡  ZS )ÚRemBertPredictionHeadTransformc                    sV   t ƒ  ¡  t |j|j¡| _t|jtƒr6t	|j | _
n|j| _
tj|j|jd| _d S r»   )r`   ra   r   r‹   rŒ   r   r¬   rÎ   rÏ   r	   Útransform_act_fnri   rj   ro   rq   r!   r%   ra   ;  s    
z'RemBertPredictionHeadTransform.__init__r   c                 C   s"   |   |¡}|  |¡}|  |¡}|S r    )r   rï   ri   rÑ   r!   r!   r%   r   D  s    


z&RemBertPredictionHeadTransform.forwardr’   r!   r!   rq   r%   rî   :  s   	rî   c                       s0   e Zd Z‡ fdd„Zejejdœdd„Z‡  ZS )ÚRemBertLMPredictionHeadc                    sR   t ƒ  ¡  t |j|j¡| _t |j|j¡| _t	|j
 | _tj|j|jd| _d S r»   )r`   ra   r   r‹   rŒ   Zoutput_embedding_sizer   rc   Údecoderr	   rÎ   rŽ   ri   rj   ro   rq   r!   r%   ra   L  s
    
z RemBertLMPredictionHead.__init__r   c                 C   s,   |   |¡}|  |¡}|  |¡}|  |¡}|S r    )r   rŽ   ri   rñ   rÑ   r!   r!   r%   r   S  s
    



zRemBertLMPredictionHead.forwardr’   r!   r!   rq   r%   rð   K  s   rð   c                       s0   e Zd Z‡ fdd„Zejejdœdd„Z‡  ZS )ÚRemBertOnlyMLMHeadc                    s   t ƒ  ¡  t|ƒ| _d S r    )r`   ra   rð   Úpredictionsro   rq   r!   r%   ra   ]  s    
zRemBertOnlyMLMHead.__init__)Úsequence_outputrw   c                 C   s   |   |¡}|S r    )ró   )rp   rô   Úprediction_scoresr!   r!   r%   r   a  s    
zRemBertOnlyMLMHead.forwardr’   r!   r!   rq   r%   rò   \  s   rò   c                   @   s*   e Zd ZU eed< eZdZdZdd„ Z	dS )ÚRemBertPreTrainedModelrQ   ÚrembertTc                 C   s¤   t |tjƒr:|jjjd| jjd |jdur |jj 	¡  nft |tj
ƒrz|jjjd| jjd |jdur |jj|j  	¡  n&t |tjƒr |jj 	¡  |jj d¡ dS )zInitialize the weightsg        )ÚmeanZstdNg      ð?)r¬   r   r‹   r.   rO   Znormal_rQ   Zinitializer_ranger0   Zzero_rb   rZ   ri   Zfill_)rp   Úmoduler!   r!   r%   Ú_init_weightsm  s    

z$RemBertPreTrainedModel._init_weightsN)
r‚   rƒ   r„   r   Ú__annotations__rX   Zload_tf_weightsZbase_model_prefixZsupports_gradient_checkpointingrú   r!   r!   r!   r%   rö   f  s
   
rö   a
  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    )Zcustom_introc                       sÂ   e Zd Zd‡ fdd„	Zdd„ Zdd„ Zdd	„ Zedee	j
 ee	j
 ee	j
 ee	j
 ee	j ee	j ee	j ee	j eeee	j   ee ee ee ee ee	j eeef dœdd„ƒZ‡  ZS )ÚRemBertModelTc                    sD   t ƒ  |¡ || _t|ƒ| _t|ƒ| _|r2t|ƒnd| _|  	¡  dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)
r`   ra   rQ   rY   r€   rÝ   ÚencoderrŠ   ÚpoolerÚ	post_init)rp   rQ   Úadd_pooling_layerrq   r!   r%   ra   ‹  s    

zRemBertModel.__init__c                 C   s   | j jS r    ©r€   rf   ©rp   r!   r!   r%   Úget_input_embeddings›  s    z!RemBertModel.get_input_embeddingsc                 C   s   || j _d S r    r  )rp   r›   r!   r!   r%   Úset_input_embeddingsž  s    z!RemBertModel.set_input_embeddingsc                 C   s*   |  ¡ D ]\}}| jj| j |¡ qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)Úitemsrý   rä   rÖ   rÈ   )rp   Zheads_to_prunerä   rÆ   r!   r!   r%   Ú_prune_heads¡  s    zRemBertModel._prune_headsN)rs   r£   rt   r]   r¤   ru   r¥   rÚ   rŸ   ræ   r¦   rç   rè   r§   rw   c                 C   s<  |d ur|n| j j}|d ur |n| j j}|d ur4|n| j j}| j jrZ|
d urP|
n| j j}
nd}
|d urx|d urxtdƒ‚n@|d ur–|  ||¡ | ¡ }n"|d ur°| ¡ d d… }ntdƒ‚|\}}|d urÎ|j	n|j	}d}|	d urt
|	tƒsþ|	d d jd n|	 ¡ }|d u r&tj||| f|d}|d u rBtj|tj|d}|  ||¡}| j jrœ|d urœ| ¡ \}}}||f}|d u rtj||d}|  |¡}nd }|  || j j¡}| j|||||d	}| j||||||	|
||||d
}|d }| jd ur|  |¡nd }|s ||f|dd …  S t|||j|j|j|jdS )NFzDYou cannot specify both input_ids and inputs_embeds at the same timer^   z5You have to specify either input_ids or inputs_embedsr   r¨   )rz   rx   )rs   r]   rt   ru   rv   )
r£   r¤   r¥   rÚ   rŸ   ræ   r¦   rç   rè   r§   r   )rê   Zpooler_outputrŸ   r   rë   rì   )rQ   r¦   rç   Úuse_return_dictrœ   ræ   rK   Z%warn_if_padding_and_no_attention_maskr{   rz   r¬   r
   rJ   Zget_seq_lengthrN   Zonesr|   r}   Zget_extended_attention_maskZinvert_attention_maskZget_head_maskrã   r€   rý   rþ   r   rŸ   r   rë   rì   )rp   rs   r£   rt   r]   r¤   ru   r¥   rÚ   rŸ   ræ   r¦   rç   rè   r§   r~   r¶   r   rz   rv   Zextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthr·   Zencoder_hidden_shapeZencoder_extended_attention_maskZembedding_outputZencoder_outputsrô   r‘   r!   r!   r%   r   ©  sˆ    ÿ


ÿý


ûõúzRemBertModel.forward)T)NNNNNNNNNNNNNN)r‚   rƒ   r„   ra   r  r  r  r   r   rN   r†   r‡   r¹   r¸   rˆ   r   r   r   r‰   r!   r!   rq   r%   rü   ~  sH                 ñ
ðrü   c                       sÄ   e Zd ZdgZ‡ fdd„Zdd„ Zdd„ Zedee	j
 ee	j
 ee	j
 ee	j
 ee	j ee	j ee	j ee	j ee	j
 ee ee ee eeef d	œd
d„ƒZddd„Zeedœdd„ƒZ‡  ZS )ÚRemBertForMaskedLMúcls.predictions.decoder.weightc                    s@   t ƒ  |¡ |jrt d¡ t|dd| _t|ƒ| _|  	¡  d S )NznIf you want to use `RemBertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.F©r   ©
r`   ra   rœ   r6   Úwarningrü   r÷   rò   r*   rÿ   ro   rq   r!   r%   ra     s    ÿ
zRemBertForMaskedLM.__init__c                 C   s
   | j jjS r    ©r*   ró   rñ   r  r!   r!   r%   Úget_output_embeddings-  s    z(RemBertForMaskedLM.get_output_embeddingsc                 C   s   || j j_d S r    r  ©rp   Znew_embeddingsr!   r!   r%   Úset_output_embeddings0  s    z(RemBertForMaskedLM.set_output_embeddingsN)rs   r£   rt   r]   r¤   ru   r¥   rÚ   Úlabelsr¦   rç   rè   rw   c                 C   s´   |dur|n| j j}| j|||||||||
||d}|d }|  |¡}d}|	durttƒ }|| d| j j¡|	 d¡ƒ}|s |f|dd…  }|durœ|f| S |S t|||j|j	dS )a£  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)
r£   rt   r]   r¤   ru   r¥   rÚ   r¦   rç   rè   r   r^   r2   ©ÚlossÚlogitsr   rë   )
rQ   r  r÷   r*   r   r«   rc   r   r   rë   )rp   rs   r£   rt   r]   r¤   ru   r¥   rÚ   r  r¦   rç   rè   rË   rô   rõ   Zmasked_lm_lossÚloss_fctrÂ   r!   r!   r%   r   3  s:    õ
üzRemBertForMaskedLM.forwardc                 K   s~   |j }|d }| jjd us"J dƒ‚tj|| |j d df¡gdd}tj|df| jjtj|jd}tj||gdd}||dœS )Nr   z.The PAD token should be defined for generationr   r^   r©   rx   )rs   r£   )	rJ   rQ   re   rN   ÚcatZ	new_zerosÚfullr}   rz   )rp   rs   r£   Zmodel_kwargsr~   Zeffective_batch_sizeZdummy_tokenr!   r!   r%   Úprepare_inputs_for_generationl  s    "ÿz0RemBertForMaskedLM.prepare_inputs_for_generation)rw   c                 C   s   dS )z¬
        Legacy correction: RemBertForMaskedLM can't call `generate()` from `GenerationMixin`, even though it has a
        `prepare_inputs_for_generation` method.
        Fr!   )r*   r!   r!   r%   Úcan_generatez  s    zRemBertForMaskedLM.can_generate)NNNNNNNNNNNN)N)r‚   rƒ   r„   Ú_tied_weights_keysra   r  r  r   r   rN   r†   r‡   r¸   r   r¹   r   r   r  Úclassmethodr  r‰   r!   r!   rq   r%   r    sF               ó
ò8
r  zS
    RemBERT Model with a `language modeling` head on top for CLM fine-tuning.
    c                       s¾   e Zd ZdgZ‡ fdd„Zdd„ Zdd„ Zedee	j
 ee	j
 ee	j
 ee	j
 ee	j ee	j ee	j ee	j eeee	j   ee	j
 ee ee ee ee eeef d	œd
d„ƒZ‡  ZS )ÚRemBertForCausalLMr	  c                    s@   t ƒ  |¡ |jst d¡ t|dd| _t|ƒ| _|  	¡  d S )NzOIf you want to use `RemBertForCausalLM` as a standalone, add `is_decoder=True.`Fr
  r  ro   rq   r!   r%   ra   ‹  s    

zRemBertForCausalLM.__init__c                 C   s
   | j jjS r    r  r  r!   r!   r%   r  —  s    z(RemBertForCausalLM.get_output_embeddingsc                 C   s   || j j_d S r    r  r  r!   r!   r%   r  š  s    z(RemBertForCausalLM.set_output_embeddingsN)rs   r£   rt   r]   r¤   ru   r¥   rÚ   rŸ   r  ræ   r¦   rç   rè   rw   c                 K   sº   |dur|n| j j}| j|||||||||	||||d}|d }|  |¡}d}|
durr| j||
fd| j ji|¤Ž}|sž|f|dd…  }|durš|f| S |S t|||j|j|j	|j
dS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, RemBertForCausalLM, RemBertConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("google/rembert")
        >>> config = RemBertConfig.from_pretrained("google/rembert")
        >>> config.is_decoder = True
        >>> model = RemBertForCausalLM.from_pretrained("google/rembert", config=config)

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```N)r£   rt   r]   r¤   ru   r¥   rÚ   rŸ   ræ   r¦   rç   rè   r   rc   r2   )r  r  rŸ   r   rë   rì   )rQ   r  r÷   r*   Zloss_functionrc   r   rŸ   r   rë   rì   )rp   rs   r£   rt   r]   r¤   ru   r¥   rÚ   rŸ   r  ræ   r¦   rç   rè   ÚkwargsrË   rô   rõ   Zlm_lossrÂ   r!   r!   r%   r     sN    )ó
þýüúzRemBertForCausalLM.forward)NNNNNNNNNNNNNN)r‚   rƒ   r„   r  ra   r  r  r   r   rN   r†   r‡   r¹   r¸   r   r   r   r‰   r!   r!   rq   r%   r  ƒ  sH                 ñ
ïr  zŸ
    RemBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                       s‚   e Zd Z‡ fdd„Zedeej eej eej eej eej eej eej ee	 ee	 ee	 e
eef dœdd„ƒZ‡  ZS )Ú RemBertForSequenceClassificationc                    sJ   t ƒ  |¡ |j| _t|ƒ| _t |j¡| _t 	|j
|j¡| _|  ¡  d S r    ©r`   ra   Ú
num_labelsrü   r÷   r   rk   Úclassifier_dropout_probrm   r‹   rŒ   r1   rÿ   ro   rq   r!   r%   ra   ù  s    
z)RemBertForSequenceClassification.__init__N©rs   r£   rt   r]   r¤   ru   r  r¦   rç   rè   rw   c                 C   s|  |
dur|
n| j j}
| j||||||||	|
d	}|d }|  |¡}|  |¡}d}|dur8| j jdu r®| jdkrzd| j _n4| jdkr¦|jtj	ksœ|jtj
kr¦d| j _nd| j _| j jdkrêtƒ }| jdkrÞ|| ¡ | ¡ ƒ}n
|||ƒ}nN| j jdkrtƒ }|| d| j¡| d¡ƒ}n| j jdkr8tƒ }|||ƒ}|
sh|f|dd…  }|durd|f| S |S t|||j|jd	S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N©r£   rt   r]   r¤   ru   r¦   rç   rè   r   Z
regressionZsingle_label_classificationZmulti_label_classificationr^   r2   r  )rQ   r  r÷   rm   r1   Zproblem_typer   ry   rN   r}   rH   r   Úsqueezer   r«   r   r   r   rë   )rp   rs   r£   rt   r]   r¤   ru   r  r¦   rç   rè   rË   r‘   r  r  r  rÂ   r!   r!   r%   r     sV    ÷




"


üz(RemBertForSequenceClassification.forward)
NNNNNNNNNN)r‚   rƒ   r„   ra   r   r   rN   r‡   r†   r¸   r   r¹   r   r   r‰   r!   r!   rq   r%   r  ò  s2   
          õ
ôr  c                       s‚   e Zd Z‡ fdd„Zedeej eej eej eej eej eej eej ee	 ee	 ee	 e
eef dœdd„ƒZ‡  ZS )ÚRemBertForMultipleChoicec                    s@   t ƒ  |¡ t|ƒ| _t |j¡| _t |j	d¡| _
|  ¡  d S )Nr   )r`   ra   rü   r÷   r   rk   r!  rm   r‹   rŒ   r1   rÿ   ro   rq   r!   r%   ra   N  s
    
z!RemBertForMultipleChoice.__init__Nr"  c                 C   st  |
dur|
n| j j}
|dur&|jd n|jd }|durJ| d| d¡¡nd}|durh| d| d¡¡nd}|dur†| d| d¡¡nd}|dur¤| d| d¡¡nd}|durÊ| d| d¡| d¡¡nd}| j||||||||	|
d	}|d }|  |¡}|  |¡}| d|¡}d}|dur0tƒ }|||ƒ}|
s`|f|dd…  }|dur\|f| S |S t	|||j
|jdS )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r^   r¨   r#  r2   r  )rQ   r  rJ   r«   r{   r÷   rm   r1   r   r   r   rë   )rp   rs   r£   rt   r]   r¤   ru   r  r¦   rç   rè   Znum_choicesrË   r‘   r  Zreshaped_logitsr  r  rÂ   r!   r!   r%   r   X  sL    ,ÿý÷



üz RemBertForMultipleChoice.forward)
NNNNNNNNNN)r‚   rƒ   r„   ra   r   r   rN   r‡   r†   r¸   r   r¹   r   r   r‰   r!   r!   rq   r%   r%  L  s2   
          õ
ôr%  c                       s‚   e Zd Z‡ fdd„Zedeej eej eej eej eej eej eej ee	 ee	 ee	 e
eef dœdd„ƒZ‡  ZS )ÚRemBertForTokenClassificationc                    sN   t ƒ  |¡ |j| _t|dd| _t |j¡| _t 	|j
|j¡| _|  ¡  d S ©NFr
  r  ro   rq   r!   r%   ra   ¶  s    z&RemBertForTokenClassification.__init__Nr"  c                 C   s¸   |
dur|
n| j j}
| j||||||||	|
d	}|d }|  |¡}|  |¡}d}|durxtƒ }|| d| j¡| d¡ƒ}|
s¤|f|dd…  }|dur |f| S |S t|||j	|j
dS )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr#  r   r^   r2   r  )rQ   r  r÷   rm   r1   r   r«   r   r   r   rë   )rp   rs   r£   rt   r]   r¤   ru   r  r¦   rç   rè   rË   rô   r  r  r  rÂ   r!   r!   r%   r   Á  s8    ÷

üz%RemBertForTokenClassification.forward)
NNNNNNNNNN)r‚   rƒ   r„   ra   r   r   rN   r‡   r†   r¸   r   r¹   r   r   r‰   r!   r!   rq   r%   r&  ´  s2             õ
ôr&  c                       sŠ   e Zd Z‡ fdd„Zedeej eej eej eej eej eej eej eej ee	 ee	 ee	 e
eef dœdd„ƒZ‡  ZS )ÚRemBertForQuestionAnsweringc                    s@   t ƒ  |¡ |j| _t|dd| _t |j|j¡| _|  	¡  d S r'  )
r`   ra   r   rü   r÷   r   r‹   rŒ   Ú
qa_outputsrÿ   ro   rq   r!   r%   ra   ù  s
    z$RemBertForQuestionAnswering.__init__N)rs   r£   rt   r]   r¤   ru   Ústart_positionsÚend_positionsr¦   rç   rè   rw   c                 C   sD  |d ur|n| j j}| j|||||||	|
|d	}|d }|  |¡}|jddd\}}| d¡}| d¡}d }|d urü|d urüt| ¡ ƒdkr–| d¡}t| ¡ ƒdkr°| d¡}| d¡}| d|¡ | d|¡ t	|d}|||ƒ}|||ƒ}|| d }|s.||f|dd …  }|d ur*|f| S |S t
||||j|jdS )	Nr#  r   r   r^   r©   )Zignore_indexr2   )r  Ústart_logitsÚ
end_logitsr   rë   )rQ   r  r÷   r)  rA   r$  rG   r{   Zclamp_r   r   r   rë   )rp   rs   r£   rt   r]   r¤   ru   r*  r+  r¦   rç   rè   rË   rô   r  r,  r-  Z
total_lossZignored_indexr  Z
start_lossZend_lossrÂ   r!   r!   r%   r     sP    ÷








ûz#RemBertForQuestionAnswering.forward)NNNNNNNNNNN)r‚   rƒ   r„   ra   r   r   rN   r‡   r†   r¸   r   r¹   r   r   r‰   r!   r!   rq   r%   r(  ÷  s6              ô
ór(  )
r  r  r%  r(  r  r&  rÓ   rü   rö   rX   )Gr…   r³   r8   Útypingr   r   rN   Ztorch.utils.checkpointr   Ztorch.nnr   r   r   Zactivationsr	   Zcache_utilsr
   r   r   Z
generationr   Zmodeling_layersr   Zmodeling_outputsr   r   r   r   r   r   r   r   Zmodeling_utilsr   Zpytorch_utilsr   r   r   Úutilsr   r   Zutils.deprecationr   Zconfiguration_rembertr   Z
get_loggerr‚   r6   rX   ÚModulerY   rŠ   r“   rº   rÀ   rÌ   rÒ   rÓ   rÝ   rî   rð   rò   rö   rü   r  r  r  r%  r&  r(  Ú__all__r!   r!   r!   r%   Ú<module>   sn   (

S7l4G\
ÿ hÿjÿTgBN