a h5 @sbdZddlZddlZddlmZmZmZddlZddl Z ddl Z ddl m Z ddl m Z ddlmZddlmZmZmZdd lmZdd lmZmZmZdd lmZdd lmZdd lmZm Z m!Z!m"Z"m#Z#ddl$m%Z%m&Z&ddl'm(Z(ddl)m*Z*m+Z+m,Z,m-Z-ddl.m/Z/ddl0m1Z1e+r8ddl2m3Z3ddl4m5Z5e-6e7Z8e j9e:e:dddZ;Gddde j<Z=d6e j>e j9e j9e j9ee j9ee?e?ee j9dddZ@Gdd d e j>ZAGd!d"d"eZBGd#d$d$eZCe*Gd%d&d&e&ZDGd'd(d(eDZEGd)d*d*eDZFe*Gd+d,d,eDZGe*d-d.Gd/d0d0eDeZHGd1d2d2eDZIGd3d4d4eDeZJgd5ZKdS)7z=PyTorch MarianMTModel model, ported from the Marian C++ repo.N)CallableOptionalUnion)nn)CrossEntropyLoss)ACT2FN)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)AttentionMaskConverter_prepare_4d_attention_mask#_prepare_4d_attention_mask_for_sdpa)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringis_torch_flex_attn_availableis_torchdynamo_compilinglogging)deprecate_kwarg) MarianConfig) BlockMask)make_flex_block_causal_mask) input_ids pad_token_iddecoder_start_token_idcCsh||j}|ddddf|ddddf<||dddf<|durTtd||dk||S)z1 Shift input ids one token to the right. Nrrz1self.model.config.pad_token_id has to be defined.i)Z new_zerosshapeclone ValueErrorZ masked_fill_)r#r$r%Zshifted_input_idsr*f/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/models/marian/modeling_marian.pyshift_tokens_rightBs (r,csfeZdZdZd eeeeddfdd ZddZe d ej eeej ej d fd d Z Z S)#MarianSinusoidalPositionalEmbeddingzDThis module produces sinusoidal positional embeddings of any length.N) num_positions embedding_dim padding_idxreturncst||dSN)super__init__)selfr.r/r0 __class__r*r+r4Usz,MarianSinusoidalPositionalEmbedding.__init__c s|jj\}tfddt|D}tj||jjdd}ddkrRdn dd}tt |dddddf|ddd|f<tt |dddddf|dd|df<t j |dd |_dS) z Identical to the XLM create_sinusoidal_embeddings except features are not interleaved. The cos features are in the 2nd half of the vector. [dim // 2:] cs$g|]fddtDqS)c s(g|] }tdd|dqS)i')nppower).0j)dimposr*r+ _zOMarianSinusoidalPositionalEmbedding._init_weight...)range)r;r=)r>r+r?_r@zDMarianSinusoidalPositionalEmbedding._init_weight..F)dtype requires_gradr8rrN)rD) weightr'r9arrayrAtorchemptyrC FloatTensorsincosr Parameter)r5Zn_posZ position_encoutsentinelr*rBr+ _init_weightXs  22z0MarianSinusoidalPositionalEmbedding._init_weightr)input_ids_shapepast_key_values_length position_idsr1cs@|dur4|dd\}}tj|||tj|jjd}t|S)z3`input_ids_shape` is expected to be [bsz x seqlen].Nr8)rCdevice)rGarangelongrErSr3forward)r5rPrQrRbszZseq_lenr6r*r+rVgs z+MarianSinusoidalPositionalEmbedding.forward)N)rN)__name__ __module__ __qualname____doc__intrr4rOrGZno_gradSizeTensorrV __classcell__r*r*r6r+r-Rsr-)modulequerykeyvalueattention_maskscalingdropout head_maskc Ks|dur|dd}t||dd|} |dur>| |} tjj| dd} |durj| |dddd} tjj| ||j d} t| |} | dd } | | fS)Nr&r8rrBrptraining) sizerGmatmul transposer functionalZsoftmaxviewrgrl contiguous) rarbrcrdrerfrgrhkwargs attn_weights attn_outputr*r*r+eager_attention_forwardus  rvcseZdZdZdeeeeeeeeeedfdd Z e d d d d de j ee j ee ee j ee j eee j eeee j ee j eee j fd ddZZS)MarianAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr`FTN) embed_dim num_headsrg is_decoderbias is_causalconfig layer_idxc st||_||_||_|||_||_|j||jkrTtd|jd|d|jd|_||_ ||_ ||_ |dur|j rt d|jjdtj|||d|_tj|||d|_tj|||d|_tj|||d|_dS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).rizInstantiating a decoder z without passing `layer_idx` is not recommended and will lead to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.r{)r3r4rxryrghead_dimr}r)rfrzr|r~logger warning_oncer7rXrLineark_projv_projq_projout_proj) r5rxryrgrzr{r|r}r~r6r*r+r4s0    zMarianAttention.__init__past_key_valuepast_key_values4.58new_nameversion) hidden_stateskey_value_statesrrelayer_head_maskoutput_attentionscache_positionrsr1cKs|du} |jdd\} } | r(|jdn| } | | d|jf} | | d|jf}||j| dd}|durt|tr|j|j }| r|j }q|j }n|}| r|n|}| r|dur|r|j |j j }|j |j j}n|||}||}|j|dd}|j|dd}|durN| s|nd}||||j d|i\}}| rNd|j|j <t}|jjdkrlt|jj}||||||f|jsdn|j|j||d |\}}|| | d}||}||fS) z#Input shape: Batch x Time x ChannelNr&rr8rTeagerr`)rgrfrrh)r'rrrqro isinstancer is_updatedgetr~Zcross_attention_cacheself_attention_cachelayerskeysvaluesrrupdatervr}_attn_implementationrrlrgrfreshaperrr)r5rrrrerrrrsZis_cross_attentionrWtgt_lenZsrc_lenZ q_input_shapeZkv_input_shapeZ query_statesrZcurr_past_key_valueZcurrent_statesZ key_statesZ value_statesZattention_interfacerurtr*r*r+rVs`          zMarianAttention.forward)r`FTFNN)NNNNFN)rXrYrZr[r\floatboolrr r4rrGr^r rrtuplerVr_r*r*r6r+rwsF' rwc s\eZdZd eeedfdd Zd ejejejee e ejeejfdddZ Z S) MarianEncoderLayerNr}r~cst|j|_t|j|j|j||d|_t |j|_ |j |_ t |j |_|j|_t|j|j|_t|j|j|_t |j|_dS)N)rxryrgr}r~)r3r4d_modelrxrwZencoder_attention_headsattention_dropout self_attnr LayerNormself_attn_layer_normrgractivation_function activation_fnactivation_dropoutrZencoder_ffn_dimfc1fc2final_layer_normr5r}r~r6r*r+r4s   zMarianEncoderLayer.__init__F)rrerrr1c Cs|}|j||||d\}}tjj||j|jd}||}||}|}|||}tjj||j|jd}| |}tjj||j|jd}||}| |}|j t j krt |st |rt |j jd}t j|| |d}|f}|r||f7}|S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)`. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. )rrerrrji)minmax)rrrprgrlrrrrrrrCrGZfloat16isinfanyisnanfinforclamp) r5rrerrresidualrtZ clamp_valueoutputsr*r*r+rV(s8        zMarianEncoderLayer.forward)N)F) rXrYrZr rr\r4rGrIrrrVr_r*r*r6r+rsrcseZdZdeeedfdd Zedddddej eej eej eej eej eej ee ee ee eej e ej ee ej ej ffd d d ZZS)MarianDecoderLayerNrc st|j|_t|j|j|jdd||d|_|j|_t |j |_ |j |_ t |j|_t|j|j|jd||d|_t |j|_t |j|j|_t |j|j|_t |j|_dS)NT)rxryrgrzr|r}r~)rgrzr}r~)r3r4rrxrwZdecoder_attention_headsrrrgrrrrrrr encoder_attnencoder_attn_layer_normrZdecoder_ffn_dimrrrrr6r*r+r4]s6   zMarianDecoderLayer.__init__rrrrFT) rreencoder_hidden_statesencoder_attention_maskrcross_attn_layer_head_maskrr use_cacherr1c  Cs|} |j|||||| d\}} tjj||j|jd}| |}||}d} |dur|} |j||||||| d\}} tjj||j|jd}| |}||}|} || |}tjj||j |jd}| |}tjj||j|jd}| |}| |}|f}|r|| | f7}|S)a9 Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape `(batch, seq_len, embed_dim)` encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)`. cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of size `(decoder_attention_heads,)`. past_key_values (`Tuple(torch.FloatTensor)`): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. It is used to update the cache in the correct position and to infer the complete sequence length. )rrrerrrrjN)rrrerrrr) rrrprgrlrrrrrrrr)r5rrerrrrrrrrrZself_attn_weightsZcross_attn_weightsrr*r*r+rV|sL#       zMarianDecoderLayer.forward)N) NNNNNNFTN)rXrYrZr rr\r4rrGr^r rrrIrVr_r*r*r6r+r\s0 rc@seZdZUeed<dZdZdZdZdZ dZ e e j e jefdddZeddZe ejd fejd d d Zee ejd fejejedddZeejeeejejedddZe ejd fe ejd fejejdddZd S)MarianPreTrainedModelr}modelT)racCs|jj}t|tjr>|jjjd|d|jdur|jj nvt|t rR| nbt|tj r|jjjd|d|j dur|jj|j  n&t|tjr|jjd|jj dS)Nr`)meanstd?)r}Zinit_stdrrrrEdataZnormal_r{Zzero_r-rO Embeddingr0rZfill_)r5rarr*r*r+ _init_weightss       z#MarianPreTrainedModel._init_weightscCs@|jj}tjgddddd|gg|jd}||||d}|S)N)r r8r r8rS)rer#decoder_input_ids)r}r$rGZtensorrSne)r5Z pad_tokenr# dummy_inputsr*r*r+rs"z"MarianPreTrainedModel.dummy_inputsN)re inputs_embedscCsv|durr|jjdkr&d|vr |nd}nL|jjdkr@t||j}n2|jjdkrft|tjrrt|dd}n t||j}|S)Nflash_attention_2rsdpaflex_attentionF)r| r}rrrCrrGr^r"r)r5rerr*r*r+_update_full_masks     z'MarianPreTrainedModel._update_full_maskr!)re input_tensorrrc Csf|jjdkrRt|tjr"t|}n,|durNttj|jd|jdf|jd}|S|jjdkrz|durv|dk rv|SdS|dur| nd}|dur|j nd}|jjdkr|st j ||||jd rdS|j}|jd}|r|} n"t|tjr|jd n ||d} |j||| |||jdd } |jjdkrb|durb|jjd vrbt|j} t | | } | S) Nrrr)rmrSrr`Fr)rrQZ is_trainingr&)sequence_length target_lengthrCr batch_size)cudaZxpuZnpu)r}rrrGr^r"onesr'rSrget_seq_lengthZis_compileabler Z_ignore_causal_mask_sdparlrCZget_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positiontyperrZ_unmask_unattended) r5rerrrZpast_seen_tokensZusing_compilable_cacherCrr causal_mask min_dtyper*r*r+_update_causal_masksd            z)MarianPreTrainedModel._update_causal_mask)rerrrCrrc KsF|dur|dkr|}n&t|j}tj||f|||jd}|dkrVtj|dd}|tj||jd|ddk9}|ddddddf |ddd}|durB| }|j d} |ddddddd| f|ddddddf |j} | dk} |ddddddd| f | ||ddddddd| f<|S) aM Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. Nr)Z fill_valuerCrSr)Zdiagonalrr&r)r=rGrrfullrSZtriurTrexpandr(r'toZ masked_fill) rerrrCrrrsrrZ mask_lengthZ padding_maskr*r*r+r[s*  $  6  zKMarianPreTrainedModel._prepare_4d_causal_attention_mask_with_cache_position)rr input_shapercCs|dur|dur|jjdkr.d|vr(|nd}nb|jjdkrPt||j|dd}n@|jjdkr|t|tjrt||ddd}nt||j|dd}|S) Nrrrr&)rrF)Z query_lengthr|r)r5rrrrr*r*r+_update_cross_attn_masks(     z-MarianPreTrainedModel._update_cross_attn_mask)rXrYrZr __annotations__base_model_prefixZsupports_gradient_checkpointingZ_supports_flash_attnZ_supports_sdpaZ_supports_flex_attnZ_can_compile_fullgraphrrrrr-rpropertyrrGr^rrr r staticmethodr\rCrr]rr*r*r*r+rs>    L9  rc seZdZdZd eeejdfdd Zd ee j ee j ee j ee j ee ee ee eee j efdddZZS) MarianEncoderz Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`MarianEncoderLayer`]. Args: config: MarianConfig embed_tokens (nn.Embedding): output embedding Nr} embed_tokenscstj|_j|_j}j|_j|_ j rBt |nd|_ |durX||_ntj||j|_tj||j|_tfddtjD|_d|_|dS)Nrcsg|] }tqSr*)r)r;_r}r*r+r?r@z*MarianEncoder.__init__..F)r3r4rgZencoder_layerdrop layerdroprr$r0max_position_embeddingsZmax_source_positionsscale_embeddingmathsqrt embed_scalerrr vocab_sizer-embed_positions ModuleListrAZencoder_layersrgradient_checkpointing post_init)r5r}rrxr6rr+r4s    zMarianEncoder.__init__)r#rerhrroutput_hidden_states return_dictr1cCs$|dur |n|jj}|dur |n|jj}|dur4|n|jj}|durV|durVtdnP|dur||||}|d|d}n"|dur|dd}ntd|dur|||j }| |} || } t j j | |j |jd} |||}|rdnd} |rdnd} |durR|dt|jksRJdt|jd |dd t|jD]\} }|rt| | f} d }|jrtg}||jkrd }|rd }n*|| ||dur|| nd|d}|d} |r\| |df} q\|r| | f} |stdd| | | fDSt| | | dS)a~ Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. NzDYou cannot specify both input_ids and inputs_embeds at the same timer&z5You have to specify either input_ids or inputs_embedsrjr*rz&The head_mask should be specified for  layers, but it is for .FT)NN)rrrcss|]}|dur|VqdSr2r*r;vr*r*r+ Mr@z(MarianEncoder.forward..last_hidden_stater attentions)r}rruse_return_dictr)Z%warn_if_padding_and_no_attention_maskrmrqrrrrrprgrlrlenr enumeraterGrandrrr)r5r#rerhrrrrrZ embed_posrZencoder_statesZall_attentionsidxZ encoder_layerZto_dropdropout_probability layer_outputsr*r*r+rVsl.         zMarianEncoder.forward)N)NNNNNNN)rXrYrZr[r rrrr4rG LongTensorr^rIrrrrrVr_r*r*r6r+rs& rcseZdZdZd eeejdfdd Zd ee j ee j ee j ee j ee j ee j ee ee j eeeeeeeeee j eee j efdddZZS) MarianDecoderz Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`MarianDecoderLayer`] Args: config: MarianConfig embed_tokens (nn.Embedding): output embedding Nrcstj|_j|_j|_j|_j r>t j nd|_ |durT||_ntjj |j|_tjj |j|_tfddtjD|_d|_|dS)Nrcsg|]}t|dqS))r~)r)r;irr*r+r?lr@z*MarianDecoder.__init__..F)r3r4rgZdecoder_layerdroprr$r0rZmax_target_positionsrrrrrrrrdecoder_vocab_sizer-rrrAZdecoder_layersrrr)r5r}rr6rr+r4\s   zMarianDecoder.__init__)r#rerrrhcross_attn_head_maskrrrrrrrr1c! Cs| dur | n|jj} | dur | n|jj} | dur4| n|jj} | durH| n|jj} |jrn|jrn| rntdd} |du|duArt dnR|dur|}|j }| d|d}n.|dur| dd}|dddddf}|dur| |}||j}| r6|dur6|dur*tt|jdt|jdn t|jd}| r\t|tr\tdt|}| dd\}}|dur|nd}| durtj||||jd } |durts||}tj|||jd }t|tr|jn|}|||| |}|||||}|j||f|| d }||}tjj ||j |jd }| rDd nd}| rRd nd}| rj|durjd nd}t!||gd dgD]V\}}|dur| dt"|j#ksJd|dt"|j#d| ddqt$|j#D]\}}| r||f7}|jrt%g}||j&krq||||||dur8||nd|durL||nd|| | | d } | d}| r|| df7}|dur|| df7}q| r||f7}| stdd|||||fDSt'|||||dS)a Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing cross-attention on hidden heads. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. It is used to update the cache in the correct position and to infer the complete sequence length. NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz:You must specify exactly one of input_ids or inputs_embedsr&rzPassing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. You should pass an instance of `EncoderDecoderCache` instead, e.g. `past_key_values=EncoderDecoderCache.from_legacy_cache(past_key_values)`.rr)rRrjr*rhrzThe `z` should be specified for rr)rrrrrrrrr8css|]}|dur|VqdSr2r*rr*r*r+rHsz(MarianDecoder.forward..)rrrrcross_attentions)(r}rrrrrrlrrr)r'rqrmrrr r rrZfrom_legacy_cacherrGrTrSrrrrrrrrprgziprrrr rr)!r5r#rerrrhrrrrrrrrinputrrZ seq_lengthrQZmask_seq_lengthZself_attn_cacherrRrZall_hidden_statesZall_self_attnsZall_cross_attentionsZ attn_maskZ mask_namer Z decoder_layerr r r*r*r+rVrsT                   zMarianDecoder.forward)N) NNNNNNNNNNNNN)rXrYrZr[r rrrr4rGr r^rIr rrrrrVr_r*r*r6r+rSs>rcseZdZddgZedfdd ZddZdd Zd d Zd d Z ddZ e e j dddZedeejeejeejeejeejeejeejeeeejefeeeejeejeeeeeeeeeejedddZZS) MarianModelzencoder.embed_tokens.weightzdecoder.embed_tokens.weightrcst||j|j}}t||j||_|jj r@|j}}nt |j}t |j}d|_t |||_ t|||_|dSr2)r3r4r$rrrrsharedr} share_encoder_decoder_embeddingscopydeepcopyrencoderrdecoderr)r5r}r0rZencoder_embed_tokensZdecoder_embed_tokensr6r*r+r4Zs      zMarianModel.__init__cCs |Sr2) get_encoderget_input_embeddingsr5r*r*r+rpsz MarianModel.get_input_embeddingscCs0|jjr$||_|j|j_|j|j_n||j_dSr2)r}rrrrrr5rdr*r*r+set_input_embeddingsts   z MarianModel.set_input_embeddingscCs|jjrtd|S)Nz`get_decoder_input_embeddings` should not be called if `config.share_encoder_decoder_embeddings` is `True`. Please use `get_input_embeddings` instead.)r}rr) get_decoderrrr*r*r+get_decoder_input_embeddings|s z(MarianModel.get_decoder_input_embeddingscCs|jjrtd||j_dS)Na `config.share_encoder_decoder_embeddings` is set to `True` meaning the decoder input embeddings are shared with the encoder. In order to set the decoder input embeddings, you should simply set the encoder input embeddings by calling `set_input_embeddings` with the appropriate embeddings.)r}rr)rrrr*r*r+set_decoder_input_embeddingss z(MarianModel.set_decoder_input_embeddingscCs|jSr2)rrr*r*r+rszMarianModel.get_encodernew_num_tokensr1cCsV|jjrtd|}|||}|||}|durB|S||j_||SNz`resize_decoder_token_embeddings` should not be called if `config.share_encoder_decoder_embeddings` is `True`. Please use `resize_token_embeddings` instead.)r}rr)r"_get_resized_embeddingsr#r tie_weights)r5r%old_embeddingsnew_embeddings model_embedsr*r*r+resize_decoder_token_embeddingss  z+MarianModel.resize_decoder_token_embeddingsN)r#rerdecoder_attention_maskrhdecoder_head_maskrencoder_outputsrrdecoder_inputs_embedsrrrrrr1cCs| dur | n|jj} |dur |n|jj}| dur4| n|jj} |durH|n|jj}|durr|j|||| | ||d}nH|rt|tst|dt|dkr|dndt|dkr|dndd}|j |||d|||| | | | |||d }|s||St |j |j |j |j|j|j |j |jdS) a? decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) Marian uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. Example: ```python >>> from transformers import AutoTokenizer, MarianModel >>> tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> model = MarianModel.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> inputs = tokenizer("Studies have been shown that owning a dog is good for you", return_tensors="pt") >>> decoder_inputs = tokenizer( ... " Studien haben gezeigt dass es hilfreich ist einen Hund zu besitzen", ... return_tensors="pt", ... add_special_tokens=False, ... ) >>> outputs = model(input_ids=inputs.input_ids, decoder_input_ids=decoder_inputs.input_ids) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 26, 512] ```N)r#rerhrrrrrrr8r r#rerrrhrrrrrrrr)rrdecoder_hidden_statesdecoder_attentionsrencoder_last_hidden_staterencoder_attentions)r}rrrrrrrrrrrrrrr)r5r#rerr-rhr.rr/rrr0rrrrrZdecoder_outputsr*r*r+rVs`> zMarianModel.forward)NNNNNNNNNNNNNNNN)rXrYrZ_tied_weights_keysr r4rr r"r#rr\rrr,rrrGr r^rrrr rIrrrVr_r*r*r6r+rVsX rzX The Marian Model with a language modeling head. Can be used for summarization. )Z custom_introcs`eZdZdZgdZddgZgdZedfdd Zd d Z d d Z d$e e e e ejdfdd Zd%e ejdddZddZe d dddZejdddZddZed&e eje eje eje eje eje eje eje eeejefe ee eje eje eje e e e e e e e e ejeddd Z ejd!d"d#Z!Z"S)' MarianMTModelr)final_logits_biaszencoder.embed_positions.weightzdecoder.embed_positions.weightz$model.encoder.embed_positions.weightz$model.decoder.embed_positions.weight)z!model.encoder.embed_tokens.weightz!model.decoder.embed_tokens.weightlm_head.weightrcs^t|t||_|jr"|jn|j}|dt d|ft j |j |dd|_ |dS)Nr8rFr)r3r4rrrrrregister_bufferrGzerosrrrlm_headr)r5r}Ztarget_vocab_sizer6r*r+r4.s   zMarianMTModel.__init__cCs |jSr2)rrrr*r*r+r9szMarianMTModel.get_encodercCs |jSr2)rr!rr*r*r+r!<szMarianMTModel.get_decoderNT)r%pad_to_multiple_of mean_resizingr1cs&t|||}|jjr"|||Sr2)r3resize_token_embeddingsr}r_resize_final_logits_bias)r5r%r=r>r*r6r*r+r??s z%MarianMTModel.resize_token_embeddingsr$cGs~|}||||}|||jjd}|jjr<||j_|jjrv|durv|jj sv|}| ||}| ||S)Nr) rr'r rEr'r}rrget_output_embeddingstie_word_embeddings_get_resized_lm_headset_output_embeddings)r5r%r=argsr)r* old_lm_head new_lm_headr*r*r+_resize_token_embeddingsHs      z&MarianMTModel._resize_token_embeddingscCs|jjrtd|j}|||}|j||durd|jjsd|}| ||}| ||j}|durz|S||j_ | | ||Sr&)r}rr)rr"r'r#rArBrCrDrr(r@)r5r%r)r*rFrGr+r*r*r+r,^s$       z-MarianMTModel.resize_decoder_token_embeddingscCsj|jjd}||kr,|jddd|f}n.tjd||f|jjd}tj|j|gdd}|d|dS)Nr&rrrBr8)r8r'rGr;rScatr:)r5r%Zold_num_tokensZnew_biasZ extra_biasr*r*r+r@~s  z'MarianMTModel._resize_final_logits_bias)r*cCs ||_dSr2)r<)r5r*r*r*r+rDsz#MarianMTModel.set_output_embeddingscCs|}|dur6t|jddr6|}|||t|jddrt|jddrt||jrjt||j}||j |j |jd}||_ | D]}t|dr| qdS) z Tie the weights between the input embeddings and the output embeddings. If the `torchscript` flag is set in the configuration, can't handle parameter sharing so we are cloning the weights instead. NrBTis_encoder_decoderFZtie_encoder_decoderr _tie_weights)rAgetattrr}r!rZ_tie_or_clone_weightshasattrrZ_tie_encoder_decoder_weightsrrZ_dynamic_tied_weights_keysmodulesrK)r5Zoutput_embeddingsZword_embeddingsZ tied_weightsrar*r*r+r(s      zMarianMTModel.tie_weights)r#rerr-rhr.rr/rrr0labelsrrrrrr1cCs|dur |n|jj}| durR| r*tdd} |durR| durRt| |jj|jj}|j||||||||| | | | ||||d}||d|j }d}| durt }|| d|jj | d}|s|f|dd}|dur|f|S|St |||j|j|j|j|j|j|jd S) uH decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) Marian uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, MarianMTModel >>> src = "fr" # source language >>> trg = "en" # target language >>> model_name = f"Helsinki-NLP/opus-mt-{src}-{trg}" >>> model = MarianMTModel.from_pretrained(model_name) >>> tokenizer = AutoTokenizer.from_pretrained(model_name) >>> sample_text = "où est l'arrêt de bus ?" >>> batch = tokenizer([sample_text], return_tensors="pt") >>> generated_ids = model.generate(**batch) >>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] "Where's the bus stop?" ``` NzJThe `use_cache` argument is changed to `False` since `labels` is provided.F)rerr/r-rhr.rrrr0rrrrrrr&r) losslogitsrr2r3rr4rr5)r}rrwarningr,r$r%rr<r8rrqrrrr2r3rr4rr5)r5r#rerr-rhr.rr/rrr0rOrrrrrrZ lm_logitsZmasked_lm_lossloss_fctoutputr*r*r+rVs\C zMarianMTModel.forward)rOcCst||jj|jjSr2)r,r}r$r%)r5rOr*r*r+%prepare_decoder_input_ids_from_labelssz3MarianMTModel.prepare_decoder_input_ids_from_labels)NT)N)NNNNNNNNNNNNNNNNN)#rXrYrZrZ_keys_to_ignore_on_load_missingZ_keys_to_ignore_on_saver6r r4rr!r\rrrrr?rHr,r@rDr(rrGr r^rrrr rIrrVrUr_r*r*r6r+r7sp     vr7cs(eZdZdZfddZddZZS)MarianDecoderWrapperz This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is used in combination with the [`EncoderDecoderModel`] framework. cst|t||_dSr2)r3r4rrr5r}r6r*r+r4(s zMarianDecoderWrapper.__init__cOs|j|i|Sr2)r)r5rErsr*r*r+rV,szMarianDecoderWrapper.forward)rXrYrZr[r4rVr_r*r*r6r+rV"s rVcseZdZdgZfddZddZddZdd Zd d Ze de e j e e j e e je e je e j e e j e ee e je e j e ee ee ee ee e j eeefd ddZZS)MarianForCausalLMr9csDd|_d|_t|t||_tj|j|j dd|_ | dS)NTFr) rzrJr3r4rVrrrZ hidden_sizerr<rrWr6r*r+r44s   zMarianForCausalLM.__init__cCs |jjjSr2rrrrr*r*r+r?sz&MarianForCausalLM.get_input_embeddingscCs||jj_dSr2rYrr*r*r+r Bsz&MarianForCausalLM.set_input_embeddingscCs ||j_dSr2rr)r5rr*r*r+ set_decoderEszMarianForCausalLM.set_decodercCs|jjSr2rZrr*r*r+r!HszMarianForCausalLM.get_decoderN)r#rerrrhrrrrOrrrrrr1cCs| dur | n|jj} | dur | n|jj} | dur4| n|jj} |jj||||||||| | | | |d }||d}d}| dur| |j} t }|| d|jj | d}| s|f|dd}|dur|f|S|St |||j |j|j|jdS)a cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, MarianForCausalLM >>> tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-fr-en") >>> model = MarianForCausalLM.from_pretrained("Helsinki-NLP/opus-mt-fr-en", add_cross_attention=False) >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> logits = outputs.logits >>> expected_shape = [1, inputs.input_ids.shape[-1], model.config.vocab_size] >>> list(logits.shape) == expected_shape True ```Nr1rr&r)rPrQrrrr)r}rrrrrr<rrSrrqrrrrrr)r5r#rerrrhrrrrOrrrrrrrQrPrSrTr*r*r+rVKsH. zMarianForCausalLM.forward)NNNNNNNNNNNNNN)rXrYrZr6r4rr r[r!rrrGr r^rIr rrrrrVr_r*r*r6r+rX1sL  rX)rXrr7r)Nr`N)Lr[rrtypingrrrnumpyr9rGZtorch.utils.checkpointrZtorch.nnrZ activationsrZ cache_utilsr r r Z generationr Zmodeling_attn_mask_utilsr rrZmodeling_flash_attention_utilsrZmodeling_layersrZmodeling_outputsrrrrrZmodeling_utilsrrZprocessing_utilsrutilsrrrrZutils.deprecationrZconfiguration_marianr Z!torch.nn.attention.flex_attentionr!Zintegrations.flex_attentionr"Z get_loggerrXrr^r\r,rr-Modulerrvrwrrrrrrr7rVrX__all__r*r*r*r+s~            ) HugIu