a h @sdZddlZddlmZmZmZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZmZmZmZmZdd lmZmZdd lmZdd lmZmZdd lmZmZm Z m!Z!dd l"m#Z#m$Z$ddl%m&Z&e!'e(Z)ddZ*ddZ+ddZ,ddZ-ddZ.Gdddejj/Z0Gdddej/Z1Gdddej/Z2dGej/ej3ej3ej3eej3e4e4eej3eed! d"d#Z5Gd$d%d%ej/Z6Gd&d'd'ej/Z7Gd(d)d)ej/Z8Gd*d+d+ej/Z9Gd,d-d-ej/Z:Gd.d/d/eZ;Gd0d1d1ej/ZeGd6d7d7e>Z?eGd8d9d9e>Z@Gd:d;d;ej/ZAedd?d?e>ZBeGd@dAdAe>ZCGdBdCdCej/ZDdDdEZEgdFZFdS)HzPyTorch ESM model.N)CallableOptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack) find_pruneable_heads_and_indicesprune_linear_layer)TransformersKwargsauto_docstringcan_return_tuplelogging)OutputRecordercheck_model_inputs) EsmConfigcCs&|jddd\}}tj| |fddS)Ndim)chunktorchcat)xx1Zx2r&`/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/models/esm/modeling_esm.py rotate_half-sr(cCs`|ddddd|jdddf}|ddddd|jdddf}||t||S)N)shaper()r$cossinr&r&r'apply_rotary_pos_emb2s&&r-cCs |ddt|tdS)zo This is the gelu implementation from the original ESM repo. Using F.gelu yields subtly wrong results. g??g@)r"erfmathsqrtr$r&r&r'gelu9sr3cCs||ddS)zJMake layer symmetric in final two dimensions, used for contact prediction.rr)) transposer2r&r&r' symmetrize@sr5cCsH|jddd}|jddd}|jddd}||}||||}|S)z=Perform average product correct, used for contact prediction.rT)Zkeepdimsr))rr))sumZdiv_)r$Za1Za2Za12Zavg normalizedr&r&r'average_product_correctEs r8cs^eZdZUdZejed<edfdd Zd ddZ ejeje ejejfd d d Z Z S) RotaryEmbeddingz Rotary position embeddings based on those in [RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer). Query and keys are transformed by rotation matrices which depend on their relative positions. inv_freqrcsTtddtjd|dtjd|}|}|d|d|_d|_d|_ dS)Nr.i'rrdtyper:) super__init__r"arangeZint64floatregister_buffer_seq_len_cached _cos_cached _sin_cached)selfr r: __class__r&r'r>Zs $ zRotaryEmbedding.__init__rcCs|j|}||jks"|jj|jkr||_tj|j||jd|j}t||j}tj ||fdd |j}| ddddddf|_| ddddddf|_ |j|j fS)Ndevicerr)r*rBrCrIr"r?type_asr:outerr#tor+r,rD)rEr$ seq_dimensionZseq_lentZfreqsZembr&r&r'_update_cos_sin_tableses z&RotaryEmbedding._update_cos_sin_tables)qkreturncCsJ|j|dd\|_|_t||j|jj|jdt||j|jj|jdfS)Nr))rMr;)rOrCrDr-rLr<)rErPrQr&r&r'forwarduszRotaryEmbedding.forward)r) __name__ __module__ __qualname____doc__r"Tensor__annotations__intr>rOtuplerS __classcell__r&r&rFr'r9Qs   r9cs2eZdZdZd eedfdd ZddZZS) EsmContactPredictionHeadzWPerforms symmetrization, apc, and computes a logistic regression on the output featuresTr) in_featureseos_idxcs4t||_||_t|d||_t|_dS)Nr) r=r>r^r_rLinear regressionZSigmoid activation)rEr^biasr_rFr&r'r>s  z!EsmContactPredictionHead.__init__c Cs||j|}|d|d}||ddddddddf}|dddddf}|dddddf}|\}}}}}||||||}||jjj}t t |}| dddd}| || dS)Nrr.rrr )ner_rL unsqueezesizeviewraweightrIr8r5Zpermuterbsqueeze) rEtokens attentionsZeos_mask batch_sizeZlayersheadsZseqlen_r&r&r'rSs" z EsmContactPredictionHead.forward)Tr)rTrUrVrWrZr>rSr\r&r&rFr'r]~s r]cs2eZdZdZfddZd ddZddZZS) EsmEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. csttj|j|j|jd|_|jr>tj |j|j d|_ nd|_ t |j |_t|dd|_|jdt|jddd|j|_|jdkrtj|j|j|jd|_|j|_|j|_dS) N) padding_idxepsposition_embedding_typeabsolute position_ids)rrF) persistent)r=r>r Embedding vocab_size hidden_sizeZ pad_token_idword_embeddingsZemb_layer_norm_before LayerNormlayer_norm_eps layer_normDropouthidden_dropout_probdropoutgetattrrsrAr"r?max_position_embeddingsexpandrpposition_embeddings token_dropout mask_token_idrEconfigrFr&r'r>s"   zEsmEmbeddings.__init__Nc Cs|dur(|durt||j}n ||}|dur:||}|}|jr|dur|||jkdd}d}|durz|dn|j d}||jkd |}|d|d|ddddf |j }|j dkr||} || }|jdur||}|dur||d |j }|S)NrgQ?rrt)"create_position_ids_from_input_idsrp&create_position_ids_from_inputs_embedsrzrZ masked_fillrrer6r*r@rLr<rsrr}) rE input_idsattention_maskru inputs_embeds embeddingsZmask_ratio_trainZ src_lengthsZmask_ratio_observedrr&r&r'rSs.   "     zEsmEmbeddings.forwardcCsN|dd}|d}tj|jd||jdtj|jd}|d|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nrrr<rIr)rfr"r?rplongrIrer)rEr input_shapeZsequence_lengthrur&r&r'rs z4EsmEmbeddings.create_position_ids_from_inputs_embeds)NNNN)rTrUrVrWr>rSrr\r&r&rFr'ros  1ror) modulequerykeyvaluerscalingr head_maskkwargscKst||dd|} t|dr|jdvr|jd} tj| tj| jd dd} tj| tj| jd dd} | | } | | |j d}|j |j d}|jd krtd ||}n.|jd krtd ||}td ||}||}| |} |dur |ddddddd|jd f}| |} tjj| dtjd |j } tjj| ||jd} |durb| |} t| |}|dd}|| fS)Nrr rs relative_keyrelative_key_queryrrrr;rzbhld,lrd->bhlrrzbhrd,lrd->bhlrr))r r<)ptraining)r"matmulr4hasattrrsr*r?rrIrgdistance_embeddingrrLr<ZeinsumrZ functionalZsoftmaxZfloat32rr contiguous)rrrrrrrrr attn_weights seq_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZ causal_mask attn_outputr&r&r'eager_attention_forwards2     &  rc s^eZdZdfdd Zd ejeejeejeejeejee e ejdddZ Z S) EsmSelfAttentionNFcs<t||_|j|jdkrDt|dsDtd|jd|jd|j|_t|j|j|_|j|j|_ t |j|j |_ t |j|j |_ t |j|j |_|j|_|pt|dd|_d|_|jdks|jd kr|j|_t d |jd |j|_n|jd krt|jd |_d|_|j|_||_|jo4| |_dS)NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()rsrtrrrrrotaryrr.)r=r>rrynum_attention_headsr ValueErrorrZattention_head_size all_head_sizerr`rrrZattention_probs_dropout_probrrrsrotary_embeddingsrrwrr9r is_decoder layer_idxZ is_causal)rErrsris_cross_attentionrFr&r'r>3s8   zEsmSelfAttention.__init__) hidden_statesrrencoder_hidden_statesencoder_attention_maskrrRcKs@|jdd\}}||d|jf} ||| dd} |du} | rH|n|} | rT|n|}|| | dd} || | dd}| |jd} |jdkr|| | \} } t }|j j dkr|jdvrt d|j j d |jd t |j j }||| | ||f|js d n|j|j|d |\}}|||d}||fS) NrrrgreagerrzESM z attention does not support z^ embeddings. Set attention explicitly to 'eager' with `model.set_attn_implementation('eager')`r)rrr)r*rrrgr4rrrsrrr_attn_implementationrrrrrZreshaper)rErrrrrrrlrZ hidden_shapeZ query_layerrZcurrent_statesZ key_layerZ value_layerZattention_interfacerrr&r&r'rSUsB         zEsmSelfAttention.forward)NNF)NNNN) rTrUrVr>r"rXr FloatTensorrrr[rSr\r&r&rFr'r2s%rcs$eZdZfddZddZZS) EsmSelfOutputcs.tt|j|j|_t|j|_dSN) r=r>rr`rydenser~rrrrFr&r'r>s zEsmSelfOutput.__init__cCs ||}||}||}|SrrrrErZ input_tensorr&r&r'rSs  zEsmSelfOutput.forwardrTrUrVr>rSr\r&r&rFr'rs rcs:eZdZd fdd ZddZd eeddd ZZS) EsmAttentionNFcsDtt|||d|_t||_t|_tj |j |j d|_ dS)N)rrrq) r=r>rrEroutputset pruned_headsrr{ryr|)rErrrrFr&r'r>s   zEsmAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrr)lenrrErrrrrrrrrrunion)rErmindexr&r&r' prune_headss zEsmAttention.prune_headsrc Ks:||}|j|f||||d|\}} |||}|S)Nrrrr)r{rEr) rErrrrrrZhidden_states_lnrrnr&r&r'rSs   zEsmAttention.forward)NF)NNNN) rTrUrVr>rrrrSr\r&r&rFr'rsrcs0eZdZfddZejejdddZZS)EsmIntermediatecs tt|j|j|_dSr)r=r>rr`ryintermediate_sizerrrFr&r'r>s zEsmIntermediate.__init__rrRcCs||}t|}|Sr)rr3)rErr&r&r'rSs zEsmIntermediate.forwardrTrUrVr>r"rXrSr\r&r&rFr'rs rcs$eZdZfddZddZZS) EsmOutputcs.tt|j|j|_t|j|_ dSr) r=r>rr`rryrr~rrrrFr&r'r>s zEsmOutput.__init__cCs ||}||}||}|Srrrr&r&r'rSs  zEsmOutput.forwardrr&r&rFr'rs rcs8eZdZfddZd eedddZddZZS) EsmLayercst|j|_d|_t||_|j|_|j|_|jrZ|jsLt|dt|dd|_ t ||_ t ||_ tj|j|jd|_dS)Nrz> should be used as a decoder model if cross attention is addedT)rrq)r=r>Zchunk_size_feed_forwardZ seq_len_dimr attentionrZadd_cross_attention RuntimeErrorcrossattentionr intermediaterrrr{ryr|rrFr&r'r>s    zEsmLayer.__init__Nrc Ksj|j|f||d|}|jr\|dur\t|ds@td|d|j|f||||d|}||}|S)N)rrrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r)rrrAttributeErrorrfeed_forward_chunk) rErrrrrrattention_output layer_outputr&r&r'rSs2    zEsmLayer.forwardcCs$||}||}|||}|Sr)r{rr)rErZattention_output_lnZintermediate_outputrr&r&r'rs   zEsmLayer.feed_forward_chunk)NNNN) rTrUrVr>rrrSrr\r&r&rFr'rs  #rcs4eZdZfddZedeedddZZS) EsmEncodercsNt|_tfddtjD|_tjj j d|_ d|_ dS)Ncsg|] }tqSr&)r).0rnrr&r' z'EsmEncoder.__init__..rqF) r=r>rrZ ModuleListrangenum_hidden_layerslayerr{ryr|emb_layer_norm_afterZgradient_checkpointingrrFrr'r>s   zEsmEncoder.__init__Nrc Ks\t|jD]6\}}|dur"||nd} ||f|| ||d|}q |jrR||}t|dS)Nr)last_hidden_state) enumeraterrr ) rErrrrrriZ layer_moduleZlayer_head_maskr&r&r'rS"s   zEsmEncoder.forward)NNNN) rTrUrVr>rrrrSr\r&r&rFr'rs rcs0eZdZfddZejejdddZZS) EsmPoolercs*tt|j|j|_t|_dSr)r=r>rr`ryrZTanhrbrrFr&r'r>?s zEsmPooler.__init__rcCs(|dddf}||}||}|SNr)rrb)rErZfirst_token_tensor pooled_outputr&r&r'rSDs  zEsmPooler.forwardrr&r&rFr'r>s rc@speZdZUeed<dZdZgdZdgZdZ dZ dZ dZ e eedddgeedd dgd Zd d Zd dZdS)EsmPreTrainedModelresmT)rZ#EsmFoldTriangularSelfAttentionBlockrozposition_embeddings.weightrr)rZ layer_namer)rrkZcross_attentionscCst|tjr:|jjjd|jjd|jdur|jj n~t|tj rz|jjjd|jjd|j dur|jj|j  n>t|tj r|jj |jj dnt|tr|jj dS)zInitialize the weightsr)meanZstdNr.) isinstancerr`rhdataZnormal_rZinitializer_rangercZzero_rwrpr{Zfill_ EsmLMHead)rErr&r&r' _init_weightsbs       z EsmPreTrainedModel._init_weightscCsdSrr&rEr&r&r'get_output_embeddingstsz(EsmPreTrainedModel.get_output_embeddingsN)rTrUrVrrYZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesZ"_keys_to_ignore_on_load_unexpectedZ_supports_flash_attnZ_supports_sdpaZ_supports_flex_attnZ_supports_attention_backendrrrZ_can_record_outputsrrr&r&r&r'rMs   rcseZdZdZdfdd ZddZddZd d Zee de e j e e j e e j e e j e e j e e j e e j e eeee j efd d dZddZZS)EsmModela  The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. TcsZt|||_t||_t||_|r2t|nd|_t |j |j dd|_ | dS)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer NT)r^rc)r=r>rrorrencoderrpoolerr]rr contact_head post_init)rEradd_pooling_layerrFr&r'r>s    zEsmModel.__init__cCs|jjSrrrzrr&r&r'get_input_embeddingsszEsmModel.get_input_embeddingscCs ||j_dSrr)rErr&r&r'set_input_embeddingsszEsmModel.set_input_embeddingscCs*|D]\}}|jj|j|qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrrr)rEZheads_to_prunerrmr&r&r' _prune_headsszEsmModel._prune_headsN) rrrurrrrrrRcKs"|du|duArtd|dur.|j||d}|jjdkrz|jdd\} } |durhtj| | f|jd}|j|| | fd}|jj r|dur| \} } } | | f}|durtj||jd}| |}nd}| ||jj }|j|f||||d|}|d }|jdur||nd}t||d S) aV input_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) position_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `((batch_size, sequence_length), hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. Nz:You must specify exactly one of input_ids or inputs_embeds)rruZflash_attention_2rrH)rrr)rZ pooler_output)rrrrr*r"ZonesrIZget_extended_attention_maskrrfZinvert_attention_maskZ get_head_maskrrrr )rErrrurrrrrrlrZencoder_batch_sizeZencoder_sequence_lengthrnZencoder_hidden_shapeZencoder_extended_attention_maskZencoder_outputssequence_outputrr&r&r'rSsL  zEsmModel.forwardcCs`|||dddj}tj|dd}||ddd9}||ddd9}|||S)NT)rZ return_dictZoutput_attentionsrrrr )rkr"stackrer)rErjrZattnsr&r&r'predict_contactss zEsmModel.predict_contacts)T)NNNNNNN)rTrUrVrWr>rrrrrrr"rXrrrr[r rSrr\r&r&rFr'rzs4 QrcseZdZdgZfddZddZddZeede e j e e j e e j e e j e e j e e j e e j e e j eeeeefd d d Zd d ZZS)EsmForMaskedLMzlm_head.decoder.weightcsHt||jrtdt|dd|_t||_| | dS)NzjIf you want to use `EsmForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Fr) r=r>rloggerwarningrrrlm_head init_weightsrrrFr&r'r> s  zEsmForMaskedLM.__init__cCs|jjSrrdecoderrr&r&r'rsz$EsmForMaskedLM.get_output_embeddingscCs ||j_dSrr)rEZnew_embeddingsr&r&r'set_output_embeddings sz$EsmForMaskedLM.set_output_embeddingsN) rrrurrrrlabelsrrRc  Ks|j|f||||||d| } | d} || } d} |durlt}|| j}|| d|jj|d} t| | | j | j dS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` )rrurrrrrNrlosslogitsrrk) rrrrLrIrgrrxr rrk)rErrrurrrrrroutputsrZprediction_scoresZmasked_lm_lossloss_fctr&r&r'rS#s2   zEsmForMaskedLM.forwardcCs|jj||dS)N)r)rr)rErjrr&r&r'rSszEsmForMaskedLM.predict_contacts)NNNNNNNN)rTrUrVZ_tied_weights_keysr>rrrrrr" LongTensorrXrrrrr[r rSrr\r&r&rFr'r s6  .rcs(eZdZdZfddZddZZS)rz&ESM Head for masked language modeling.cs^tt|j|j|_tj|j|jd|_tj|j|j dd|_ t t |j |_dS)NrqF)rc)r=r>rr`ryrr{r|r}rxr Parameterr"ZzerosrcrrFr&r'r>Zs  zEsmLMHead.__init__cKs0||}t|}||}|||j}|Sr)rr3r}rrcrEfeaturesrr$r&r&r'rSbs   zEsmLMHead.forwardrTrUrVrWr>rSr\r&r&rFr'rWs rz ESM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. )Z custom_introc sreZdZfddZeedeejeej eejeej eej eeje e e eefdddZZS)EsmForSequenceClassificationcsFt||j|_||_t|dd|_t||_|| dSNFr) r=r> num_labelsrrrEsmClassificationHead classifierrrrrFr&r'r>ss  z%EsmForSequenceClassification.__init__NrrrurrrrrRc Ks:|j|f||||d|}|d} || } d} |dur&|| j}|jjdur|jdkrhd|j_n4|jdkr|jtj ks|jtj krd|j_nd|j_|jjdkrt } |jdkr| | | } n | | |} nN|jjdkrt } | | d|j|d} n|jjdkr&t} | | |} t| | |j|jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). rrurrrNrraZsingle_label_classificationZmulti_label_classificationrr)rrrLrIrZ problem_typerr<r"rrZrrirrgrrrrk rErrrurrrrr rr rr r&r&r'rSsL      "     z$EsmForSequenceClassification.forward)NNNNNN)rTrUrVr>rrrr"r rXrrrrr[rrSr\r&r&rFr'rls&  rc sreZdZfddZeedeejeej eejeej eej eeje e e eefdddZZS)EsmForTokenClassificationcsVt||j|_t|dd|_t|j|_t |j |j|_ | | dSr)r=r>rrrrr~rrr`ryrrrrrFr&r'r>s z"EsmForTokenClassification.__init__Nrc Ks|j|f||||d|}|d} || } || } d} |durpt} || j}| | d|j|d} t| | |j |j dS)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. rrNrr) rrrrrLrIrgrrrrkrr&r&r'rSs0    z!EsmForTokenClassification.forward)NNNNNN)rTrUrVr>rrrr"r rXrrrrr[rrSr\r&r&rFr'rs&  rcs(eZdZdZfddZddZZS)rz-Head for sentence-level classification tasks.cs@tt|j|j|_t|j|_t|j|j |_ dSr) r=r>rr`ryrr~rrrout_projrrFr&r'r>s zEsmClassificationHead.__init__cKsL|dddddf}||}||}t|}||}||}|Sr)rrr"tanhrrr&r&r'rSs     zEsmClassificationHead.forwardrr&r&rFr'rs rcCs2||}tj|dd||}||S)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rr)rdrZr"ZcumsumrJr)rrpmaskZincremental_indicesr&r&r'r s r)rrrrr)rN)GrWr0typingrrrr"Ztorch.utils.checkpointrZtorch.nnrrrZmodeling_layersr Zmodeling_outputsr r r rrZmodeling_utilsrrZprocessing_utilsrZ pytorch_utilsrrutilsrrrrZ utils.genericrrZconfiguration_esmrZ get_loggerrTrr(r-r3r5r8Moduler9r]rorXr@rrrrrrrrrrrrrrrrr__all__r&r&r&r'sz      -#f 2Y 0  :$,ML;