a
    ãÀhS	  ã                   @  s2   d Z ddlmZ ddlZddlZG dd„ dƒZdS )a@  
This file contains deprecated code that can only be used with the old `model.fit`-style Sentence Transformers v2.X training.
It exists for backwards compatibility with the `model.old_fit` method, but will be removed in a future version.

Nowadays, with Sentence Transformers v3+, it is recommended to use the `SentenceTransformerTrainer` class to train models.
See https://www.sbert.net/docs/sentence_transformer/training_overview.html for more information.

In particular, you can pass "no_duplicates" to `batch_sampler` in the `SentenceTransformerTrainingArguments` class.
é    )ÚannotationsNc                   @  s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚNoDuplicatesDataLoaderc                 C  s(   || _ d| _d| _|| _t | j¡ dS )z±
        A special data loader to be used with MultipleNegativesRankingLoss.
        The data loader ensures that there are no duplicate sentences within the same batch
        r   N)Ú
batch_sizeÚdata_pointerÚ
collate_fnÚtrain_examplesÚrandomÚshuffle)Úselfr   r   © r   úq/var/www/html/assistant/venv/lib/python3.9/site-packages/sentence_transformers/datasets/NoDuplicatesDataLoader.pyÚ__init__   s
    zNoDuplicatesDataLoader.__init__c                 c  sþ   t |  ¡ ƒD ]ì}g }tƒ }t|ƒ| jk rÞ| j| j }d}|jD ].}t|t	ƒsTt	|ƒ}| 
¡  ¡ |v r>d} qnq>|r¬| |¡ |jD ](}t|t	ƒs˜t	|ƒ}| | 
¡  ¡ ¡ q‚|  jd7  _| jt| jƒkrd| _t | j¡ q| jd urò|  |¡n|V  qd S )NTFé   r   )ÚrangeÚ__len__ÚsetÚlenr   r   r   ZtextsÚ
isinstanceÚstrÚstripÚlowerÚappendÚaddr   r	   r   )r
   Ú_ÚbatchZtexts_in_batchZexampleZvalid_exampleÚtextr   r   r   Ú__iter__   s.    




zNoDuplicatesDataLoader.__iter__c                 C  s   t  t| jƒ| j ¡S )N)ÚmathÚfloorr   r   r   )r
   r   r   r   r   ;   s    zNoDuplicatesDataLoader.__len__N)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r   r   r   r      s   r   )Ú__doc__Ú
__future__r   r   r   r   r   r   r   r   Ú<module>   s   
