a
    hF                     @   s   d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 e
eZddd	d
Zdd Zdd Zdd ZG dd deZdgZdS )zTokenization classes for FSMT.    N)Optional   )PreTrainedTokenizer)loggingzvocab-src.jsonzvocab-tgt.jsonz
merges.txt)src_vocab_filetgt_vocab_filemerges_filec                 C   s6   t  }| d }| dd D ]}|||f |}q|S )z
    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length
    strings)
    r      N)setadd)wordpairsZ	prev_charchar r   f/var/www/html/assistant/venv/lib/python3.9/site-packages/transformers/models/fsmt/tokenization_fsmt.py	get_pairs$   s    r   c                 C   s  |  dd} tdd| } |  dd} |  dd} |  dd} |  d	d
} |  dd
} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  d d!} |  d"d#} |  d$d%} |  d&d'} |  d(d)} |  d*d+} |  d,d-} td.d| } |  d/d0} |  d1d2} |  d3d4} |  d5d6} |  d7d8} |  d9d:} |  d;d<} |  d=d>} |  d?d@} | S )Azz
    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
    u   ，,u   。\s*z. u   、u   ”"u   “u   ∶:u   ：u   ？?u   《u   》u   ）)u   ！!u   （(u   ；;u   １1u   」u   「u   ０0u   ３3u   ２2u   ５5u   ６6u   ９9u   ７7u   ８8u   ４4u   ．\s*u   ～~u   ’'u   …z...u   ━-u   〈<u   〉>u   【[u   】]u   ％%)replaceresub)textr   r   r   replace_unicode_punct1   sJ    r0   c                 C   s8   g }| D ]$}t |}|dr"q|| qd|S )zw
    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/remove-non-printing-char.perl
    C )unicodedatacategory
startswithappendjoin)r/   outputr   catr   r   r   remove_non_printing_char\   s    

r:   c                	       sB  e Zd ZdZeZddgZd9 fd
d	Zee	e
f dddZee
dddZdd Zdd Zdd Zdd Zedd Zedd Zdd Zdd  Zd!d" Zd:d$d%Zd&d' Zd(d) Zd*d+ Zd;ee
 eee
  ee
 d,d-d.Zd<ee
 eee
  eee
 d/ fd0d1Zd=e	ee	 e e	 d2d3d4Z!d5d6 Z"d7d8 Z#  Z$S )>FSMTTokenizera	  
    Construct an FAIRSEQ Transformer tokenizer. Based on Byte-Pair Encoding. The tokenization process is the following:

    - Moses preprocessing and tokenization.
    - Normalizing all inputs text.
    - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols (like
      "__classify__") to a vocabulary.
    - The argument `langs` defines a pair of languages.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        langs (`List[str]`, *optional*):
            A list of two languages to translate from and to, for instance `["en", "ru"]`.
        src_vocab_file (`str`, *optional*):
            File containing the vocabulary for the source language.
        tgt_vocab_file (`st`, *optional*):
            File containing the vocabulary for the target language.
        merges_file (`str`, *optional*):
            File containing the merges.
        do_lower_case (`bool`, *optional*, defaults to `False`):
            Whether or not to lowercase the input when tokenizing.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.

    Z	input_idsZattention_maskNF<unk><s></s><pad>c
                    s  zdd l }W n ty&   tdY n0 || _|| _|| _|| _|| _i | _i | _i | _	|rvt
|dkrv|\| _| _ntd| dt|dd}t|| _W d    n1 s0    Y  t|dd.}t|}dd	 | D | _W d    n1 s0    Y  t|dd&}| d
d d }W d    n1 sB0    Y  dd |D }tt|tt
|| _i | _t jf |||||||||	d	|
 d S )Nr   nYou need to install sacremoses to use XLMTokenizer. See https://pypi.org/project/sacremoses/ for installation.   zFarg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got zw. Usually that means that tokenizer can't find a mapping for the given model path in  and other maps of this tokenizer.utf-8encodingc                 S   s   i | ]\}}||qS r   r   .0kvr   r   r   
<dictcomp>       z*FSMTTokenizer.__init__.<locals>.<dictcomp>
c                 S   s    g | ]}t | d d qS )NrA   )tuplesplit)rF   merger   r   r   
<listcomp>   rJ   z*FSMTTokenizer.__init__.<locals>.<listcomp>)	langsr   r   r   do_lower_case	unk_token	bos_token	sep_token	pad_token)
sacremosesImportErrorsmr   r   r   rR   cache_moses_punct_normalizercache_moses_tokenizercache_moses_detokenizerlensrc_langtgt_lang
ValueErroropenjsonloadencoderitemsdecoderreadrN   dictziprange	bpe_rankscachesuper__init__)selfrQ   r   r   r   rR   rS   rT   rU   rV   kwargsrW   Zsrc_vocab_handleZtgt_vocab_handle	tgt_vocabZmerges_handleZmerges	__class__r   r   rn      sT    

*
26
zFSMTTokenizer.__init__)returnc                 C   s   |   S N)get_src_vocabro   r   r   r   	get_vocab   s    zFSMTTokenizer.get_vocabc                 C   s   | j S ru   )src_vocab_sizerw   r   r   r   
vocab_size   s    zFSMTTokenizer.vocab_sizec                 C   s2   || j vr"| jj|d}|| j |< | j | |S Nlang)rZ   rY   ZMosesPunctNormalizer	normalize)ro   r/   r}   Zpunct_normalizerr   r   r   moses_punct_norm   s    

zFSMTTokenizer.moses_punct_normc                 C   s:   || j vr"| jj|d}|| j |< | j | j|ddddS )Nr|   TF)Zaggressive_dash_splitsZ
return_strescape)r[   rY   ZMosesTokenizertokenize)ro   r/   r}   Zmoses_tokenizerr   r   r   moses_tokenize   s    


zFSMTTokenizer.moses_tokenizec                 C   s2   || j vr"| jj|d}|| j |< | j | |S r{   )r\   rY   ZMosesDetokenizerZ
detokenize)ro   tokensr}   Zmoses_detokenizerr   r   r   moses_detokenize   s    

zFSMTTokenizer.moses_detokenizec                 C   s    t |}| ||}t|}|S ru   )r0   r   r:   )ro   r/   r}   r   r   r   moses_pipeline  s    zFSMTTokenizer.moses_pipelinec                 C   s
   t | jS ru   )r]   rd   rw   r   r   r   ry     s    zFSMTTokenizer.src_vocab_sizec                 C   s
   t | jS ru   )r]   rf   rw   r   r   r   tgt_vocab_size  s    zFSMTTokenizer.tgt_vocab_sizec                 C   s   t | jfi | jS ru   )rh   rd   Zadded_tokens_encoderrw   r   r   r   rv     s    zFSMTTokenizer.get_src_vocabc                 C   s   t | jfi | jS ru   )rh   rf   Zadded_tokens_decoderrw   r   r   r   get_tgt_vocab  s    zFSMTTokenizer.get_tgt_vocabc           
         s  t |d d |d d f }| jv r2 j| S t|}|sF|d S t| fddd}| jvrhqd|\}}g }d}|t|k r:z|||}	W n* ty   |||d   Y q:Y n0 ||||	  |	}|| |kr"|t|d k r"||d  |kr"|	||  |d7 }qx|	||  |d7 }qxt |}|}t|dkrZqdqFt|}qFd	
|}|d
kr|d}| j|< |S )NrL   </w>c                    s    j | tdS )Ninf)rk   getfloat)pairrw   r   r   <lambda>   rJ   z#FSMTTokenizer.bpe.<locals>.<lambda>keyr   r	   rA    z
  </w>z
</w>)rM   rl   r   minrk   r]   indexr`   extendr6   r7   )
ro   tokenr   r   ZbigramfirstsecondZnew_wordijr   rw   r   bpe  sF    


2





zFSMTTokenizer.bpeenc                 C   sn   | j }| jr| }|r"| }n| j||d}| j||d}g }|D ]"}|rF|t| |d qF|S )av  
        Tokenize a string given language code using Moses.

        Details of tokenization:

            - [sacremoses](https://github.com/alvations/sacremoses): port of Moses
            - Install with `pip install sacremoses`

        Args:
            - lang: ISO language code (default = 'en') (string). Languages should belong of the model supported
              languages. However, we don't enforce it.
            - bypass_tokenizer: Allow users to preprocess and tokenize the sentences externally (default = False)
              (bool). If True, we only apply BPE.

        Returns:
            List of tokens.
        r|   r   )	r^   rR   lowerrN   r   r   r   listr   )ro   r/   r}   Zbypass_tokenizerZsplit_tokensr   r   r   r   	_tokenizeB  s    
zFSMTTokenizer._tokenizec                 C   s   | j || j | jS )z0Converts a token (str) in an id using the vocab.)rd   r   rS   )ro   r   r   r   r   _convert_token_to_idi  s    z"FSMTTokenizer._convert_token_to_idc                 C   s   | j || jS )z=Converts an index (integer) in a token (str) using the vocab.)rf   r   rS   )ro   r   r   r   r   _convert_id_to_tokenm  s    z"FSMTTokenizer._convert_id_to_tokenc                 C   s.   dd |D }d | }| || j}|S )z:Converts a sequence of tokens (string) in a single string.c                 S   s    g | ]}| d d dd qS )r   r2   r   )r,   )rF   tr   r   r   rP   u  rJ   z:FSMTTokenizer.convert_tokens_to_string.<locals>.<listcomp>r2   )r7   rN   r   r_   )ro   r   r/   r   r   r   convert_tokens_to_stringq  s    z&FSMTTokenizer.convert_tokens_to_string)token_ids_0token_ids_1rt   c                 C   s(   | j g}|du r|| S || | | S )a  
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A FAIRSEQ Transformer sequence has the following format:

        - single sequence: `<s> X </s>`
        - pair of sequences: `<s> A </s> B </s>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        N)Zsep_token_id)ro   r   r   sepr   r   r    build_inputs_with_special_tokens{  s    z.FSMTTokenizer.build_inputs_with_special_tokens)r   r   already_has_special_tokensrt   c                    sZ   |rt  j||ddS |durFdgt| dg dgt|  dg S dgt| dg S )a  
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        T)r   r   r   Nr   r	   )rm   get_special_tokens_maskr]   )ro   r   r   r   rr   r   r   r     s    (z%FSMTTokenizer.get_special_tokens_mask)save_directoryfilename_prefixrt   c              	   C   s  t j|s"td| d d S t j||r6|d ndtd  }t j||rX|d ndtd  }t j||rz|d ndtd  }t|dd	d
.}|t	j
| jddddd  W d    n1 s0    Y  t|dd	d
@}dd | j D }|t	j
|ddddd  W d    n1 s*0    Y  d}t|dd	d
n}	t| j dd dD ]F\}
}||krtd| d |}|	d|
d  |d7 }q^W d    n1 s0    Y  |||fS )NzVocabulary path (z) should be a directoryr&   r2   r   r   r   wrB   rC   rA   TF)indent	sort_keysensure_asciirK   c                 S   s   i | ]\}}||qS r   r   rE   r   r   r   rI     rJ   z1FSMTTokenizer.save_vocabulary.<locals>.<dictcomp>r   c                 S   s   | d S )Nr	   r   )kvr   r   r   r     rJ   z/FSMTTokenizer.save_vocabulary.<locals>.<lambda>r   zSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!r   r	   )ospathisdirloggererrorr7   VOCAB_FILES_NAMESra   writerb   dumpsrd   rf   re   sortedrk   warning)ro   r   r   r   r   r   frq   r   writerZ
bpe_tokensZtoken_indexr   r   r   save_vocabulary  s8    <<

,zFSMTTokenizer.save_vocabularyc                 C   s   | j  }d |d< |S )NrY   )__dict__copy)ro   stater   r   r   __getstate__  s    
zFSMTTokenizer.__getstate__c                 C   s8   || _ zdd l}W n ty,   tdY n0 || _d S )Nr   r@   )r   rW   rX   rY   )ro   drW   r   r   r   __setstate__  s    
zFSMTTokenizer.__setstate__)	NNNNFr<   r=   r>   r?   )r   F)N)NF)N)%__name__
__module____qualname____doc__r   Zvocab_files_namesZmodel_input_namesrn   rh   strintrx   propertyrz   r   r   r   r   ry   r   rv   r   r   r   r   r   r   r   r   r   boolr   rM   r   r   r   __classcell__r   r   rr   r   r;   r   sX   -         C

,
'  $r;   )r   rb   r   r-   r3   typingr   Ztokenization_utilsr   utilsr   Z
get_loggerr   r   r   r   r0   r:   r;   __all__r   r   r   r   <module>   s&   
+  x