

import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase				:   Union[str, Any]					   =							get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
     from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_lowerCamelCase				:   Optional[int]					   =							2_5_6_0_4_7
_lowerCamelCase				:   Dict					   =							2_5_6_1_4_5





@require_sentencepiece
@require_tokenizers
class 					__UpperCAmelCase     (       __A			,  unittest.TestCase						):
   UpperCamelCase        =     NllbTokenizer
   UpperCamelCase        =     NllbTokenizerFast
   UpperCamelCase        =     True
   UpperCamelCase        =     True
   UpperCamelCase        =     {}



   def        __magic_name__    (							self							:     List[str]						):
     super().setUp()

     # We have a SentencePiece fixture for testing
     UpperCAmelCase       :							Optional[Any]  = NllbTokenizer(_UpperCamelCase,		keep_accents=_UpperCamelCase						)
     tokenizer.save_pretrained(self.tmpdirname						)



   def        __magic_name__    (							self							:     List[str]						):
     UpperCAmelCase       :							Dict  = NllbTokenizer(_UpperCamelCase,		keep_accents=_UpperCamelCase						)

     UpperCAmelCase       :							int  = tokenizer.tokenize('''This is a test'''						)
     self.assertListEqual(_UpperCamelCase,		['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']						)

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(_UpperCamelCase						),		[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]],		)

     UpperCAmelCase       :							Optional[Any]  = tokenizer.tokenize('''I was born in 92000, and this is falsé.'''						)
     self.assertListEqual(
         _UpperCamelCase,		[
             SPIECE_UNDERLINE + '''I''',
             SPIECE_UNDERLINE + '''was''',
             SPIECE_UNDERLINE + '''b''',
             '''or''',
             '''n''',
             SPIECE_UNDERLINE + '''in''',
             SPIECE_UNDERLINE + '''''',
             '''9''',
             '''2''',
             '''0''',
             '''0''',
             '''0''',
             ''',''',
             SPIECE_UNDERLINE + '''and''',
             SPIECE_UNDERLINE + '''this''',
             SPIECE_UNDERLINE + '''is''',
             SPIECE_UNDERLINE + '''f''',
             '''al''',
             '''s''',
             '''é''',
             '''.''',
         ],		)
     UpperCAmelCase       :							List[str]  = tokenizer.convert_tokens_to_ids(_UpperCamelCase						)
     self.assertListEqual(
         _UpperCamelCase,		[
             value + tokenizer.fairseq_offset
             for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
         ],		)

     UpperCAmelCase       :							str  = tokenizer.convert_ids_to_tokens(_UpperCamelCase						)
     self.assertListEqual(
         _UpperCamelCase,		[
             SPIECE_UNDERLINE + '''I''',
             SPIECE_UNDERLINE + '''was''',
             SPIECE_UNDERLINE + '''b''',
             '''or''',
             '''n''',
             SPIECE_UNDERLINE + '''in''',
             SPIECE_UNDERLINE + '''''',
             '''<unk>''',
             '''2''',
             '''0''',
             '''0''',
             '''0''',
             ''',''',
             SPIECE_UNDERLINE + '''and''',
             SPIECE_UNDERLINE + '''this''',
             SPIECE_UNDERLINE + '''is''',
             SPIECE_UNDERLINE + '''f''',
             '''al''',
             '''s''',
             '''<unk>''',
             '''.''',
         ],		)



   def        __magic_name__    (							self							:     List[str]						):
     UpperCAmelCase       :							List[str]  = (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         UpperCAmelCase       :							str  = self.rust_tokenizer_class.from_pretrained(_UpperCamelCase,		**_UpperCamelCase						)
         UpperCAmelCase       :							List[str]  = self.tokenizer_class.from_pretrained(_UpperCamelCase,		**_UpperCamelCase						)

         UpperCAmelCase       :							Any  = tempfile.mkdtemp()

         UpperCAmelCase       :							List[Any]  = tokenizer_r.save_pretrained(_UpperCamelCase						)
         UpperCAmelCase       :							Any  = tokenizer_p.save_pretrained(_UpperCamelCase						)

         # Checks it save with the same files + the tokenizer.json file for the fast one
         self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files						)						)
         UpperCAmelCase       :							Dict  = tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f						)
         self.assertSequenceEqual(_UpperCamelCase,		_UpperCamelCase						)

         # Checks everything loads correctly in the same way
         UpperCAmelCase       :							int  = tokenizer_r.from_pretrained(_UpperCamelCase						)
         UpperCAmelCase       :							Optional[int]  = tokenizer_p.from_pretrained(_UpperCamelCase						)

         # Check special tokens are set accordingly on Rust and Python
         for key in tokenizer_pp.special_tokens_map:
           self.assertTrue(hasattr(_UpperCamelCase,		_UpperCamelCase						)						)

         shutil.rmtree(_UpperCamelCase						)

         # Save tokenizer rust, legacy_format=True
         UpperCAmelCase       :							Optional[Any]  = tempfile.mkdtemp()

         UpperCAmelCase       :							Tuple  = tokenizer_r.save_pretrained(_UpperCamelCase,		legacy_format=_UpperCamelCase						)
         UpperCAmelCase       :							Optional[Any]  = tokenizer_p.save_pretrained(_UpperCamelCase						)

         # Checks it save with the same files
         self.assertSequenceEqual(_UpperCamelCase,		_UpperCamelCase						)

         # Checks everything loads correctly in the same way
         UpperCAmelCase       :							Optional[int]  = tokenizer_r.from_pretrained(_UpperCamelCase						)
         UpperCAmelCase       :							str  = tokenizer_p.from_pretrained(_UpperCamelCase						)

         # Check special tokens are set accordingly on Rust and Python
         for key in tokenizer_pp.special_tokens_map:
           self.assertTrue(hasattr(_UpperCamelCase,		_UpperCamelCase						)						)

         shutil.rmtree(_UpperCamelCase						)

         # Save tokenizer rust, legacy_format=False
         UpperCAmelCase       :							Tuple  = tempfile.mkdtemp()

         UpperCAmelCase       :							Dict  = tokenizer_r.save_pretrained(_UpperCamelCase,		legacy_format=_UpperCamelCase						)
         UpperCAmelCase       :							List[Any]  = tokenizer_p.save_pretrained(_UpperCamelCase						)

         # Checks it saved the tokenizer.json file
         self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files						)						)

         # Checks everything loads correctly in the same way
         UpperCAmelCase       :							Union[str, Any]  = tokenizer_r.from_pretrained(_UpperCamelCase						)
         UpperCAmelCase       :							Any  = tokenizer_p.from_pretrained(_UpperCamelCase						)

         # Check special tokens are set accordingly on Rust and Python
         for key in tokenizer_pp.special_tokens_map:
           self.assertTrue(hasattr(_UpperCamelCase,		_UpperCamelCase						)						)

         shutil.rmtree(_UpperCamelCase						)



   @require_torch
   def        __magic_name__    (							self							:     Any						):
     if not self.test_seqaseq:
       return

     UpperCAmelCase       :							int  = self.get_tokenizers()
     for tokenizer in tokenizers:
       with self.subTest(F'''{tokenizer.__class__.__name__}'''						):
         # Longer text that will definitely require truncation.
         UpperCAmelCase       :							str  = [
             ''' UN Chief Says There Is No Military Solution in Syria''',
             ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
             ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
             ''' will only worsen the violence and misery for millions of people.''',
         ]
         UpperCAmelCase       :							Any  = [
             '''Şeful ONU declară că nu există o soluţie militară în Siria''',
             '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
             ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
             ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
         ]
         try:
           UpperCAmelCase       :							int  = tokenizer.prepare_seqaseq_batch(
               src_texts=_UpperCamelCase,		tgt_texts=_UpperCamelCase,		max_length=3,		max_target_length=1_0,		return_tensors='''pt''',		src_lang='''eng_Latn''',		tgt_lang='''ron_Latn''',		)
         except NotImplementedError:
           return
         self.assertEqual(batch.input_ids.shape[1],		3						)
         self.assertEqual(batch.labels.shape[1],		1_0						)
         # max_target_length will default to max_length if not specified
         UpperCAmelCase       :							Tuple  = tokenizer.prepare_seqaseq_batch(
             _UpperCamelCase,		tgt_texts=_UpperCamelCase,		max_length=3,		return_tensors='''pt'''						)
         self.assertEqual(batch.input_ids.shape[1],		3						)
         self.assertEqual(batch.labels.shape[1],		3						)

         UpperCAmelCase       :							Dict  = tokenizer.prepare_seqaseq_batch(
             src_texts=_UpperCamelCase,		max_length=3,		max_target_length=1_0,		return_tensors='''pt'''						)
         self.assertEqual(batch_encoder_only.input_ids.shape[1],		3						)
         self.assertEqual(batch_encoder_only.attention_mask.shape[1],		3						)
         self.assertNotIn('''decoder_input_ids''',		_UpperCamelCase						)



   @unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''						)
   def        __magic_name__    (							self							:     List[str]						):
     pass





   def        __magic_name__    (							self							:     Any						):
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
         UpperCAmelCase       :							int  = [AddedToken('''<special>''',		lstrip=_UpperCamelCase						)]

         UpperCAmelCase       :							Union[str, Any]  = self.rust_tokenizer_class.from_pretrained(
             _UpperCamelCase,		additional_special_tokens=_UpperCamelCase,		**_UpperCamelCase						)
         UpperCAmelCase       :							List[Any]  = tokenizer_r.encode('''Hey this is a <special> token'''						)

         UpperCAmelCase       :							List[Any]  = tokenizer_r.encode('''<special>''',		add_special_tokens=_UpperCamelCase						)[0]

         self.assertTrue(special_token_id in r_output						)

         if self.test_slow_tokenizer:
           UpperCAmelCase       :							Tuple  = self.rust_tokenizer_class.from_pretrained(
               _UpperCamelCase,		additional_special_tokens=_UpperCamelCase,		**_UpperCamelCase,		)
           UpperCAmelCase       :							List[str]  = self.tokenizer_class.from_pretrained(
               _UpperCamelCase,		additional_special_tokens=_UpperCamelCase,		**_UpperCamelCase						)

           UpperCAmelCase       :							Tuple  = tokenizer_p.encode('''Hey this is a <special> token'''						)

           UpperCAmelCase       :							List[Any]  = tokenizer_cr.encode('''Hey this is a <special> token'''						)

           self.assertEqual(_UpperCamelCase,		_UpperCamelCase						)
           self.assertEqual(_UpperCamelCase,		_UpperCamelCase						)
           self.assertTrue(special_token_id in p_output						)
           self.assertTrue(special_token_id in cr_output						)






@require_torch
@require_sentencepiece
@require_tokenizers
class 					__UpperCAmelCase     (       unittest.TestCase						):
   UpperCamelCase        =     "facebook/nllb-200-distilled-600M"
   UpperCamelCase        =     [
       " UN Chief Says There Is No Military Solution in Syria",
       " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
   ]
   UpperCamelCase        =     [
       "Şeful ONU declară că nu există o soluţie militară în Siria",
       "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
       " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
       " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
   ]
   UpperCamelCase        =     [
       2_5_6_0_4_7,
       1_6_2_9_7,
       1_3_4_4_0_8,
       8_1_6_5,
       2_4_8_0_6_6,
       1_4_7_3_4,
       9_5_0,
       1_1_3_5,
       1_0_5_7_2_1,
       3_5_7_3,
       8_3,
       2_7_3_5_2,
       1_0_8,
       4_9_4_8_6,
       2,
   ]



   @classmethod
   def        __magic_name__    (							cls							:     Optional[int]						):
     UpperCAmelCase       :							Union[str, Any]  = NllbTokenizer.from_pretrained(
         cls.checkpoint_name,		src_lang='''eng_Latn''',		tgt_lang='''ron_Latn'''						)
     UpperCAmelCase       :							Union[str, Any]  = 1
     return cls



   def        __magic_name__    (							self							:     Optional[int]						):
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab'''],		2_5_6_0_0_1						)
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn'''],		2_5_6_0_0_2						)
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn'''],		2_5_6_0_5_7						)



   def        __magic_name__    (							self							:     str						):
     UpperCAmelCase       :							Any  = self.tokenizer.batch_encode_plus(self.src_text						).input_ids[0]
     self.assertListEqual(self.expected_src_tokens,		_UpperCamelCase						)



   def        __magic_name__    (							self							:     Any						):
     self.assertIn(_UpperCamelCase,		self.tokenizer.all_special_ids						)
     # fmt: off
     UpperCAmelCase       :							Tuple  = [RO_CODE, 4_2_5_4, 9_8_0_6_8, 1_1_2_9_2_3, 3_9_0_7_2, 3_9_0_9, 7_1_3, 1_0_2_7_6_7, 2_6, 1_7_3_1_4, 3_5_6_4_2, 1_4_6_8_3, 3_3_1_1_8, 2_0_2_2, 6_6_9_8_7, 2, 2_5_6_0_4_7]
     # fmt: on

     UpperCAmelCase       :							int  = self.tokenizer.decode(_UpperCamelCase,		skip_special_tokens=_UpperCamelCase						)
     UpperCAmelCase       :							int  = self.tokenizer.decode(generated_ids[1:],		skip_special_tokens=_UpperCamelCase						)
     self.assertEqual(_UpperCamelCase,		_UpperCamelCase						)
     self.assertNotIn(self.tokenizer.eos_token,		_UpperCamelCase						)



   def        __magic_name__    (							self							:     Dict						):
     UpperCAmelCase       :							int  = ['''this is gunna be a long sentence ''' * 2_0]
     assert isinstance(src_text[0],		_UpperCamelCase						)
     UpperCAmelCase       :							List[Any]  = 1_0
     UpperCAmelCase       :							Optional[Any]  = self.tokenizer(_UpperCamelCase,		max_length=_UpperCamelCase,		truncation=_UpperCamelCase						).input_ids[0]
     self.assertEqual(ids[-1],		2						)
     self.assertEqual(ids[0],		_UpperCamelCase						)
     self.assertEqual(len(_UpperCamelCase						),		_UpperCamelCase						)



   def        __magic_name__    (							self							:     Tuple						):
     self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']						),		[2_5_6_2_0_3, 3]						)



   def        __magic_name__    (							self							:     Dict						):
     UpperCAmelCase       :							Optional[int]  = tempfile.mkdtemp()
     UpperCAmelCase       :							Optional[Any]  = self.tokenizer.fairseq_tokens_to_ids
     self.tokenizer.save_pretrained(_UpperCamelCase						)
     UpperCAmelCase       :							Union[str, Any]  = NllbTokenizer.from_pretrained(_UpperCamelCase						)
     self.assertDictEqual(new_tok.fairseq_tokens_to_ids,		_UpperCamelCase						)



   @require_torch
   def        __magic_name__    (							self							:     str						):
     UpperCAmelCase       :							Union[str, Any]  = self.tokenizer(
         self.src_text,		text_target=self.tgt_text,		padding=_UpperCamelCase,		truncation=_UpperCamelCase,		max_length=len(self.expected_src_tokens						),		return_tensors='''pt''',		)
     UpperCAmelCase       :							int  = shift_tokens_right(
         batch['''labels'''],		self.tokenizer.pad_token_id,		self.tokenizer.lang_code_to_id['''ron_Latn''']						)

     self.assertIsInstance(_UpperCamelCase,		_UpperCamelCase						)

     self.assertEqual((2, 1_5),		batch.input_ids.shape						)
     self.assertEqual((2, 1_5),		batch.attention_mask.shape						)
     UpperCAmelCase       :							Optional[int]  = batch.input_ids.tolist()[0]
     self.assertListEqual(self.expected_src_tokens,		_UpperCamelCase						)
     self.assertEqual(_UpperCamelCase,		batch.decoder_input_ids[0, 0]						)  # EOS
     # Test that special tokens are reset
     self.assertEqual(self.tokenizer.prefix_tokens,		[EN_CODE]						)
     self.assertEqual(self.tokenizer.suffix_tokens,		[self.tokenizer.eos_token_id]						)



   def        __magic_name__    (							self							:     List[Any]						):
     UpperCAmelCase       :							int  = self.tokenizer(self.src_text,		padding=_UpperCamelCase,		truncation=_UpperCamelCase,		max_length=3,		return_tensors='''pt'''						)
     UpperCAmelCase       :							Dict  = self.tokenizer(
         text_target=self.tgt_text,		padding=_UpperCamelCase,		truncation=_UpperCamelCase,		max_length=1_0,		return_tensors='''pt'''						)
     UpperCAmelCase       :							Optional[int]  = targets['''input_ids''']
     UpperCAmelCase       :							Dict  = shift_tokens_right(
         _UpperCamelCase,		self.tokenizer.pad_token_id,		decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang],		)

     self.assertEqual(batch.input_ids.shape[1],		3						)
     self.assertEqual(batch.decoder_input_ids.shape[1],		1_0						)



   @require_torch
   def        __magic_name__    (							self							:     Optional[int]						):
     UpperCAmelCase       :							int  = self.tokenizer._build_translation_inputs(
         '''A test''',		return_tensors='''pt''',		src_lang='''eng_Latn''',		tgt_lang='''fra_Latn'''						)

     self.assertEqual(
         nested_simplify(_UpperCamelCase						),		{
             # A, test, EOS, en_XX
             '''input_ids''': [[2_5_6_0_4_7, 7_0, 7_3_5_6, 2]],
             '''attention_mask''': [[1, 1, 1, 1]],
             # ar_AR
             '''forced_bos_token_id''': 2_5_6_0_5_7,
         },		)





   @require_torch
   def        __magic_name__    (							self							:     List[str]						):
     UpperCAmelCase       :							Any  = True
     UpperCAmelCase       :							int  = self.tokenizer(
         '''UN Chief says there is no military solution in Syria''',		src_lang='''eng_Latn''',		tgt_lang='''fra_Latn'''						)
     self.assertEqual(
         inputs.input_ids,		[1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2, 2_5_6_0_4_7]						)

     UpperCAmelCase       :							List[str]  = False
     UpperCAmelCase       :							Dict  = self.tokenizer(
         '''UN Chief says there is no military solution in Syria''',		src_lang='''eng_Latn''',		tgt_lang='''fra_Latn'''						)
     self.assertEqual(
         inputs.input_ids,		[2_5_6_0_4_7, 1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2]						)








import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_       	=	logging.get_logger(__name__)

lowerCAmelCase_       	=	{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase_       	=	{
    '''vocab_file''': {
        '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model''',
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'''
        ),
    }
}

lowerCAmelCase_       	=	{
    '''google/bigbird-roberta-base''': 40_96,
    '''google/bigbird-roberta-large''': 40_96,
    '''google/bigbird-base-trivia-itc''': 40_96,
}





class 	snake_case_   (		__A					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   List[Any]									=				VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE     :   List[Any]									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE     :   List[Any]									=				["input_ids", "attention_mask"]
			SCREAMING_SNAKE_CASE     :   List[int]									=				[]







			def __init__(     self :				List[str]							,   _UpperCamelCase :				List[str]							,   _UpperCamelCase :				Dict="<unk>"							,   _UpperCamelCase :				List[str]="<s>"							,   _UpperCamelCase :				Tuple="</s>"							,   _UpperCamelCase :				Any="<pad>"							,   _UpperCamelCase :				Any="[SEP]"							,   _UpperCamelCase :				Optional[Any]="[MASK]"							,   _UpperCamelCase :				Any="[CLS]"							,   _UpperCamelCase :				Optional[Dict[str, Any]] = None							,   **_UpperCamelCase :				Dict							,   )      ->None:
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else bos_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else eos_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else unk_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else pad_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else cls_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else sep_token

									# Mask token behave like a normal word, i.e. include the space before it
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else mask_token

									snake_case_   =	{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    bos_token=_UpperCamelCase							,   eos_token=_UpperCamelCase							,   unk_token=_UpperCamelCase							,   pad_token=_UpperCamelCase							,   sep_token=_UpperCamelCase							,   mask_token=_UpperCamelCase							,   cls_token=_UpperCamelCase							,   sp_model_kwargs=self.sp_model_kwargs							,   **_UpperCamelCase							,   )

									snake_case_   =	vocab_file

									snake_case_   =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
									self.sp_model.Load(_UpperCamelCase							)







			@property
			def 				snake_case__(     self :				str							)      ->List[Any]:
									return self.sp_model.get_piece_size()







			def 				snake_case__(     self :				int							)      ->Union[str, Any]:
									snake_case_   =	{self.convert_ids_to_tokens(_UpperCamelCase							): i for i in range(self.vocab_size							)}
									vocab.update(self.added_tokens_encoder							)
									return vocab







			def __getstate__(     self :				Tuple							)      ->Any:
									snake_case_   =	self.__dict__.copy()
									snake_case_   =	None
									return state







			def __setstate__(     self :				str							,   _UpperCamelCase :				List[Any]							)      ->List[str]:
									snake_case_   =	d

									# for backward compatibility
									if not hasattr(self							,   '''sp_model_kwargs'''							):
															snake_case_   =	{}

									snake_case_   =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
									self.sp_model.Load(self.vocab_file							)







			def 				snake_case__(     self :				Optional[int]							,   _UpperCamelCase :				str							)      ->List[str]:
									return self.sp_model.encode(_UpperCamelCase							,   out_type=_UpperCamelCase							)







			def 				snake_case__(     self :				str							,   _UpperCamelCase :				List[str]							)      ->Tuple:
									return self.sp_model.piece_to_id(_UpperCamelCase							)







			def 				snake_case__(     self :				Union[str, Any]							,   _UpperCamelCase :				str							)      ->List[Any]:
									snake_case_   =	self.sp_model.IdToPiece(_UpperCamelCase							)
									return token







			def 				snake_case__(     self :				Dict							,   _UpperCamelCase :				Optional[int]							)      ->List[str]:
									snake_case_   =	[]
									snake_case_   =	''''''
									snake_case_   =	False
									for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																					if not prev_is_special:
																											out_string += " "
																					out_string += self.sp_model.decode(_UpperCamelCase							) + token
																					snake_case_   =	True
																					snake_case_   =	[]
															else:
																					current_sub_tokens.append(_UpperCamelCase							)
																					snake_case_   =	False
									out_string += self.sp_model.decode(_UpperCamelCase							)
									return out_string.strip()







			def 				snake_case__(     self :				int							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				bool = False							,   _UpperCamelCase :				bool = None							,   _UpperCamelCase :				bool = True							,   **_UpperCamelCase :				List[str]							,   )      ->str:
									snake_case_   =	kwargs.pop('''use_source_tokenizer'''							,   _UpperCamelCase							)

									snake_case_   =	self.convert_ids_to_tokens(_UpperCamelCase							,   skip_special_tokens=_UpperCamelCase							)

									# To avoid mixing byte-level and unicode for byte-level BPT
									# we need to build string separately for added tokens and byte-level tokens
									# cf. https://github.com/huggingface/transformers/issues/1133
									snake_case_   =	[]
									snake_case_   =	[]
									for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																					continue
															if token in self.added_tokens_encoder:
																					if current_sub_text:
																											sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase							)							)
																											snake_case_   =	[]
																					sub_texts.append(_UpperCamelCase							)
															else:
																					current_sub_text.append(_UpperCamelCase							)
									if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase							)							)

									# Mimic the behavior of the Rust tokenizer:
									# No space before [MASK] and [SEP]
									if spaces_between_special_tokens:
															snake_case_   =	re.sub(R''' (\[(MASK|SEP)\])'''							,   R'''\1'''							,   ''' '''.join(_UpperCamelCase							)							)
									else:
															snake_case_   =	''''''.join(_UpperCamelCase							)

									snake_case_   =	(
									    clean_up_tokenization_spaces
									    if clean_up_tokenization_spaces is not None
									    else self.clean_up_tokenization_spaces
									)
									if clean_up_tokenization_spaces:
															snake_case_   =	self.clean_up_tokenization(_UpperCamelCase							)
															return clean_text
									else:
															return text







			def 				snake_case__(     self :				List[Any]							,   _UpperCamelCase :				str							,   _UpperCamelCase :				Optional[str] = None							)      ->Tuple[str]:
									if not os.path.isdir(_UpperCamelCase							):
															logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
															return
									snake_case_   =	os.path.join(
									    _UpperCamelCase							,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

									if os.path.abspath(self.vocab_file							) != os.path.abspath(_UpperCamelCase							) and os.path.isfile(self.vocab_file							):
															copyfile(self.vocab_file							,   _UpperCamelCase							)
									elif not os.path.isfile(self.vocab_file							):
															with open(_UpperCamelCase							,   '''wb'''							) as fi:
																					snake_case_   =	self.sp_model.serialized_model_proto()
																					fi.write(_UpperCamelCase							)

									return (out_vocab_file,)







			def 				snake_case__(     self :				Tuple							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							)      ->List[int]:
									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									snake_case_   =	[self.cls_token_id]
									snake_case_   =	[self.sep_token_id]
									return cls + token_ids_a + sep + token_ids_a + sep







			def 				snake_case__(     self :				List[str]							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							,   _UpperCamelCase :				bool = False							)      ->List[int]:
									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=_UpperCamelCase							,   token_ids_a=_UpperCamelCase							,   already_has_special_tokens=_UpperCamelCase							)

									if token_ids_a is None:
															return [1] + ([0] * len(_UpperCamelCase							)) + [1]
									return [1] + ([0] * len(_UpperCamelCase							)) + [1] + ([0] * len(_UpperCamelCase							)) + [1]







			def 				snake_case__(     self :				List[Any]							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							)      ->List[int]:
									snake_case_   =	[self.sep_token_id]
									snake_case_   =	[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep							) * [0]
									return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]