


"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation

lowerCamelCase_   :   Tuple									=  logging.get_logger(__name__)


lowerCamelCase_   :   List[str]									=  {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

lowerCamelCase_   :   Any									=  {
    """vocab_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"""},
    """merges_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"""},
    """tokenizer_config_file""": {
        """facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"""
    },
}

lowerCamelCase_   :   Any									=  {"""facebook/blenderbot-3B""": 1_2_8}
@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def _A						(						):







    """simple docstring"""
    a 		=(
        list(range(ord('''!'''		)       ,    ord('''~'''		) + 1		)		) + list(range(ord('''¡'''		)       ,    ord('''¬'''		) + 1		)		) + list(range(ord('''®'''		)       ,    ord('''ÿ'''		) + 1		)		)
    )
    a 		=bs[:]
    a 		=0
    for b in range(2**8		):
        if b not in bs:
            bs.append(lowercase		)
            cs.append(2**8 + n		)
            n += 1
    a 		=[chr(lowercase		) for n in cs]
    return dict(zip(lowercase       ,    lowercase		)		)
def _A						(						lowercase		):







    """simple docstring"""
    a 		=set()
    a 		=word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)		)
        a 		=char
    return pairs






class       __A				(       _SCREAMING_SNAKE_CASE			):






 """simple docstring"""




 __lowerCAmelCase 					=  VOCAB_FILES_NAMES
 __lowerCAmelCase 					=  PRETRAINED_VOCAB_FILES_MAP
 __lowerCAmelCase 					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCAmelCase 					=  ["input_ids", "attention_mask"]


 def __init__(		self ,    __A ,    __A ,    __A="replace" ,    __A="<s>" ,    __A="</s>" ,    __A="</s>" ,    __A="<s>" ,    __A="<unk>" ,    __A="<pad>" ,    __A="<mask>" ,    __A=False ,    **__A ,    )					->     Union[str, Any]:
     a 		=AddedToken(__A ,    lstrip=__A ,    rstrip=__A						) if isinstance(__A ,    __A						) else bos_token
     a 		=AddedToken(__A ,    lstrip=__A ,    rstrip=__A						) if isinstance(__A ,    __A						) else eos_token
     a 		=AddedToken(__A ,    lstrip=__A ,    rstrip=__A						) if isinstance(__A ,    __A						) else sep_token
     a 		=AddedToken(__A ,    lstrip=__A ,    rstrip=__A						) if isinstance(__A ,    __A						) else cls_token
     a 		=AddedToken(__A ,    lstrip=__A ,    rstrip=__A						) if isinstance(__A ,    __A						) else unk_token
     a 		=AddedToken(__A ,    lstrip=__A ,    rstrip=__A						) if isinstance(__A ,    __A						) else pad_token

     # Mask token behave like a normal word, i.e. include the space before it
     a 		=AddedToken(__A ,    lstrip=__A ,    rstrip=__A						) if isinstance(__A ,    __A						) else mask_token

     super().__init__(
         errors=__A ,    bos_token=__A ,    eos_token=__A ,    unk_token=__A ,    sep_token=__A ,    cls_token=__A ,    pad_token=__A ,    mask_token=__A ,    add_prefix_space=__A ,    **__A ,    )

     with open(__A ,    encoding='''utf-8'''						) as vocab_handle:
         a 		=json.load(__A						)
     a 		={v: k for k, v in self.encoder.items()}
     a 		=errors  # how to handle errors in decoding
     a 		=bytes_to_unicode()
     a 		={v: k for k, v in self.byte_encoder.items()}
     with open(__A ,    encoding='''utf-8'''						) as merges_handle:
         a 		=merges_handle.read().split('''\n'''						)[1:-1]
     a 		=[tuple(merge.split()						) for merge in bpe_merges]
     a 		=dict(zip(__A ,    range(len(__A						)						)						)						)
     a 		={}
     a 		=add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     a 		=re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)


 @property
 # Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
 def       SCREAMING_SNAKE_CASE							(		self						)					->     List[Any]:
     return len(self.encoder						)


 def       SCREAMING_SNAKE_CASE							(		self						)					->     List[Any]:
     return dict(self.encoder ,    **self.added_tokens_encoder						)


 def       SCREAMING_SNAKE_CASE							(		self ,    __A						)					->     Optional[Any]:
     if token in self.cache:
         return self.cache[token]
     a 		=tuple(__A						)
     a 		=get_pairs(__A						)

     if not pairs:
         return token

     while True:
         a 		=min(__A ,    key=lambda __A						: self.bpe_ranks.get(__A ,    float('''inf'''						)						)						)
         if bigram not in self.bpe_ranks:
             break
         a		,      a 		=bigram
         a 		=[]
         a 		=0
         while i < len(__A						):
             try:
                 a 		=word.index(__A ,    __A						)
             except ValueError:
                 new_word.extend(word[i:]						)
                 break
             else:
                 new_word.extend(word[i:j]						)
                 a 		=j

             if word[i] == first and i < len(__A						) - 1 and word[i + 1] == second:
                 new_word.append(first + second						)
                 i += 2
             else:
                 new_word.append(word[i]						)
                 i += 1
         a 		=tuple(__A						)
         a 		=new_word
         if len(__A						) == 1:
             break
         else:
             a 		=get_pairs(__A						)
     a 		=''' '''.join(__A						)
     a 		=word
     return word


 def       SCREAMING_SNAKE_CASE							(		self ,    __A						)					->     int:
     a 		=[]
     for token in re.findall(self.pat ,    __A						):
         a 		=''''''.join(
             self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
         bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A						).split(''' '''						)						)
     return bpe_tokens


 def       SCREAMING_SNAKE_CASE							(		self ,    __A						)					->     Tuple:
     return self.encoder.get(__A ,    self.encoder.get(self.unk_token						)						)


 def       SCREAMING_SNAKE_CASE							(		self ,    __A						)					->     List[str]:
     return self.decoder.get(__A						)


 def       SCREAMING_SNAKE_CASE							(		self ,    __A						)					->     Dict:
     a 		=''''''.join(__A						)
     a 		=bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8''' ,    errors=self.errors						)
     return text


 def       SCREAMING_SNAKE_CASE							(		self ,    __A ,    __A = None						)					->     Tuple[str]:
     if not os.path.isdir(__A						):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
         return
     a 		=os.path.join(
         __A ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
     a 		=os.path.join(
         __A ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

     with open(__A ,    '''w''' ,    encoding='''utf-8'''						) as f:
         f.write(json.dumps(self.encoder ,    indent=2 ,    sort_keys=__A ,    ensure_ascii=__A						) + '''\n'''						)

     a 		=0
     with open(__A ,    '''w''' ,    encoding='''utf-8'''						) as writer:
         writer.write('''#version: 0.2\n'''						)
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,    key=lambda __A						: kv[1]						):
             if index != token_index:
                 logger.warning(
                     f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                     ''' Please check that the tokenizer is not corrupted!'''						)
                 a 		=token_index
             writer.write(''' '''.join(__A						) + '''\n'''						)
             index += 1

     return vocab_file, merge_file


 def       SCREAMING_SNAKE_CASE							(		self ,    __A ,    __A = None ,    __A = False						)					->     List[int]:
     if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=__A ,    token_ids_a=__A ,    already_has_special_tokens=__A						)

     if token_ids_a is None:
         return [1] + ([0] * len(__A						)) + [1]
     return [1] + ([0] * len(__A						)) + [1, 1] + ([0] * len(__A						)) + [1]


 def       SCREAMING_SNAKE_CASE							(		self ,    __A ,    __A = None						)					->     List[int]:
     a 		=[self.sep_token_id]
     a 		=[self.cls_token_id]

     if token_ids_a is None:
         return len(cls + token_ids_a + sep						) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]


 def       SCREAMING_SNAKE_CASE							(		self ,    __A ,    __A=False ,    **__A						)					->     Union[str, Any]:
     a 		=kwargs.pop('''add_prefix_space''' ,    self.add_prefix_space						)
     if (is_split_into_words or add_prefix_space) and (len(__A						) > 0 and not text[0].isspace()):
         a 		=''' ''' + text
     return (text, kwargs)


 def       SCREAMING_SNAKE_CASE							(		self ,    __A ,    __A = None						)					->     str:
     return token_ids_a + [self.eos_token_id]





 def       SCREAMING_SNAKE_CASE							(		self ,    __A						)					->     List[int]:
     a 		=[]
     for is_user, text in conversation.iter_texts():
         if is_user:
             # We need to space prefix as it's being done within blenderbot
             inputs.append(''' ''' + text						)
         else:
             # Generated responses should contain them already.
             inputs.append(__A						)

     a 		='''  '''.join(__A						)
     a 		=self.encode(__A						)
     if len(__A						) > self.model_max_length:
         a 		=input_ids[-self.model_max_length :]
         logger.warning(f'''Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'''						)
     return input_ids







'''simple docstring'''







import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
			_A	:			List[Any]         ='''pt'''
elif is_tf_available():
			_A	:			Any         ='''tf'''
else:
			_A	:			List[str]         ='''jax'''

class _lowercase	(				_lowercase				,      unittest.TestCase		):
							a				      =       ByTaTokenizer
							a				      =       False




							def 		lowerCamelCase_				(							self:				str     ):
														super().setUp()
														lowerCamelCase__					:	str         =       ByTaTokenizer()
														tokenizer.save_pretrained(self.tmpdirname     )




							@cached_property
							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														return ByTaTokenizer.from_pretrained("""google/byt5-small"""     )




							def 		lowerCamelCase_				(							self:				Any			,							**UpperCamelCase__:				Tuple     ):
														return self.tokenizer_class.from_pretrained(self.tmpdirname			,							**UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Union[str, Any]			,							UpperCamelCase__:				str			,							UpperCamelCase__:				Any=False			,							UpperCamelCase__:				Union[str, Any]=20			,							UpperCamelCase__:				Optional[int]=5     ):
														# XXX The default common tokenizer tests assume that every ID is decodable on its own.
														# This assumption is invalid for ByT5 because single bytes might not be
														# valid utf-8 (byte 128 for instance).
														# Here we're overriding the smallest possible method to provide
														# a clean sequence without making the same assumption.

														lowerCamelCase__					:	List[str]         =       []
														for i in range(len(UpperCamelCase__     )     ):
																					try:
																												lowerCamelCase__					:	Tuple         =       tokenizer.decode([i]			,							clean_up_tokenization_spaces=UpperCamelCase__     )
																					except UnicodeDecodeError:
																												pass
																					toks.append((i, tok)     )

														lowerCamelCase__					:	Union[str, Any]         =       list(filter(lambda UpperCamelCase__     : re.match(R"""^[ a-zA-Z]+$"""			,							t[1]     )			,							UpperCamelCase__     )     )
														lowerCamelCase__					:	Tuple         =       list(filter(lambda UpperCamelCase__     : [t[0]] == tokenizer.encode(t[1]			,							add_special_tokens=UpperCamelCase__     )			,							UpperCamelCase__     )     )
														if max_length is not None and len(UpperCamelCase__     ) > max_length:
																					lowerCamelCase__					:	Dict         =       toks[:max_length]
														if min_length is not None and len(UpperCamelCase__     ) < min_length and len(UpperCamelCase__     ) > 0:
																					while len(UpperCamelCase__     ) < min_length:
																												lowerCamelCase__					:	Optional[Any]         =       toks + toks
        # toks_str = [t[1] for t in toks]
														lowerCamelCase__					:	Union[str, Any]         =       [t[0] for t in toks]

														# Ensure consistency
														lowerCamelCase__					:	Union[str, Any]         =       tokenizer.decode(UpperCamelCase__			,							clean_up_tokenization_spaces=UpperCamelCase__     )
														if " " not in output_txt and len(UpperCamelCase__     ) > 1:
																					lowerCamelCase__					:	Optional[int]         =       (
																					    tokenizer.decode([toks_ids[0]]			,							clean_up_tokenization_spaces=UpperCamelCase__     )
																					    + """ """
																					    + tokenizer.decode(toks_ids[1:]			,							clean_up_tokenization_spaces=UpperCamelCase__     )
																					)
														if with_prefix_space:
																					lowerCamelCase__					:	str         =       """ """ + output_txt
														lowerCamelCase__					:	List[Any]         =       tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
														return output_txt, output_ids




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	str         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Union[str, Any]         =       tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]     )
														lowerCamelCase__					:	Optional[int]         =       tokenizer(["""hi""", """I went to the gym""", """"""]     )
														self.assertListEqual(batch_with_eos_added["""input_ids"""]			,							batch_without_eos_added["""input_ids"""]     )




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	Optional[Any]         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Dict         =       """Unicode €."""
														lowerCamelCase__					:	List[Any]         =       tokenizer(UpperCamelCase__     )
														lowerCamelCase__					:	List[str]         =       [88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
														self.assertEqual(encoded["""input_ids"""]			,							UpperCamelCase__     )

														# decoding
														lowerCamelCase__					:	Tuple         =       tokenizer.decode(UpperCamelCase__     )
														self.assertEqual(UpperCamelCase__			,							"""Unicode €.</s>"""     )

														lowerCamelCase__					:	List[Any]         =       tokenizer("""e è é ê ë"""     )
														lowerCamelCase__					:	List[Any]         =       [104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
														self.assertEqual(encoded["""input_ids"""]			,							UpperCamelCase__     )
														# decoding
														lowerCamelCase__					:	str         =       tokenizer.decode(UpperCamelCase__     )
														self.assertEqual(UpperCamelCase__			,							"""e è é ê ë</s>"""     )

														# encode/decode, but with `encode` instead of `__call__`
														self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""     )     )			,							"""e è é ê ë</s>"""     )




							def 		lowerCamelCase_				(							self:				Any     ):
														lowerCamelCase__					:	int         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Tuple         =       ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
														# fmt: off
														lowerCamelCase__					:	List[str]         =       [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
														# fmt: on
														lowerCamelCase__					:	int         =       tokenizer(UpperCamelCase__			,							padding=UpperCamelCase__			,							return_tensors=UpperCamelCase__     )
														self.assertIsInstance(UpperCamelCase__			,							UpperCamelCase__     )

														if FRAMEWORK != "jax":
																					lowerCamelCase__					:	Any         =       list(batch.input_ids.numpy()[0]     )
														else:
																					lowerCamelCase__					:	str         =       list(batch.input_ids.tolist()[0]     )

														self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )

														self.assertEqual((2, 37)			,							batch.input_ids.shape     )
														self.assertEqual((2, 37)			,							batch.attention_mask.shape     )




							def 		lowerCamelCase_				(							self:				List[Any]     ):
														lowerCamelCase__					:	List[str]         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Tuple         =       ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
														lowerCamelCase__					:	Tuple         =       tokenizer(UpperCamelCase__			,							padding=UpperCamelCase__			,							return_tensors=UpperCamelCase__     )
														# check if input_ids are returned and no decoder_input_ids
														self.assertIn("""input_ids"""			,							UpperCamelCase__     )
														self.assertIn("""attention_mask"""			,							UpperCamelCase__     )
														self.assertNotIn("""decoder_input_ids"""			,							UpperCamelCase__     )
														self.assertNotIn("""decoder_attention_mask"""			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	str         =       self.ta_base_tokenizer
														lowerCamelCase__					:	List[Any]         =       [
														    """Summary of the text.""",
														    """Another summary.""",
														]
														lowerCamelCase__					:	Union[str, Any]         =       tokenizer(
														    text_target=UpperCamelCase__			,							max_length=32			,							padding="""max_length"""			,							truncation=UpperCamelCase__			,							return_tensors=UpperCamelCase__     )
														self.assertEqual(32			,							targets["""input_ids"""].shape[1]     )




							def 		lowerCamelCase_				(							self:				str     ):
														lowerCamelCase__					:	Tuple         =       self.ta_base_tokenizer
														lowerCamelCase__					:	str         =       ["""A long paragraph for summarization. </s>"""]
														lowerCamelCase__					:	Optional[Any]         =       ["""Summary of the text. </s>"""]
														# fmt: off
														lowerCamelCase__					:	Tuple         =       [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
														lowerCamelCase__					:	Any         =       [86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
														# fmt: on

														lowerCamelCase__					:	Any         =       tokenizer(UpperCamelCase__			,							text_target=UpperCamelCase__     )

														self.assertEqual(UpperCamelCase__			,							batch["""input_ids"""][0]     )
														self.assertEqual(UpperCamelCase__			,							batch["""labels"""][0]     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														# safety check on max_len default value so we are sure the test works
														lowerCamelCase__					:	Any         =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												self.assertNotEqual(tokenizer.model_max_length			,							42     )

        # Now let's start the test
														lowerCamelCase__					:	Any         =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												# Isolate this from the other tests because we save additional tokens/etc
																												lowerCamelCase__					:	int         =       tempfile.mkdtemp()

																												lowerCamelCase__					:	List[str]         =       """ He is very happy, UNwant\u00E9d,running"""
																												lowerCamelCase__					:	List[Any]         =       tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												tokenizer.save_pretrained(UpperCamelCase__     )

																												lowerCamelCase__					:	List[Any]         =       tokenizer.__class__.from_pretrained(UpperCamelCase__     )
																												lowerCamelCase__					:	Optional[int]         =       after_tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )

																												shutil.rmtree(UpperCamelCase__     )

														lowerCamelCase__					:	Any         =       self.get_tokenizers(model_max_length=42     )
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												# Isolate this from the other tests because we save additional tokens/etc
																												lowerCamelCase__					:	Any         =       tempfile.mkdtemp()

																												lowerCamelCase__					:	Optional[Any]         =       """ He is very happy, UNwant\u00E9d,running"""
																												tokenizer.add_tokens(["""bim""", """bambam"""]     )
																												lowerCamelCase__					:	Optional[int]         =       tokenizer.additional_special_tokens
																												additional_special_tokens.append("""new_additional_special_token"""     )
																												tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}     )
																												lowerCamelCase__					:	List[Any]         =       tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												tokenizer.save_pretrained(UpperCamelCase__     )

																												lowerCamelCase__					:	Union[str, Any]         =       tokenizer.__class__.from_pretrained(UpperCamelCase__     )
																												lowerCamelCase__					:	int         =       after_tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )
																												self.assertIn("""new_additional_special_token"""			,							after_tokenizer.additional_special_tokens     )
																												self.assertEqual(after_tokenizer.model_max_length			,							42     )

																												lowerCamelCase__					:	Any         =       tokenizer.__class__.from_pretrained(UpperCamelCase__			,							model_max_length=43     )
																												self.assertEqual(tokenizer.model_max_length			,							43     )

																												shutil.rmtree(UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														lowerCamelCase__					:	List[Any]         =       []
														if self.test_slow_tokenizer:
																					tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

														if self.test_rust_tokenizer:
																					tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

														for tokenizer_class, tokenizer_utils in tokenizer_list:
																					with tempfile.TemporaryDirectory() as tmp_dir:
																												tokenizer_utils.save_pretrained(UpperCamelCase__     )

																												with open(os.path.join(UpperCamelCase__			,							"""special_tokens_map.json"""     )			,							encoding="""utf-8"""     ) as json_file:
																																			lowerCamelCase__					:	Union[str, Any]         =       json.load(UpperCamelCase__     )

																												with open(os.path.join(UpperCamelCase__			,							"""tokenizer_config.json"""     )			,							encoding="""utf-8"""     ) as json_file:
																																			lowerCamelCase__					:	Optional[Any]         =       json.load(UpperCamelCase__     )

																												lowerCamelCase__					:	Optional[int]         =       [F'''<extra_id_{i}>''' for i in range(125     )]

																												lowerCamelCase__					:	int         =       added_tokens_extra_ids + [
																												    """an_additional_special_token"""
																												]
																												lowerCamelCase__					:	Optional[Any]         =       added_tokens_extra_ids + [
																												    """an_additional_special_token"""
																												]

																												with open(os.path.join(UpperCamelCase__			,							"""special_tokens_map.json"""     )			,							"""w"""			,							encoding="""utf-8"""     ) as outfile:
																																			json.dump(UpperCamelCase__			,							UpperCamelCase__     )
																												with open(os.path.join(UpperCamelCase__			,							"""tokenizer_config.json"""     )			,							"""w"""			,							encoding="""utf-8"""     ) as outfile:
																																			json.dump(UpperCamelCase__			,							UpperCamelCase__     )

																												# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																												# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																												# "special_tokens_map.json" files
																												lowerCamelCase__					:	Dict         =       tokenizer_class.from_pretrained(
																												    UpperCamelCase__			,							)
																												self.assertIn(
																												    """an_additional_special_token"""			,							tokenizer_without_change_in_init.additional_special_tokens     )
																												# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																												self.assertEqual(
																												    ["""an_additional_special_token"""]			,							tokenizer_without_change_in_init.convert_ids_to_tokens(
																												        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]     )     )			,							)

																												# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																												lowerCamelCase__					:	Optional[Any]         =       added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""			,							lstrip=UpperCamelCase__     )]
																												lowerCamelCase__					:	Any         =       tokenizer_class.from_pretrained(
																												    UpperCamelCase__			,							additional_special_tokens=UpperCamelCase__			,							)

																												self.assertIn("""a_new_additional_special_token"""			,							tokenizer.additional_special_tokens     )
																												self.assertEqual(
																												    ["""a_new_additional_special_token"""]			,							tokenizer.convert_ids_to_tokens(
																												        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]     )     )			,							)




							def 		lowerCamelCase_				(							self:				Union[str, Any]     ):
														lowerCamelCase__					:	Dict         =       []
														if self.test_slow_tokenizer:
																					tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

														if self.test_rust_tokenizer:
																					tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

														for tokenizer_class, tokenizer_utils in tokenizer_list:
																					with tempfile.TemporaryDirectory() as tmp_dir:
																												tokenizer_utils.save_pretrained(UpperCamelCase__     )

																												lowerCamelCase__					:	str         =       tokenizer_class.from_pretrained(UpperCamelCase__     )

																												self.assertTrue(tokenizer.decode([255]     ) == """"""     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														pass




							def 		lowerCamelCase_				(							self:				str     ):
														pass




							def 		lowerCamelCase_				(							self:				List[str]     ):
														pass




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														pass




							def 		lowerCamelCase_				(							self:				int     ):
														# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
														# and special added tokens as tokens
														lowerCamelCase__					:	Dict         =       self.get_tokenizers(fast=UpperCamelCase__			,							do_lower_case=UpperCamelCase__     )
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												lowerCamelCase__					:	Dict         =       ["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
																												lowerCamelCase__					:	Optional[int]         =       tokenizer.convert_tokens_to_string(UpperCamelCase__     )

																												self.assertIsInstance(UpperCamelCase__			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Any     ):
														lowerCamelCase__					:	str         =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												lowerCamelCase__					:	str         =       [
																												    """bos_token""",
																												    """eos_token""",
																												    """unk_token""",
																												    """sep_token""",
																												    """pad_token""",
																												    """cls_token""",
																												    """mask_token""",
																												]

																												lowerCamelCase__					:	str         =       0
																												lowerCamelCase__					:	Any         =       tokenizer.convert_ids_to_tokens(
																												    UpperCamelCase__			,							skip_special_tokens=UpperCamelCase__     )

																												for attr in attributes_list:
																																			setattr(UpperCamelCase__			,							attr + """_id"""			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							UpperCamelCase__     )			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							attr + """_id"""     )			,							UpperCamelCase__     )

																																			setattr(UpperCamelCase__			,							attr + """_id"""			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							UpperCamelCase__     )			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							attr + """_id"""     )			,							UpperCamelCase__     )

																												setattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""			,							[]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens"""     )			,							[]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""     )			,							[]     )

																												setattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""			,							[token_id_to_test_setters]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens"""     )			,							[token_to_test_setters]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""     )			,							[token_id_to_test_setters]     )

