



'''simple docstring'''



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_lowercase       :						Optional[Any]						=logging.get_logger(__name__)

_lowercase       :						List[Any]						={
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

_lowercase       :						Optional[int]						={
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

_lowercase       :						Dict						={
    """openai-gpt""": 512,
}


def 							__UpperCAmelCase				(				UpperCamelCase__				:int			)		->						Any:
				snake_case__      :   Optional[int]      =    text.strip()
				if not text:
								return []
				snake_case__      :   int      =    text.split()
				return tokens






class 			_SCREAMING_SNAKE_CASE				(__lowercase      ):



			def __init__(     self		:						List[Any]    ,			__UpperCamelCase		:						Any=True    ,			__UpperCamelCase		:						Tuple=None    ,			__UpperCamelCase		:						List[str]=True    ,			__UpperCamelCase		:						Any=None    ,			__UpperCamelCase		:						List[Any]=True    ,			)      ->      List[str]:
							"""simple docstring"""





							if never_split is None:
											snake_case__      :   str      =    []
							snake_case__      :   Optional[int]      =    do_lower_case
							snake_case__      :   Tuple      =    set(_UpperCamelCase			)
							snake_case__      :   Optional[Any]      =    tokenize_chinese_chars
							snake_case__      :   Dict      =    strip_accents
							snake_case__      :   Optional[int]      =    do_split_on_punc



			def 				lowerCAmelCase (     self		:						int    ,			__UpperCamelCase		:						str    ,			__UpperCamelCase		:						Any=None			)      ->      List[str]:
							"""simple docstring"""





							snake_case__      :   Optional[int]      =    self.never_split.union(set(_UpperCamelCase			)			) if never_split else self.never_split
							snake_case__      :   Union[str, Any]      =    self._clean_text(_UpperCamelCase			)

							# This was added on November 1st, 2018 for the multilingual and Chinese
							# models. This is also applied to the English models now, but it doesn't
							# matter since the English models were not trained on any Chinese data
							# and generally don't have any Chinese data in them (there are Chinese
							# characters in the vocabulary because Wikipedia does have some Chinese
							# words in the English Wikipedia.).
							if self.tokenize_chinese_chars:
											snake_case__      :   List[Any]      =    self._tokenize_chinese_chars(_UpperCamelCase			)
							# prevents treating the same character with different unicode codepoints as different characters
							snake_case__      :   str      =    unicodedata.normalize('''NFC'''    ,			_UpperCamelCase			)
							snake_case__      :   Optional[int]      =    whitespace_tokenize(_UpperCamelCase			)
							snake_case__      :   Optional[int]      =    []
							for token in orig_tokens:
											if token not in never_split:
															if self.do_lower_case:
																			snake_case__      :   Tuple      =    token.lower()
																			if self.strip_accents is not False:
																							snake_case__      :   Any      =    self._run_strip_accents(_UpperCamelCase			)
															elif self.strip_accents:
																			snake_case__      :   List[str]      =    self._run_strip_accents(_UpperCamelCase			)
											split_tokens.extend(self._run_split_on_punc(_UpperCamelCase    ,			_UpperCamelCase			)			)

							snake_case__      :   List[str]      =    whitespace_tokenize(''' '''.join(_UpperCamelCase			)			)
							return output_tokens



			def 				lowerCAmelCase (     self		:						int    ,			__UpperCamelCase		:						str			)      ->      Union[str, Any]:
							"""simple docstring"""





							snake_case__      :   str      =    unicodedata.normalize('''NFD'''    ,			_UpperCamelCase			)
							snake_case__      :   Optional[int]      =    []
							for char in text:
											snake_case__      :   str      =    unicodedata.category(_UpperCamelCase			)
											if cat == "Mn":
															continue
											output.append(_UpperCamelCase			)
							return "".join(_UpperCamelCase			)



			def 				lowerCAmelCase (     self		:						Optional[int]    ,			__UpperCamelCase		:						Tuple    ,			__UpperCamelCase		:						Any=None			)      ->      Dict:
							"""simple docstring"""





							if not self.do_split_on_punc or (never_split is not None and text in never_split):
											return [text]
							snake_case__      :   Dict      =    list(_UpperCamelCase			)
							snake_case__      :   List[Any]      =    0
							snake_case__      :   Any      =    True
							snake_case__      :   List[Any]      =    []
							while i < len(_UpperCamelCase			):
											snake_case__      :   Optional[int]      =    chars[i]
											if _is_punctuation(_UpperCamelCase			):
															output.append([char]			)
															snake_case__      :   Tuple      =    True
											else:
															if start_new_word:
																			output.append([]			)
															snake_case__      :   Optional[Any]      =    False
															output[-1].append(_UpperCamelCase			)
											i += 1

							return ["".join(_UpperCamelCase			) for x in output]



			def 				lowerCAmelCase (     self		:						Optional[Any]    ,			__UpperCamelCase		:						Union[str, Any]			)      ->      Any:
							"""simple docstring"""





							snake_case__      :   List[str]      =    []
							for char in text:
											snake_case__      :   List[Any]      =    ord(_UpperCamelCase			)
											if self._is_chinese_char(_UpperCamelCase			):
															output.append(''' '''			)
															output.append(_UpperCamelCase			)
															output.append(''' '''			)
											else:
															output.append(_UpperCamelCase			)
							return "".join(_UpperCamelCase			)



			def 				lowerCAmelCase (     self		:						Dict    ,			__UpperCamelCase		:						List[str]			)      ->      Optional[Any]:
							"""simple docstring"""





							if (
							    (cp >= 0X4E_00 and cp <= 0X9F_FF)
							    or (cp >= 0X34_00 and cp <= 0X4D_BF)  #
							    or (cp >= 0X2_00_00 and cp <= 0X2_A6_DF)  #
							    or (cp >= 0X2_A7_00 and cp <= 0X2_B7_3F)  #
							    or (cp >= 0X2_B7_40 and cp <= 0X2_B8_1F)  #
							    or (cp >= 0X2_B8_20 and cp <= 0X2_CE_AF)  #
							    or (cp >= 0XF9_00 and cp <= 0XFA_FF)
							    or (cp >= 0X2_F8_00 and cp <= 0X2_FA_1F)  #
							):  #
											return True

							return False







			def 				lowerCAmelCase (     self		:						Optional[Any]    ,			__UpperCamelCase		:						Any			)      ->      Optional[int]:
							"""simple docstring"""





							snake_case__      :   Tuple      =    []
							for char in text:
											snake_case__      :   Optional[int]      =    ord(_UpperCamelCase			)
											if cp == 0 or cp == 0XFF_FD or _is_control(_UpperCamelCase			):
															continue
											if _is_whitespace(_UpperCamelCase			):
															output.append(''' '''			)
											else:
															output.append(_UpperCamelCase			)
							return "".join(_UpperCamelCase			)



def 							__UpperCAmelCase				(				UpperCamelCase__				:int			)		->						Dict:
				snake_case__      :   str      =    set()
				snake_case__      :   Optional[int]      =    word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)			)
								snake_case__      :   Optional[Any]      =    char
				return pairs


def 							__UpperCAmelCase				(				UpperCamelCase__				:Union[str, Any]			)		->						int:
				snake_case__      :   List[Any]      =    text.replace('''—'''   ,				'''-'''			)
				snake_case__      :   str      =    text.replace('''–'''   ,				'''-'''			)
				snake_case__      :   List[str]      =    text.replace('''―'''   ,				'''-'''			)
				snake_case__      :   Optional[int]      =    text.replace('''…'''   ,				'''...'''			)
				snake_case__      :   Union[str, Any]      =    text.replace('''´'''   ,				'''\''''			)
				snake_case__      :   Union[str, Any]      =    re.sub(R'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''   ,				R''' \1 '''   ,				UpperCamelCase_			)
				snake_case__      :   Any      =    re.sub(R'''\s*\n\s*'''   ,				''' \n '''   ,				UpperCamelCase_			)
				snake_case__      :   Optional[int]      =    re.sub(R'''[^\S\n]+'''   ,				''' '''   ,				UpperCamelCase_			)
				return text.strip()






class 			_SCREAMING_SNAKE_CASE				(__lowercase      ):
			A__								=  VOCAB_FILES_NAMES
			A__								=  PRETRAINED_VOCAB_FILES_MAP
			A__								=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A__								=  ['''input_ids''', '''attention_mask''']



			def __init__(     self		:						List[Any]    ,			__UpperCamelCase		:						List[Any]    ,			__UpperCamelCase		:						Union[str, Any]    ,			__UpperCamelCase		:						Optional[Any]="<unk>"    ,			**__UpperCamelCase		:						List[str]			)      ->      int:
							"""simple docstring"""





							super().__init__(unk_token=_UpperCamelCase    ,			**_UpperCamelCase			)

							try:
											import ftfy
											from spacy.lang.en import English

											snake_case__      :   List[Any]      =    English()
											snake_case__      :   str      =    _nlp.tokenizer
											snake_case__      :   List[str]      =    ftfy.fix_text
							except ImportError:
											logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''			)
											snake_case__      :   Any      =    BasicTokenizer(do_lower_case=_UpperCamelCase			)
											snake_case__      :   Union[str, Any]      =    None

							with open(_UpperCamelCase    ,			encoding='''utf-8'''			) as vocab_handle:
											snake_case__      :   List[Any]      =    json.load(_UpperCamelCase			)
							snake_case__      :   Tuple      =    {v: k for k, v in self.encoder.items()}
							with open(_UpperCamelCase    ,			encoding='''utf-8'''			) as merges_handle:
											snake_case__      :   Tuple      =    merges_handle.read().split('''\n'''			)[1:-1]
							snake_case__      :   Optional[Any]      =    [tuple(merge.split()			) for merge in merges]
							snake_case__      :   Dict      =    dict(zip(_UpperCamelCase    ,			range(len(_UpperCamelCase			)			)			)			)
							snake_case__      :   Dict      =    {}



			@property
			def 				lowerCAmelCase (     self		:						Union[str, Any]			)      ->      Optional[Any]:
							"""simple docstring"""





							return True



			@property
			def 				lowerCAmelCase (     self		:						Optional[int]			)      ->      str:
							"""simple docstring"""





							return len(self.encoder			)



			def 				lowerCAmelCase (     self		:						List[Any]			)      ->      Dict:
							"""simple docstring"""





							return dict(self.encoder    ,			**self.added_tokens_encoder			)



			def 				lowerCAmelCase (     self		:						List[str]    ,			__UpperCamelCase		:						Union[str, Any]			)      ->      List[str]:
							"""simple docstring"""





							snake_case__      :   Tuple      =    tuple(token[:-1]			) + (token[-1] + '''</w>''',)
							if token in self.cache:
											return self.cache[token]
							snake_case__      :   List[Any]      =    get_pairs(_UpperCamelCase			)

							if not pairs:
											return token + "</w>"

							while True:
											snake_case__      :   Optional[int]      =    min(_UpperCamelCase    ,			key=lambda __UpperCamelCase			: self.bpe_ranks.get(_UpperCamelCase    ,			float('''inf'''			)			)			)
											if bigram not in self.bpe_ranks:
															break
											snake_case__  ,      snake_case__      :   str      =    bigram
											snake_case__      :   Optional[int]      =    []
											snake_case__      :   Optional[Any]      =    0
											while i < len(_UpperCamelCase			):
															try:
																			snake_case__      :   Dict      =    word.index(_UpperCamelCase    ,			_UpperCamelCase			)
															except ValueError:
																			new_word.extend(word[i:]			)
																			break
															else:
																			new_word.extend(word[i:j]			)
																			snake_case__      :   Dict      =    j

															if word[i] == first and i < len(_UpperCamelCase			) - 1 and word[i + 1] == second:
																			new_word.append(first + second			)
																			i += 2
															else:
																			new_word.append(word[i]			)
																			i += 1
											snake_case__      :   Dict      =    tuple(_UpperCamelCase			)
											snake_case__      :   Optional[int]      =    new_word
											if len(_UpperCamelCase			) == 1:
															break
											else:
															snake_case__      :   int      =    get_pairs(_UpperCamelCase			)
							snake_case__      :   Tuple      =    ''' '''.join(_UpperCamelCase			)
							if word == "\n  </w>":
											snake_case__      :   List[Any]      =    '''\n</w>'''
							snake_case__      :   str      =    word
							return word



			def 				lowerCAmelCase (     self		:						Tuple    ,			__UpperCamelCase		:						Optional[Any]			)      ->      Any:
							"""simple docstring"""





							snake_case__      :   str      =    []
							if self.fix_text is None:
											# Using BERT's BasicTokenizer
											snake_case__      :   str      =    self.nlp.tokenize(_UpperCamelCase			)
											for token in text:
															split_tokens.extend(list(self.bpe(_UpperCamelCase			).split(''' '''			)			)			)
							else:
											# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
											snake_case__      :   List[str]      =    self.nlp(text_standardize(self.fix_text(_UpperCamelCase			)			)			)
											for token in text:
															split_tokens.extend(list(self.bpe(token.text.lower()			).split(''' '''			)			)			)
							return split_tokens



			def 				lowerCAmelCase (     self		:						List[Any]    ,			__UpperCamelCase		:						Any			)      ->      Optional[int]:
							"""simple docstring"""





							return self.encoder.get(_UpperCamelCase    ,			self.encoder.get(self.unk_token			)			)



			def 				lowerCAmelCase (     self		:						int    ,			__UpperCamelCase		:						Optional[Any]			)      ->      int:
							"""simple docstring"""





							return self.decoder.get(_UpperCamelCase    ,			self.unk_token			)



			def 				lowerCAmelCase (     self		:						str    ,			__UpperCamelCase		:						Optional[Any]			)      ->      Tuple:
							"""simple docstring"""





							snake_case__      :   List[str]      =    ''''''.join(_UpperCamelCase			).replace('''</w>'''    ,			''' '''			).strip()
							return out_string







			def 				lowerCAmelCase (     self		:						List[str]    ,			__UpperCamelCase		:						Optional[Any]    ,			__UpperCamelCase		:						str = None			)      ->      int:
							"""simple docstring"""





							if not os.path.isdir(_UpperCamelCase			):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
											return
							snake_case__      :   Dict      =    os.path.join(
							    _UpperCamelCase    ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
							snake_case__      :   Optional[Any]      =    os.path.join(
							    _UpperCamelCase    ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']			)

							with open(_UpperCamelCase    ,			'''w'''    ,			encoding='''utf-8'''			) as f:
											f.write(json.dumps(self.encoder    ,			indent=2    ,			sort_keys=_UpperCamelCase    ,			ensure_ascii=_UpperCamelCase			) + '''\n'''			)

							snake_case__      :   Union[str, Any]      =    0
							with open(_UpperCamelCase    ,			'''w'''    ,			encoding='''utf-8'''			) as writer:
											writer.write('''#version: 0.2\n'''			)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,			key=lambda __UpperCamelCase			: kv[1]			):
															if index != token_index:
																			logger.warning(
																			    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																			    ''' Please check that the tokenizer is not corrupted!'''			)
																			snake_case__      :   Tuple      =    token_index
															writer.write(''' '''.join(_UpperCamelCase			) + '''\n'''			)
															index += 1

							return vocab_file, merge_file












import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase		,       unittest.TestCase):
						_SCREAMING_SNAKE_CASE						:							Tuple				=       PreTrainedTokenizerFast
						_SCREAMING_SNAKE_CASE						:							Any				=       False
						_SCREAMING_SNAKE_CASE						:							Optional[int]				=       True
						_SCREAMING_SNAKE_CASE						:							int				=       '''tokenizer_file'''






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    False  # because we don't have pretrained_vocab_files_map
												super().setUp()
												lowerCAmelCase__         =    True

												lowerCAmelCase__         =    ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
												lowerCAmelCase__         =    'SaulLu/dummy-tokenizer-bytelevel-bpe'

												# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
												lowerCAmelCase__         =    [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

												lowerCAmelCase__         =    PreTrainedTokenizerFast.from_pretrained(model_paths[0] )
												tokenizer.save_pretrained(self.tmpdirname )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																								try:
																														lowerCAmelCase__         =    tempfile.mkdtemp()
																														lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(_UpperCamelCase  ,    **_UpperCamelCase )

																														tokenizer.save_pretrained(self.tmpdirname )
																														super().test_training_new_tokenizer()
																								finally:
																														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																														# is restored
																														shutil.rmtree(self.tmpdirname )
																														lowerCAmelCase__         =    tmpdirname_orig






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																								try:
																														lowerCAmelCase__         =    tempfile.mkdtemp()
																														lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(_UpperCamelCase  ,    **_UpperCamelCase )

																														tokenizer.save_pretrained(self.tmpdirname )
																														super().test_training_new_tokenizer_with_special_tokens_change()
																								finally:
																														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																														# is restored
																														shutil.rmtree(self.tmpdirname )
																														lowerCAmelCase__         =    tmpdirname_orig







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name )

												lowerCAmelCase__         =    ('a' for _ in range(10_00 ))
												lowerCAmelCase__         =    tokenizer.train_new_from_iterator(text_iterator=_UpperCamelCase  ,    length=10_00  ,    vocab_size=50 )

												lowerCAmelCase__         =    new_tokenizer.encode('a🤗' )
												self.assertEqual(_UpperCamelCase  ,    [64, 1_72, 2_53, 97, 2_45] )




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    AutoTokenizer.from_pretrained('bert-base-cased' )
												lowerCAmelCase__         =    json.loads(tokenizer._tokenizer.to_str() )
												lowerCAmelCase__         =    len(_UpperCamelCase )

												with tempfile.TemporaryDirectory() as tmp_dir:
																		# Hack to save this in the tokenizer_config.json
																		lowerCAmelCase__         =    ['tokenizer.4.0.0.json']
																		tokenizer.save_pretrained(_UpperCamelCase )
																		json.dump(_UpperCamelCase  ,    open(os.path.join(_UpperCamelCase  ,    'tokenizer.4.0.0.json' )  ,    'w' ) )

																		# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
																		lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
																		self.assertEqual(len(_UpperCamelCase )  ,    len(_UpperCamelCase ) + 1 )
																		lowerCAmelCase__         =    json.loads(new_tokenizer._tokenizer.to_str() )
																		self.assertIn('huggingface'  ,    json_tokenizer['model']['vocab'] )

																		# Will need to be adjusted if we reach v42 and this test is still here.
																		# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
																		shutil.move(os.path.join(_UpperCamelCase  ,    'tokenizer.4.0.0.json' )  ,    os.path.join(_UpperCamelCase  ,    'tokenizer.42.0.0.json' ) )
																		lowerCAmelCase__         =    ['tokenizer.42.0.0.json']
																		tokenizer.save_pretrained(_UpperCamelCase )
																		lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
																		self.assertEqual(len(_UpperCamelCase )  ,    len(_UpperCamelCase ) )
																		lowerCAmelCase__         =    json.loads(new_tokenizer._tokenizer.to_str() )
																		self.assertNotIn('huggingface'  ,    json_tokenizer['model']['vocab'] )







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
												lowerCAmelCase__         =    'hf-internal-testing/test-two-tokenizers'

												# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
												lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    2_89_97 )
												lowerCAmelCase__         =    json.loads(tokenizer._tokenizer.to_str() )
												self.assertIn('huggingface'  ,    json_tokenizer['model']['vocab'] )

												# Testing an older version by monkey-patching the version in the module it's used.
												import transformers as old_transformers

												lowerCAmelCase__         =    '3.0.0'
												lowerCAmelCase__         =    old_transformers.models.auto.AutoTokenizer.from_pretrained(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    2_89_96 )
												lowerCAmelCase__         =    json.loads(old_tokenizer._tokenizer.to_str() )
												self.assertNotIn('huggingface'  ,    json_tokenizer['model']['vocab'] )




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# See https://github.com/huggingface/transformers/pull/12550
												# and https://github.com/huggingface/tokenizers/issues/537
												lowerCAmelCase__         =    PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel' )
												lowerCAmelCase__         =    'The Matrix is a 1999 science fiction action film.'

												with concurrent.futures.ThreadPoolExecutor() as executor:
																		lowerCAmelCase__         =    [executor.submit(self.fetch  ,    _UpperCamelCase  ,    _UpperCamelCase ) for i in range(10 )]
																		lowerCAmelCase__         =    [future.result() for future in futures]
																		self.assertEqual(_UpperCamelCase  ,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10 )] )







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												return tokenizer.encode(_UpperCamelCase  ,    truncation='longest_first'  ,    padding='longest' )





