





'''simple docstring'''






import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       lowerCamelCase       (    __UpperCAmelCase  ,				unittest.TestCase	):
					_SCREAMING_SNAKE_CASE				 =     PreTrainedTokenizerFast
					_SCREAMING_SNAKE_CASE				 =     False
					_SCREAMING_SNAKE_CASE				 =     True
					_SCREAMING_SNAKE_CASE				 =     "tokenizer_file"


					def  SCREAMING_SNAKE_CASE_		(       self						:	List[Any]							):





											'''simple docstring'''





											_snake_case: List[str]	       =		False  # because we don't have pretrained_vocab_files_map
											super().setUp()
											_snake_case: Any	       =		True

											_snake_case: Optional[Any]	       =		['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
											_snake_case: str	       =		'SaulLu/dummy-tokenizer-bytelevel-bpe'

											# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
											_snake_case: int	       =		[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

											_snake_case: Optional[Any]	       =		PreTrainedTokenizerFast.from_pretrained(model_paths[0]							)
											tokenizer.save_pretrained(self.tmpdirname							)


					def  SCREAMING_SNAKE_CASE_		(       self						:	str							):





											'''simple docstring'''





											pass


					def  SCREAMING_SNAKE_CASE_		(       self						:	Any							):





											'''simple docstring'''





											pass


					def  SCREAMING_SNAKE_CASE_		(       self						:	List[Any]							):





											'''simple docstring'''





											pass


					def  SCREAMING_SNAKE_CASE_		(       self						:	List[Any]							):





											'''simple docstring'''





											pass


					def  SCREAMING_SNAKE_CASE_		(       self						:	List[Any]							):





											'''simple docstring'''





											_snake_case: List[str]	       =		self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																							try:
																													_snake_case: Dict	       =		tempfile.mkdtemp()
																													_snake_case: int	       =		self.rust_tokenizer_class.from_pretrained(__snake_case							,  **__snake_case							)

																													tokenizer.save_pretrained(self.tmpdirname							)
																													super().test_training_new_tokenizer()
																							finally:
																													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																													# is restored
																													shutil.rmtree(self.tmpdirname							)
																													_snake_case: Tuple	       =		tmpdirname_orig


					def  SCREAMING_SNAKE_CASE_		(       self						:	Any							):





											'''simple docstring'''





											_snake_case: List[str]	       =		self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																							try:
																													_snake_case: List[str]	       =		tempfile.mkdtemp()
																													_snake_case: str	       =		self.rust_tokenizer_class.from_pretrained(__snake_case							,  **__snake_case							)

																													tokenizer.save_pretrained(self.tmpdirname							)
																													super().test_training_new_tokenizer_with_special_tokens_change()
																							finally:
																													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																													# is restored
																													shutil.rmtree(self.tmpdirname							)
																													_snake_case: Any	       =		tmpdirname_orig




					def  SCREAMING_SNAKE_CASE_		(       self						:	Union[str, Any]							):





											'''simple docstring'''





											_snake_case: int	       =		self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name							)

											_snake_case: int	       =		('a' for _ in range(10_00							))
											_snake_case: int	       =		tokenizer.train_new_from_iterator(text_iterator=__snake_case							,  length=10_00							,  vocab_size=50							)

											_snake_case: str	       =		new_tokenizer.encode('a🤗'							)
											self.assertEqual(__snake_case							,  [64, 1_72, 2_53, 97, 2_45]							)







@require_tokenizers
class       lowerCamelCase       (    unittest.TestCase	):


					def  SCREAMING_SNAKE_CASE_		(       self						:	List[Any]							):





											'''simple docstring'''





											_snake_case: List[str]	       =		AutoTokenizer.from_pretrained('bert-base-cased'							)
											_snake_case: Any	       =		json.loads(tokenizer._tokenizer.to_str()							)
											_snake_case: str	       =		len(__snake_case							)

											with tempfile.TemporaryDirectory() as tmp_dir:
																	# Hack to save this in the tokenizer_config.json
																	_snake_case: Dict	       =		['tokenizer.4.0.0.json']
																	tokenizer.save_pretrained(__snake_case							)
																	json.dump(__snake_case							,  open(os.path.join(__snake_case							,  'tokenizer.4.0.0.json'							)							,  'w'							)							)

																	# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
																	_snake_case: Optional[int]	       =		AutoTokenizer.from_pretrained(__snake_case							)
																	self.assertEqual(len(__snake_case							)							,  len(__snake_case							) + 1							)
																	_snake_case: Union[str, Any]	       =		json.loads(new_tokenizer._tokenizer.to_str()							)
																	self.assertIn('huggingface'							,  json_tokenizer['model']['vocab']							)

																	# Will need to be adjusted if we reach v42 and this test is still here.
																	# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
																	shutil.move(os.path.join(__snake_case							,  'tokenizer.4.0.0.json'							)							,  os.path.join(__snake_case							,  'tokenizer.42.0.0.json'							)							)
																	_snake_case: Union[str, Any]	       =		['tokenizer.42.0.0.json']
																	tokenizer.save_pretrained(__snake_case							)
																	_snake_case: str	       =		AutoTokenizer.from_pretrained(__snake_case							)
																	self.assertEqual(len(__snake_case							)							,  len(__snake_case							)							)
																	_snake_case: Optional[int]	       =		json.loads(new_tokenizer._tokenizer.to_str()							)
																	self.assertNotIn('huggingface'							,  json_tokenizer['model']['vocab']							)




					def  SCREAMING_SNAKE_CASE_		(       self						:	Tuple							):





											'''simple docstring'''





											_snake_case: Any	       =		'hf-internal-testing/test-two-tokenizers'

											# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
											_snake_case: List[str]	       =		AutoTokenizer.from_pretrained(__snake_case							)
											self.assertEqual(len(__snake_case							)							,  2_89_97							)
											_snake_case: str	       =		json.loads(tokenizer._tokenizer.to_str()							)
											self.assertIn('huggingface'							,  json_tokenizer['model']['vocab']							)

											# Testing an older version by monkey-patching the version in the module it's used.
											import transformers as old_transformers

											_snake_case: Tuple	       =		'3.0.0'
											_snake_case: Optional[Any]	       =		old_transformers.models.auto.AutoTokenizer.from_pretrained(__snake_case							)
											self.assertEqual(len(__snake_case							)							,  2_89_96							)
											_snake_case: Tuple	       =		json.loads(old_tokenizer._tokenizer.to_str()							)
											self.assertNotIn('huggingface'							,  json_tokenizer['model']['vocab']							)







@require_tokenizers
class       lowerCamelCase       (    unittest.TestCase	):


					def  SCREAMING_SNAKE_CASE_		(       self						:	int							):





											'''simple docstring'''





											_snake_case: Tuple	       =		PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'							)
											_snake_case: int	       =		'The Matrix is a 1999 science fiction action film.'

											with concurrent.futures.ThreadPoolExecutor() as executor:
																	_snake_case: int	       =		[executor.submit(self.fetch							,  __snake_case							,  __snake_case							) for i in range(10							)]
																	_snake_case: Any	       =		[future.result() for future in futures]
																	self.assertEqual(__snake_case							,  [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10							)]							)




					def  SCREAMING_SNAKE_CASE_		(       self						:	Dict							,  __snake_case						:	str							,  __snake_case						:	Optional[int]							):





											'''simple docstring'''





											return tokenizer.encode(__snake_case							,  truncation='longest_first'							,  padding='longest'							)












'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
		A :			Optional[Any]							   =      None


A :			Tuple							   =      logging.get_logger(__name__)

A :			Optional[int]							   =      {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

A :			Tuple							   =      {
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
        ),
    },
}

A :			int							   =      {
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}







class       lowerCamelCase       (    __UpperCAmelCase	):
					_SCREAMING_SNAKE_CASE				 =     VOCAB_FILES_NAMES
					_SCREAMING_SNAKE_CASE				 =     PRETRAINED_VOCAB_FILES_MAP
					_SCREAMING_SNAKE_CASE				 =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_SCREAMING_SNAKE_CASE				 =     ["input_ids", "attention_mask"]
					_SCREAMING_SNAKE_CASE				 =     XLMRobertaTokenizer


					def __init__(       self						:	Any							,  __snake_case						:	Tuple=None							,  __snake_case						:	List[Any]=None							,  __snake_case						:	List[str]="<s>"							,  __snake_case						:	int="</s>"							,  __snake_case						:	str="</s>"							,  __snake_case						:	Any="<s>"							,  __snake_case						:	int="<unk>"							,  __snake_case						:	Optional[Any]="<pad>"							,  __snake_case						:	Optional[int]="<mask>"							,  **__snake_case						:	List[str]							,  ):





											'''simple docstring'''





											_snake_case: Optional[Any]	       =		AddedToken(__snake_case							,  lstrip=__snake_case							,  rstrip=__snake_case							) if isinstance(__snake_case							,  __snake_case							) else mask_token

											super().__init__(
											    __snake_case							,  tokenizer_file=__snake_case							,  bos_token=__snake_case							,  eos_token=__snake_case							,  sep_token=__snake_case							,  cls_token=__snake_case							,  unk_token=__snake_case							,  pad_token=__snake_case							,  mask_token=__snake_case							,  **__snake_case							,  )

											_snake_case: int	       =		vocab_file
											_snake_case: List[Any]	       =		False if not self.vocab_file else True


					def  SCREAMING_SNAKE_CASE_		(       self						:	Tuple							,  __snake_case						:	List[int]							,  __snake_case						:	Optional[List[int]] = None							):





											'''simple docstring'''





											if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											_snake_case: Union[str, Any]	       =		[self.cls_token_id]
											_snake_case: int	       =		[self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep


					def  SCREAMING_SNAKE_CASE_		(       self						:	Union[str, Any]							,  __snake_case						:	List[int]							,  __snake_case						:	Optional[List[int]] = None							):





											'''simple docstring'''





											_snake_case: Union[str, Any]	       =		[self.sep_token_id]
											_snake_case: List[str]	       =		[self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep							) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]




					def  SCREAMING_SNAKE_CASE_		(       self						:	Any							,  __snake_case						:	str							,  __snake_case						:	Optional[str] = None							):





											'''simple docstring'''





											if not self.can_save_slow_tokenizer:
																	raise ValueError(
																	    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
																	    'tokenizer.'							)

											if not os.path.isdir(__snake_case							):
																	logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''							)
																	return
											_snake_case: Optional[int]	       =		os.path.join(
											    __snake_case							,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

											if os.path.abspath(self.vocab_file							) != os.path.abspath(__snake_case							):
																	copyfile(self.vocab_file							,  __snake_case							)

											return (out_vocab_file,)





