




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class        __lowerCAmelCase					(			UpperCAmelCase__   ,						unittest.TestCase					):
			snake_case_ :					str						=	PreTrainedTokenizerFast
			snake_case_ :					Dict						=	False
			snake_case_ :					List[Any]						=	True
			snake_case_ :					Tuple						=	"tokenizer_file"







			def UpperCamelCase     (			self       :     int ):




									"""simple docstring"""


									_UpperCAmelCase						=	False  # because we don't have pretrained_vocab_files_map
									super().setUp()
									_UpperCAmelCase						=	True

									_UpperCAmelCase						=	["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
									_UpperCAmelCase						=	"SaulLu/dummy-tokenizer-bytelevel-bpe"

									# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
									_UpperCAmelCase						=	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

									_UpperCAmelCase						=	PreTrainedTokenizerFast.from_pretrained(model_paths[0] )
									tokenizer.save_pretrained(self.tmpdirname )







			def UpperCamelCase     (			self       :     Dict ):




									"""simple docstring"""


									pass







			def UpperCamelCase     (			self       :     Dict ):




									"""simple docstring"""


									pass







			def UpperCamelCase     (			self       :     Optional[Any] ):




									"""simple docstring"""


									pass







			def UpperCamelCase     (			self       :     List[str] ):




									"""simple docstring"""


									pass







			def UpperCamelCase     (			self       :     Any ):




									"""simple docstring"""


									_UpperCAmelCase						=	self.tmpdirname
									# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
																					try:
																											_UpperCAmelCase						=	tempfile.mkdtemp()
																											_UpperCAmelCase						=	self.rust_tokenizer_class.from_pretrained(snake_case__       ,       **snake_case__ )

																											tokenizer.save_pretrained(self.tmpdirname )
																											super().test_training_new_tokenizer()
																					finally:
																											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																											# is restored
																											shutil.rmtree(self.tmpdirname )
																											_UpperCAmelCase						=	tmpdirname_orig







			def UpperCamelCase     (			self       :     Any ):




									"""simple docstring"""


									_UpperCAmelCase						=	self.tmpdirname
									# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
																					try:
																											_UpperCAmelCase						=	tempfile.mkdtemp()
																											_UpperCAmelCase						=	self.rust_tokenizer_class.from_pretrained(snake_case__       ,       **snake_case__ )

																											tokenizer.save_pretrained(self.tmpdirname )
																											super().test_training_new_tokenizer_with_special_tokens_change()
																					finally:
																											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																											# is restored
																											shutil.rmtree(self.tmpdirname )
																											_UpperCAmelCase						=	tmpdirname_orig







			def UpperCamelCase     (			self       :     int ):




									"""simple docstring"""


									_UpperCAmelCase						=	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name )

									_UpperCAmelCase						=	("a" for _ in range(1_000 ))
									_UpperCAmelCase						=	tokenizer.train_new_from_iterator(text_iterator=snake_case__       ,       length=1_000       ,       vocab_size=50 )

									_UpperCAmelCase						=	new_tokenizer.encode("a🤗" )
									self.assertEqual(snake_case__       ,       [64, 172, 253, 97, 245] )






@require_tokenizers
class        __lowerCAmelCase					(			unittest.TestCase					):







			def UpperCamelCase     (			self       :     List[str] ):




									"""simple docstring"""


									_UpperCAmelCase						=	AutoTokenizer.from_pretrained("bert-base-cased" )
									_UpperCAmelCase						=	json.loads(tokenizer._tokenizer.to_str() )
									_UpperCAmelCase						=	len(snake_case__ )

									with tempfile.TemporaryDirectory() as tmp_dir:
															# Hack to save this in the tokenizer_config.json
															_UpperCAmelCase						=	["tokenizer.4.0.0.json"]
															tokenizer.save_pretrained(snake_case__ )
															json.dump(snake_case__       ,       open(os.path.join(snake_case__       ,       "tokenizer.4.0.0.json" )       ,       "w" ) )

															# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
															_UpperCAmelCase						=	AutoTokenizer.from_pretrained(snake_case__ )
															self.assertEqual(len(snake_case__ )       ,       len(snake_case__ ) + 1 )
															_UpperCAmelCase						=	json.loads(new_tokenizer._tokenizer.to_str() )
															self.assertIn("huggingface"       ,       json_tokenizer["model"]["vocab"] )

															# Will need to be adjusted if we reach v42 and this test is still here.
															# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
															shutil.move(os.path.join(snake_case__       ,       "tokenizer.4.0.0.json" )       ,       os.path.join(snake_case__       ,       "tokenizer.42.0.0.json" ) )
															_UpperCAmelCase						=	["tokenizer.42.0.0.json"]
															tokenizer.save_pretrained(snake_case__ )
															_UpperCAmelCase						=	AutoTokenizer.from_pretrained(snake_case__ )
															self.assertEqual(len(snake_case__ )       ,       len(snake_case__ ) )
															_UpperCAmelCase						=	json.loads(new_tokenizer._tokenizer.to_str() )
															self.assertNotIn("huggingface"       ,       json_tokenizer["model"]["vocab"] )







			def UpperCamelCase     (			self       :     str ):




									"""simple docstring"""


									_UpperCAmelCase						=	"hf-internal-testing/test-two-tokenizers"

									# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
									_UpperCAmelCase						=	AutoTokenizer.from_pretrained(snake_case__ )
									self.assertEqual(len(snake_case__ )       ,       28_997 )
									_UpperCAmelCase						=	json.loads(tokenizer._tokenizer.to_str() )
									self.assertIn("huggingface"       ,       json_tokenizer["model"]["vocab"] )

									# Testing an older version by monkey-patching the version in the module it's used.
									import transformers as old_transformers

									_UpperCAmelCase						=	"3.0.0"
									_UpperCAmelCase						=	old_transformers.models.auto.AutoTokenizer.from_pretrained(snake_case__ )
									self.assertEqual(len(snake_case__ )       ,       28_996 )
									_UpperCAmelCase						=	json.loads(old_tokenizer._tokenizer.to_str() )
									self.assertNotIn("huggingface"       ,       json_tokenizer["model"]["vocab"] )






@require_tokenizers
class        __lowerCAmelCase					(			unittest.TestCase					):







			def UpperCamelCase     (			self       :     Dict ):




									"""simple docstring"""


									_UpperCAmelCase						=	PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel" )
									_UpperCAmelCase						=	"The Matrix is a 1999 science fiction action film."

									with concurrent.futures.ThreadPoolExecutor() as executor:
															_UpperCAmelCase						=	[executor.submit(self.fetch       ,       snake_case__       ,       snake_case__ ) for i in range(10 )]
															_UpperCAmelCase						=	[future.result() for future in futures]
															self.assertEqual(snake_case__       ,       [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10 )] )







			def UpperCamelCase     (			self       :     Tuple       ,       snake_case__       :     Any       ,       snake_case__       :     Union[str, Any] ):




									"""simple docstring"""


									return tokenizer.encode(snake_case__       ,       truncation="longest_first"       ,       padding="longest" )










import torch
import torch.nn.functional as F

from diffusers import VQDiffusionScheduler

from .test_schedulers import SchedulerCommonTest






class        __lowerCAmelCase					(			UpperCAmelCase__					):
			snake_case_ :					List[str]						=	(VQDiffusionScheduler,)







			def UpperCamelCase     (			self       :     Dict       ,       **snake_case__       :     int ):




									"""simple docstring"""


									_UpperCAmelCase						=	{
									    "num_vec_classes": 4_097,
									    "num_train_timesteps": 100,
									}

									config.update(**snake_case__ )
									return config







			def UpperCamelCase     (			self       :     Optional[int]       ,       snake_case__       :     Any ):




									"""simple docstring"""


									_UpperCAmelCase						=	4
									_UpperCAmelCase						=	8
									_UpperCAmelCase						=	8

									_UpperCAmelCase						=	torch.randint(0       ,       snake_case__       ,       (batch_size, height * width) )

									return sample







			@property
			def UpperCamelCase     (			self       :     str ):




									"""simple docstring"""


									assert False







			def UpperCamelCase     (			self       :     List[str]       ,       snake_case__       :     Optional[Any] ):




									"""simple docstring"""


									def model(snake_case__       :     Dict       ,       snake_case__       :     Tuple       ,       *snake_case__       :     str ):
															_UpperCAmelCase				,					_UpperCAmelCase						=	sample.shape
															_UpperCAmelCase						=	torch.rand((batch_size, num_vec_classes - 1, num_latent_pixels) )
															_UpperCAmelCase						=	F.log_softmax(logits.double()       ,       dim=1 ).float()
															return return_value

									return model







			def UpperCamelCase     (			self       :     Optional[int] ):




									"""simple docstring"""


									for timesteps in [2, 5, 100, 1_000]:
															self.check_over_configs(num_train_timesteps=snake_case__ )







			def UpperCamelCase     (			self       :     Optional[int] ):




									"""simple docstring"""


									for num_vec_classes in [5, 100, 1_000, 4_000]:
															self.check_over_configs(num_vec_classes=snake_case__ )







			def UpperCamelCase     (			self       :     Optional[Any] ):




									"""simple docstring"""


									for t in [0, 50, 99]:
															self.check_over_forward(time_step=snake_case__ )







			def UpperCamelCase     (			self       :     Tuple ):




									"""simple docstring"""


									pass




