'''simple docstring'''
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class       __lowercase		(    _lowerCamelCase      ,				unittest.TestCase    ):







     """simple docstring"""






     SCREAMING_SNAKE_CASE__		 =    PreTrainedTokenizerFast
     SCREAMING_SNAKE_CASE__		 =    False
     SCREAMING_SNAKE_CASE__		 =    True
     SCREAMING_SNAKE_CASE__		 =    '''tokenizer_file'''







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Any		     =	False  # because we don't have pretrained_vocab_files_map
         super().setUp()
         __UpperCamelCase						:						Union[str, Any]		     =	True

         __UpperCamelCase						:						List[Any]		     =	['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
         __UpperCamelCase						:						Optional[Any]		     =	'SaulLu/dummy-tokenizer-bytelevel-bpe'

         # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
         __UpperCamelCase						:						Dict		     =	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

         __UpperCamelCase						:						List[str]		     =	PreTrainedTokenizerFast.from_pretrained(model_paths[0]     )
         tokenizer.save_pretrained(self.tmpdirname     )







     def lowerCAmelCase  (		self     ):
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass







     def lowerCAmelCase  (		self     ):
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass







     def lowerCAmelCase  (		self     ):
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass







     def lowerCAmelCase  (		self     ):
         # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
         pass







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Union[str, Any]		     =	self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
                 try:
                     __UpperCamelCase						:						Tuple		     =	tempfile.mkdtemp()
                     __UpperCamelCase						:						List[str]		     =	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase     )

                     tokenizer.save_pretrained(self.tmpdirname     )
                     super().test_training_new_tokenizer()
                 finally:
                     # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                     # is restored
                     shutil.rmtree(self.tmpdirname     )
                     __UpperCamelCase						:						Any		     =	tmpdirname_orig







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						int		     =	self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
                 try:
                     __UpperCamelCase						:						Optional[int]		     =	tempfile.mkdtemp()
                     __UpperCamelCase						:						str		     =	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase     )

                     tokenizer.save_pretrained(self.tmpdirname     )
                     super().test_training_new_tokenizer_with_special_tokens_change()
                 finally:
                     # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                     # is restored
                     shutil.rmtree(self.tmpdirname     )
                     __UpperCamelCase						:						Tuple		     =	tmpdirname_orig







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Any		     =	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name     )

         __UpperCamelCase						:						Dict		     =	('a' for _ in range(1_0_0_0     ))
         __UpperCamelCase						:						Any		     =	tokenizer.train_new_from_iterator(text_iterator=_lowerCamelCase  ,      length=1_0_0_0  ,      vocab_size=5_0     )

         __UpperCamelCase						:						Optional[int]		     =	new_tokenizer.encode('a🤗'     )
         self.assertEqual(_lowerCamelCase  ,      [6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]     )






@require_tokenizers
class       __lowercase		(    unittest.TestCase    ):







     """simple docstring"""






     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Any		     =	AutoTokenizer.from_pretrained('bert-base-cased'     )
         __UpperCamelCase						:						Union[str, Any]		     =	json.loads(tokenizer._tokenizer.to_str()     )
         __UpperCamelCase						:						str		     =	len(_lowerCamelCase     )

         with tempfile.TemporaryDirectory() as tmp_dir:
             # Hack to save this in the tokenizer_config.json
             __UpperCamelCase						:						List[Any]		     =	['tokenizer.4.0.0.json']
             tokenizer.save_pretrained(_lowerCamelCase     )
             json.dump(_lowerCamelCase  ,      open(os.path.join(_lowerCamelCase  ,      'tokenizer.4.0.0.json'     )  ,      'w'     )     )

             # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
             __UpperCamelCase						:						str		     =	AutoTokenizer.from_pretrained(_lowerCamelCase     )
             self.assertEqual(len(_lowerCamelCase     )  ,      len(_lowerCamelCase     ) + 1     )
             __UpperCamelCase						:						int		     =	json.loads(new_tokenizer._tokenizer.to_str()     )
             self.assertIn('huggingface'  ,      json_tokenizer['model']['vocab']     )

             # Will need to be adjusted if we reach v42 and this test is still here.
             # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
             shutil.move(os.path.join(_lowerCamelCase  ,      'tokenizer.4.0.0.json'     )  ,      os.path.join(_lowerCamelCase  ,      'tokenizer.42.0.0.json'     )     )
             __UpperCamelCase						:						Tuple		     =	['tokenizer.42.0.0.json']
             tokenizer.save_pretrained(_lowerCamelCase     )
             __UpperCamelCase						:						Optional[Any]		     =	AutoTokenizer.from_pretrained(_lowerCamelCase     )
             self.assertEqual(len(_lowerCamelCase     )  ,      len(_lowerCamelCase     )     )
             __UpperCamelCase						:						str		     =	json.loads(new_tokenizer._tokenizer.to_str()     )
             self.assertNotIn('huggingface'  ,      json_tokenizer['model']['vocab']     )







     def lowerCAmelCase  (		self     ):
         # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
         __UpperCamelCase						:						Any		     =	'hf-internal-testing/test-two-tokenizers'

         # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
         __UpperCamelCase						:						Dict		     =	AutoTokenizer.from_pretrained(_lowerCamelCase     )
         self.assertEqual(len(_lowerCamelCase     )  ,      2_8_9_9_7     )
         __UpperCamelCase						:						List[str]		     =	json.loads(tokenizer._tokenizer.to_str()     )
         self.assertIn('huggingface'  ,      json_tokenizer['model']['vocab']     )

         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers

         __UpperCamelCase						:						Optional[Any]		     =	'3.0.0'
         __UpperCamelCase						:						List[Any]		     =	old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCamelCase     )
         self.assertEqual(len(_lowerCamelCase     )  ,      2_8_9_9_6     )
         __UpperCamelCase						:						Dict		     =	json.loads(old_tokenizer._tokenizer.to_str()     )
         self.assertNotIn('huggingface'  ,      json_tokenizer['model']['vocab']     )






@require_tokenizers
class       __lowercase		(    unittest.TestCase    ):







     """simple docstring"""






     def lowerCAmelCase  (		self     ):
         # See https://github.com/huggingface/transformers/pull/12550
         # and https://github.com/huggingface/tokenizers/issues/537
         __UpperCamelCase						:						Union[str, Any]		     =	PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'     )
         __UpperCamelCase						:						Optional[int]		     =	'The Matrix is a 1999 science fiction action film.'

         with concurrent.futures.ThreadPoolExecutor() as executor:
             __UpperCamelCase						:						List[Any]		     =	[executor.submit(self.fetch  ,      _lowerCamelCase  ,      _lowerCamelCase     ) for i in range(1_0     )]
             __UpperCamelCase						:						Tuple		     =	[future.result() for future in futures]
             self.assertEqual(_lowerCamelCase  ,      [[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0     )]     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase     ):
         return tokenizer.encode(_lowerCamelCase  ,      truncation='longest_first'  ,      padding='longest'     )




'''simple docstring'''
import random
import unittest

import numpy as np
import torch

from diffusers import DDIMScheduler, LDMSuperResolutionPipeline, UNetaDModel, VQModel
from diffusers.utils import PIL_INTERPOLATION, floats_tensor, load_image, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch


enable_full_determinism()






class       __lowercase		(    unittest.TestCase    ):







     """simple docstring"""






     @property
     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Dict		     =	1
         __UpperCamelCase						:						Union[str, Any]		     =	3
         __UpperCamelCase						:						Tuple		     =	(3_2, 3_2)

         __UpperCamelCase						:						List[Any]		     =	floats_tensor((batch_size, num_channels) + sizes  ,      rng=random.Random(0     )     ).to(_lowerCamelCase     )
         return image







     @property
     def lowerCAmelCase  (		self     ):
         torch.manual_seed(0     )
         __UpperCamelCase						:						Dict		     =	UNetaDModel(
             block_out_channels=(3_2, 6_4)  ,      layers_per_block=2  ,      sample_size=3_2  ,      in_channels=6  ,      out_channels=3  ,      down_block_types=('DownBlock2D', 'AttnDownBlock2D')  ,      up_block_types=('AttnUpBlock2D', 'UpBlock2D')  ,      )
         return model







     @property
     def lowerCAmelCase  (		self     ):
         torch.manual_seed(0     )
         __UpperCamelCase						:						List[str]		     =	VQModel(
             block_out_channels=[3_2, 6_4]  ,      in_channels=3  ,      out_channels=3  ,      down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']  ,      up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']  ,      latent_channels=3  ,      )
         return model







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Tuple		     =	'cpu'
         __UpperCamelCase						:						Union[str, Any]		     =	self.dummy_uncond_unet
         __UpperCamelCase						:						List[str]		     =	DDIMScheduler()
         __UpperCamelCase						:						Any		     =	self.dummy_vq_model

         __UpperCamelCase						:						Tuple		     =	LDMSuperResolutionPipeline(unet=_lowerCamelCase  ,      vqvae=_lowerCamelCase  ,      scheduler=_lowerCamelCase     )
         ldm.to(_lowerCamelCase     )
         ldm.set_progress_bar_config(disable=_lowerCamelCase     )

         __UpperCamelCase						:						int		     =	self.dummy_image.to(_lowerCamelCase     )

         __UpperCamelCase						:						Union[str, Any]		     =	torch.Generator(device=_lowerCamelCase     ).manual_seed(0     )
         __UpperCamelCase						:						Union[str, Any]		     =	ldm(image=_lowerCamelCase  ,      generator=_lowerCamelCase  ,      num_inference_steps=2  ,      output_type='numpy'     ).images

         __UpperCamelCase						:						str		     =	image[0, -3:, -3:, -1]

         assert image.shape == (1, 6_4, 6_4, 3)
         __UpperCamelCase						:						str		     =	np.array([0.8_6_7_8, 0.8_2_4_5, 0.6_3_8_1, 0.6_8_3_0, 0.4_3_8_5, 0.5_5_9_9, 0.4_6_4_1, 0.6_2_0_1, 0.5_1_5_0]     )

         assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-2







     @unittest.skipIf(torch_device != 'cuda'  ,      'This test requires a GPU'     )
     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	self.dummy_uncond_unet
         __UpperCamelCase						:						Optional[int]		     =	DDIMScheduler()
         __UpperCamelCase						:						Any		     =	self.dummy_vq_model

         # put models in fp16
         __UpperCamelCase						:						str		     =	unet.half()
         __UpperCamelCase						:						Optional[Any]		     =	vqvae.half()

         __UpperCamelCase						:						Optional[int]		     =	LDMSuperResolutionPipeline(unet=_lowerCamelCase  ,      vqvae=_lowerCamelCase  ,      scheduler=_lowerCamelCase     )
         ldm.to(_lowerCamelCase     )
         ldm.set_progress_bar_config(disable=_lowerCamelCase     )

         __UpperCamelCase						:						Dict		     =	self.dummy_image.to(_lowerCamelCase     )

         __UpperCamelCase						:						Optional[Any]		     =	ldm(_lowerCamelCase  ,      num_inference_steps=2  ,      output_type='numpy'     ).images

         assert image.shape == (1, 6_4, 6_4, 3)






@slow
@require_torch
class       __lowercase		(    unittest.TestCase    ):







     """simple docstring"""






     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						List[Any]		     =	load_image(
             'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
             '/vq_diffusion/teddy_bear_pool.png'     )
         __UpperCamelCase						:						List[Any]		     =	init_image.resize((6_4, 6_4)  ,      resample=PIL_INTERPOLATION['lanczos']     )

         __UpperCamelCase						:						List[str]		     =	LDMSuperResolutionPipeline.from_pretrained('duongna/ldm-super-resolution'  ,      device_map='auto'     )
         ldm.set_progress_bar_config(disable=_lowerCamelCase     )

         __UpperCamelCase						:						Union[str, Any]		     =	torch.manual_seed(0     )
         __UpperCamelCase						:						Dict		     =	ldm(image=_lowerCamelCase  ,      generator=_lowerCamelCase  ,      num_inference_steps=2_0  ,      output_type='numpy'     ).images

         __UpperCamelCase						:						List[Any]		     =	image[0, -3:, -3:, -1]

         assert image.shape == (1, 2_5_6, 2_5_6, 3)
         __UpperCamelCase						:						str		     =	np.array([0.7_6_4_4, 0.7_6_7_9, 0.7_6_4_2, 0.7_6_3_3, 0.7_6_6_6, 0.7_5_6_0, 0.7_4_2_5, 0.7_2_5_7, 0.6_9_0_7]     )

         assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1E-2



