


import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_A		       =						get_tests_dir('fixtures/spiece.model')






@require_sentencepiece
@require_tokenizers
class     UpperCAmelCase__       (					A_   ,  unittest.TestCase		):



    """simple docstring"""






    UpperCAmelCase__						:		List[str]      =  DebertaVaTokenizer
    UpperCAmelCase__						:		Tuple      =  DebertaVaTokenizerFast
    UpperCAmelCase__						:		Optional[int]      =  True
    UpperCAmelCase__						:		Optional[Any]      =  True






    def _a					(			self    )  ->       List[str]:
         super().setUp()

         # We have a SentencePiece fixture for testing
         __UpperCamelCase    							=DebertaVaTokenizer(A_					,   unk_token='<unk>'    )
         tokenizer.save_pretrained(self.tmpdirname    )






    def _a					(			self					,   A_    )  ->       Dict:
         __UpperCamelCase    							='this is a test'
         __UpperCamelCase    							='this is a test'
         return input_text, output_text






    def _a					(			self    )  ->       Tuple:
         __UpperCamelCase    							='<pad>'
         __UpperCamelCase    							=0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_    )					,   A_    )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_    )					,   A_    )






    def _a					(			self    )  ->       Optional[Any]:
         __UpperCamelCase    							=list(self.get_tokenizer().get_vocab().keys()    )
         self.assertEqual(vocab_keys[0]					,   '<pad>'    )
         self.assertEqual(vocab_keys[1]					,   '<unk>'    )
         self.assertEqual(vocab_keys[-1]					,   '[PAD]'    )
         self.assertEqual(len(A_    )					,   30001    )






    def _a					(			self    )  ->       Dict:
         self.assertEqual(self.get_tokenizer().vocab_size					,   30000    )






    def _a					(			self    )  ->       List[Any]:
         # fmt: off
         __UpperCamelCase    							=' \tHeLLo!how  \n Are yoU?  '
         __UpperCamelCase    							=['▁hello', '!', 'how', '▁are', '▁you', '?']
         # fmt: on

         __UpperCamelCase    							=DebertaVaTokenizer(A_					,   do_lower_case=A_    )
         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=DebertaVaTokenizerFast(A_					,   do_lower_case=A_    )
         __UpperCamelCase    							=rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )






    @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'    )
    def _a					(			self    )  ->       Optional[int]:
         pass






    @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'    )
    def _a					(			self    )  ->       str:
         pass






    def _a					(			self    )  ->       Dict:
         # fmt: off
         __UpperCamelCase    							='I was born in 92000, and this is falsé.'
         __UpperCamelCase    							=['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
         # fmt: on

         __UpperCamelCase    							=DebertaVaTokenizer(A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=DebertaVaTokenizerFast(A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )






    def _a					(			self    )  ->       Optional[int]:
         # fmt: off
         __UpperCamelCase    							='I was born in 92000, and this is falsé.'
         __UpperCamelCase    							=['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
         # fmt: on

         __UpperCamelCase    							=DebertaVaTokenizer(A_					,   do_lower_case=A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(tokenizer.encode(A_					,   add_special_tokens=A_    )    )
         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=DebertaVaTokenizerFast(A_					,   do_lower_case=A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_					,   add_special_tokens=A_    )    )
         self.assertListEqual(A_					,   A_    )






    def _a					(			self    )  ->       Any:
         # fmt: off
         __UpperCamelCase    							='I was born in 92000, and this is falsé.'
         __UpperCamelCase    							=['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
         # fmt: on

         __UpperCamelCase    							=DebertaVaTokenizer(A_					,   do_lower_case=A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=DebertaVaTokenizerFast(A_					,   do_lower_case=A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )






    def _a					(			self    )  ->       Union[str, Any]:
         # fmt: off
         __UpperCamelCase    							='I was born in 92000, and this is falsé.'
         __UpperCamelCase    							=['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
         # fmt: on

         __UpperCamelCase    							=DebertaVaTokenizer(A_					,   do_lower_case=A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=DebertaVaTokenizerFast(A_					,   do_lower_case=A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )






    def _a					(			self    )  ->       Tuple:
         # fmt: off
         __UpperCamelCase    							=' \tHeLLo!how  \n Are yoU?  '
         __UpperCamelCase    							=['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
         # fmt: on

         __UpperCamelCase    							=DebertaVaTokenizer(A_					,   do_lower_case=A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=DebertaVaTokenizerFast(A_					,   do_lower_case=A_					,   split_by_punct=A_    )
         __UpperCamelCase    							=rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_					,   add_special_tokens=A_    )    )

         self.assertListEqual(A_					,   A_    )






    def _a					(			self    )  ->       Union[str, Any]:
         __UpperCamelCase    							=self.get_tokenizer()
         __UpperCamelCase    							=self.get_rust_tokenizer()

         __UpperCamelCase    							='I was born in 92000, and this is falsé.'

         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(tokenizer.encode(A_					,   add_special_tokens=A_    )    )
         __UpperCamelCase    							=rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_					,   add_special_tokens=A_    )    )
         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=tokenizer.encode(A_					,   add_special_tokens=A_    )
         __UpperCamelCase    							=rust_tokenizer.encode(A_					,   add_special_tokens=A_    )
         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=self.get_rust_tokenizer()
         __UpperCamelCase    							=tokenizer.encode(A_    )
         __UpperCamelCase    							=rust_tokenizer.encode(A_    )
         self.assertListEqual(A_					,   A_    )






    def _a					(			self    )  ->       Tuple:
         __UpperCamelCase    							='This is a test'
         __UpperCamelCase    							=[13, 1, 4398, 25, 21, 1289]
         __UpperCamelCase    							=['▁', 'T', 'his', '▁is', '▁a', '▁test']
         __UpperCamelCase    							=['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

         __UpperCamelCase    							=DebertaVaTokenizer(A_					,   keep_accents=A_    )
         __UpperCamelCase    							=DebertaVaTokenizerFast(A_					,   keep_accents=A_    )

         __UpperCamelCase    							=tokenizer.encode(A_					,   add_special_tokens=A_    )
         self.assertListEqual(A_					,   A_    )
         __UpperCamelCase    							=tokenizer.tokenize(A_    )
         self.assertListEqual(A_					,   A_    )
         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(A_    )
         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=rust_tokenizer.encode(A_					,   add_special_tokens=A_    )
         self.assertListEqual(A_					,   A_    )
         __UpperCamelCase    							=rust_tokenizer.tokenize(A_    )
         self.assertListEqual(A_					,   A_    )
         __UpperCamelCase    							=rust_tokenizer.convert_ids_to_tokens(A_    )
         self.assertListEqual(A_					,   A_    )

         # fmt: off
         __UpperCamelCase    							='I was born in 92000, and this is falsé.'
         __UpperCamelCase    							=[13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
         __UpperCamelCase    							=['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
         __UpperCamelCase    							=['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
         # fmt: on

         __UpperCamelCase    							=tokenizer.encode(A_					,   add_special_tokens=A_    )
         self.assertListEqual(A_					,   A_    )
         __UpperCamelCase    							=tokenizer.tokenize(A_    )
         self.assertListEqual(A_					,   A_    )
         __UpperCamelCase    							=tokenizer.convert_ids_to_tokens(A_    )
         self.assertListEqual(A_					,   A_    )

         __UpperCamelCase    							=rust_tokenizer.encode(A_					,   add_special_tokens=A_    )
         self.assertListEqual(A_					,   A_    )
         __UpperCamelCase    							=rust_tokenizer.tokenize(A_    )
         self.assertListEqual(A_					,   A_    )
         __UpperCamelCase    							=rust_tokenizer.convert_ids_to_tokens(A_    )
         self.assertListEqual(A_					,   A_    )






    def _a					(			self    )  ->       Optional[int]:
         __UpperCamelCase    							=DebertaVaTokenizer(A_    )

         __UpperCamelCase    							=tokenizer.encode('sequence builders'    )
         __UpperCamelCase    							=tokenizer.encode('multi-sequence build'    )

         __UpperCamelCase    							=tokenizer.build_inputs_with_special_tokens(A_    )
         __UpperCamelCase    							=tokenizer.build_inputs_with_special_tokens(A_					,   A_    )

         self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]					,   A_    )
         self.assertEqual(
             [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]					,   A_					,   )






    @slow
    def _a					(			self    )  ->       Optional[int]:
         # fmt: off
         __UpperCamelCase    							={'input_ids': [[1, 39867, 36, 19390, 486, 27, 35052, 81436, 18, 60685, 1225, 7, 35052, 81436, 18, 9367, 16899, 18, 15937, 53, 594, 773, 18, 16287, 30465, 36, 15937, 6, 41139, 38, 36979, 60763, 191, 6, 34132, 99, 6, 50538, 390, 43230, 6, 34132, 2779, 20850, 14, 699, 1072, 1194, 36, 382, 10901, 53, 7, 699, 1072, 2084, 36, 20422, 630, 53, 19, 105, 3049, 1896, 1053, 16899, 1506, 11, 37978, 4243, 7, 1237, 31869, 200, 16566, 654, 6, 35052, 81436, 7, 55630, 13593, 4, 2], [1, 26, 15011, 13, 667, 8, 1053, 18, 23611, 1237, 72356, 12820, 34, 104134, 1209, 35, 13313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 15785, 14951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=A_					,   model_name='microsoft/deberta-v2-xlarge'					,   revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'					,   )









import unittest

import torch

from diffusers import VQModel
from diffusers.utils import floats_tensor, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()







class      lowerCAmelCase__       (				a     ,    a     ,    unittest.TestCase):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE  			= VQModel
     __SCREAMING_SNAKE_CASE  			= "sample"


     @property
     def  _lowerCamelCase (						self ,						__lowerCamelCase=(3_2, 3_2))			->				Optional[Any]:
            _A      :       Optional[int]				    = 4
            _A      :       Tuple				    = 3

            _A      :       List[Any]				    = floats_tensor((batch_size, num_channels) + sizes).to(__lowerCamelCase)

            return {"sample": image}


     @property
     def  _lowerCamelCase (						self)			->				int:
            return (3, 3_2, 3_2)


     @property
     def  _lowerCamelCase (						self)			->				List[Any]:
            return (3, 3_2, 3_2)


     def  _lowerCamelCase (						self)			->				Union[str, Any]:
            _A      :       List[Any]				    = {
                "block_out_channels": [3_2, 6_4],
                "in_channels": 3,
                "out_channels": 3,
                "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"],
                "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"],
                "latent_channels": 3,
            }
            _A      :       int				    = self.dummy_input
            return init_dict, inputs_dict


     def  _lowerCamelCase (						self)			->				Union[str, Any]:
            pass


     def  _lowerCamelCase (						self)			->				Any:
            pass


     def  _lowerCamelCase (						self)			->				Any:
            _A    ,			_A      :       List[Any]				    = VQModel.from_pretrained("fusing/vqgan-dummy" ,						output_loading_info=__lowerCamelCase)
            self.assertIsNotNone(__lowerCamelCase)
            self.assertEqual(len(loading_info["missing_keys"]) ,						0)

            model.to(__lowerCamelCase)
            _A      :       str				    = model(**self.dummy_input)

            assert image is not None, "Make sure output is not None"


     def  _lowerCamelCase (						self)			->				Union[str, Any]:
            _A      :       Optional[Any]				    = VQModel.from_pretrained("fusing/vqgan-dummy")
            model.to(__lowerCamelCase).eval()

            torch.manual_seed(0)
            if torch.cuda.is_available():
                   torch.cuda.manual_seed_all(0)

            _A      :       Tuple				    = torch.randn(1 ,						model.config.in_channels ,						model.config.sample_size ,						model.config.sample_size)
            _A      :       Optional[int]				    = image.to(__lowerCamelCase)
            with torch.no_grad():
                   _A      :       List[str]				    = model(__lowerCamelCase).sample

            _A      :       int				    = output[0, -1, -3:, -3:].flatten().cpu()
            # fmt: off
            _A      :       Optional[Any]				    = torch.tensor([-0.0_1_5_3, -0.4_0_4_4, -0.1_8_8_0, -0.5_1_6_1, -0.2_4_1_8, -0.4_0_7_2, -0.1_6_1_2, -0.0_6_3_3, -0.0_1_4_3])
            # fmt: on
            self.assertTrue(torch.allclose(__lowerCamelCase ,						__lowerCamelCase ,						atol=1e-3))
