


import gc
import unittest

import torch
from parameterized import parameterized

from diffusers import AutoencoderKL
from diffusers.utils import floats_tensor, load_hf_numpy, require_torch_gpu, slow, torch_all_close, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()







class 		_snake_case  (      _snake_case						,    _snake_case						,    unittest.TestCase							):
 SCREAMING_SNAKE_CASE__      =       AutoencoderKL
 SCREAMING_SNAKE_CASE__      =       'sample'
 SCREAMING_SNAKE_CASE__      =       1e-2




 @property
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:str 							=       4
       a						:int 							=       3
       a						:int 							=       (32, 32)

       a						:List[Any] 							=       floats_tensor((batch_size, num_channels) + sizes    ).to(_lowerCamelCase    )

       return {"sample": image}




 @property
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       return (3, 32, 32)




 @property
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       return (3, 32, 32)




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Tuple 							=       {
           '''block_out_channels''': [32, 64],
           '''in_channels''': 3,
           '''out_channels''': 3,
           '''down_block_types''': ['''DownEncoderBlock2D''', '''DownEncoderBlock2D'''],
           '''up_block_types''': ['''UpDecoderBlock2D''', '''UpDecoderBlock2D'''],
           '''latent_channels''': 4,
       }
       a						:Any 							=       self.dummy_input
       return init_dict, inputs_dict




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       pass




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       pass




 @unittest.skipIf(torch_device == '''mps'''					,  '''Gradient checkpointing skipped on MPS'''    )
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       # enable deterministic behavior for gradient checkpointing
       a							,				a						:Any 							=       self.prepare_init_args_and_inputs_for_common()
       a						:Dict 							=       self.model_class(**_lowerCamelCase    )
       model.to(_lowerCamelCase    )

       assert not model.is_gradient_checkpointing and model.training

       a						:List[str] 							=       model(**_lowerCamelCase    ).sample
       # run the backwards pass on the model. For backwards pass, for simplicity purpose,
       # we won't calculate the loss and rather backprop on out.sum()
       model.zero_grad()

       a						:Tuple 							=       torch.randn_like(_lowerCamelCase    )
       a						:List[str] 							=       (out - labels).mean()
       loss.backward()

       # re-instantiate the model now enabling gradient checkpointing
       a						:Optional[Any] 							=       self.model_class(**_lowerCamelCase    )
       # clone model
       model_a.load_state_dict(model.state_dict()    )
       model_a.to(_lowerCamelCase    )
       model_a.enable_gradient_checkpointing()

       assert model_a.is_gradient_checkpointing and model_a.training

       a						:Optional[int] 							=       model_a(**_lowerCamelCase    ).sample
       # run the backwards pass on the model. For backwards pass, for simplicity purpose,
       # we won't calculate the loss and rather backprop on out.sum()
       model_a.zero_grad()
       a						:List[Any] 							=       (out_a - labels).mean()
       loss_a.backward()

       # compare the output and parameters gradients
       self.assertTrue((loss - loss_a).abs() < 1e-5    )
       a						:str 							=       dict(model.named_parameters()    )
       a						:Any 							=       dict(model_a.named_parameters()    )
       for name, param in named_params.items():
             self.assertTrue(torch_all_close(param.grad.data					,  named_params_a[name].grad.data					,  atol=5e-5    )    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a							,				a						:Optional[Any] 							=       AutoencoderKL.from_pretrained('''fusing/autoencoder-kl-dummy'''					,  output_loading_info=_lowerCamelCase    )
       self.assertIsNotNone(_lowerCamelCase    )
       self.assertEqual(len(loading_info['''missing_keys''']    )					,  0    )

       model.to(_lowerCamelCase    )
       a						:str 							=       model(**self.dummy_input    )

       assert image is not None, "Make sure output is not None"




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:int 							=       AutoencoderKL.from_pretrained('''fusing/autoencoder-kl-dummy'''    )
       a						:Optional[int] 							=       model.to(_lowerCamelCase    )
       model.eval()

       if torch_device == "mps":
             a						:List[str] 							=       torch.manual_seed(0    )
       else:
             a						:List[Any] 							=       torch.Generator(device=_lowerCamelCase    ).manual_seed(0    )

       a						:Any 							=       torch.randn(
           1					,  model.config.in_channels					,  model.config.sample_size					,  model.config.sample_size					,  generator=torch.manual_seed(0    )					,  )
       a						:List[Any] 							=       image.to(_lowerCamelCase    )
       with torch.no_grad():
             a						:Tuple 							=       model(_lowerCamelCase					,  sample_posterior=_lowerCamelCase					,  generator=_lowerCamelCase    ).sample

       a						:Any 							=       output[0, -1, -3:, -3:].flatten().cpu()

       # Since the VAE Gaussian prior's generator is seeded on the appropriate device,
       # the expected output slices are not the same for CPU and GPU.
       if torch_device == "mps":
             a						:Optional[Any] 							=       torch.tensor(
                 [
                     -4.00_78e-01,
                     -3.83_23e-04,
                     -1.26_81e-01,
                     -1.14_62e-01,
                     2.00_95e-01,
                     1.08_93e-01,
                     -8.82_47e-02,
                     -3.03_61e-01,
                     -9.86_44e-03,
                 ]    )
       elif torch_device == "cpu":
             a						:Tuple 							=       torch.tensor(
                 [-0.1352, 0.0878, 0.0419, -0.0818, -0.1069, 0.0688, -0.1458, -0.4446, -0.0026]    )
       else:
             a						:Optional[int] 							=       torch.tensor(
                 [-0.2421, 0.4642, 0.2507, -0.0438, 0.0682, 0.3160, -0.2018, -0.0727, 0.2485]    )

       self.assertTrue(torch_all_close(_lowerCamelCase					,  _lowerCamelCase					,  rtol=1e-2    )    )







@slow
class 		_snake_case  (      unittest.TestCase							):




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase    ):
       return F'''gaussian_noise_s={seed}_shape={'_'.join([str(_lowerCamelCase    ) for s in shape]    )}.npy'''




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       # clean up the VRAM after each test
       super().tearDown()
       gc.collect()
       torch.cuda.empty_cache()




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase=0					,  _lowerCamelCase=(4, 3, 512, 512)					,  _lowerCamelCase=False    ):
       a						:Union[str, Any] 							=       torch.floataa if fpaa else torch.floataa
       a						:Any 							=       torch.from_numpy(load_hf_numpy(self.get_file_format(_lowerCamelCase					,  _lowerCamelCase    )    )    ).to(_lowerCamelCase    ).to(_lowerCamelCase    )
       return image




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase="CompVis/stable-diffusion-v1-4"					,  _lowerCamelCase=False    ):
       a						:Optional[Any] 							=       '''fp16''' if fpaa else None
       a						:Optional[int] 							=       torch.floataa if fpaa else torch.floataa

       a						:Any 							=       AutoencoderKL.from_pretrained(
           _lowerCamelCase					,  subfolder='''vae'''					,  torch_dtype=_lowerCamelCase					,  revision=_lowerCamelCase					,  )
       model.to(_lowerCamelCase    ).eval()

       return model




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase=0    ):
       if torch_device == "mps":
             return torch.manual_seed(_lowerCamelCase    )
       return torch.Generator(device=_lowerCamelCase    ).manual_seed(_lowerCamelCase    )




 @parameterized.expand(
     [
         # fmt: off
         [33, [-0.1603, 0.9878, -0.0495, -0.0790, -0.2709, 0.8375, -0.2060, -0.0824], [-0.2395, 0.0098, 0.0102, -0.0709, -0.2840, -0.0274, -0.0718, -0.1824]],
         [47, [-0.2376, 0.1168, 0.1332, -0.4840, -0.2508, -0.0791, -0.0493, -0.4089], [0.0350, 0.0847, 0.0467, 0.0344, -0.0842, -0.0547, -0.0633, -0.1131]],
         # fmt: on
     ]    )
 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:Optional[int] 							=       self.get_sd_vae_model()
       a						:str 							=       self.get_sd_image(_lowerCamelCase    )
       a						:Dict 							=       self.get_generator(_lowerCamelCase    )

       with torch.no_grad():
             a						:int 							=       model(_lowerCamelCase					,  generator=_lowerCamelCase					,  sample_posterior=_lowerCamelCase    ).sample

       assert sample.shape == image.shape

       a						:List[str] 							=       sample[-1, -2:, -2:, :2].flatten().float().cpu()
       a						:Optional[int] 							=       torch.tensor(expected_slice_mps if torch_device == '''mps''' else expected_slice    )

       assert torch_all_close(_lowerCamelCase					,  _lowerCamelCase					,  atol=3e-3    )




 @parameterized.expand(
     [
         # fmt: off
         [33, [-0.0513, 0.0289, 1.3799, 0.2166, -0.2573, -0.0871, 0.5103, -0.0999]],
         [47, [-0.4128, -0.1320, -0.3704, 0.1965, -0.4116, -0.2332, -0.3340, 0.2247]],
         # fmt: on
     ]    )
 @require_torch_gpu
 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:List[Any] 							=       self.get_sd_vae_model(fpaa=_lowerCamelCase    )
       a						:str 							=       self.get_sd_image(_lowerCamelCase					,  fpaa=_lowerCamelCase    )
       a						:Any 							=       self.get_generator(_lowerCamelCase    )

       with torch.no_grad():
             a						:int 							=       model(_lowerCamelCase					,  generator=_lowerCamelCase					,  sample_posterior=_lowerCamelCase    ).sample

       assert sample.shape == image.shape

       a						:Optional[int] 							=       sample[-1, -2:, :2, -2:].flatten().float().cpu()
       a						:int 							=       torch.tensor(_lowerCamelCase    )

       assert torch_all_close(_lowerCamelCase					,  _lowerCamelCase					,  atol=1e-2    )




 @parameterized.expand(
     [
         # fmt: off
         [33, [-0.1609, 0.9866, -0.0487, -0.0777, -0.2716, 0.8368, -0.2055, -0.0814], [-0.2395, 0.0098, 0.0102, -0.0709, -0.2840, -0.0274, -0.0718, -0.1824]],
         [47, [-0.2377, 0.1147, 0.1333, -0.4841, -0.2506, -0.0805, -0.0491, -0.4085], [0.0350, 0.0847, 0.0467, 0.0344, -0.0842, -0.0547, -0.0633, -0.1131]],
         # fmt: on
     ]    )
 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:Optional[int] 							=       self.get_sd_vae_model()
       a						:Optional[Any] 							=       self.get_sd_image(_lowerCamelCase    )

       with torch.no_grad():
             a						:str 							=       model(_lowerCamelCase    ).sample

       assert sample.shape == image.shape

       a						:Union[str, Any] 							=       sample[-1, -2:, -2:, :2].flatten().float().cpu()
       a						:Optional[int] 							=       torch.tensor(expected_slice_mps if torch_device == '''mps''' else expected_slice    )

       assert torch_all_close(_lowerCamelCase					,  _lowerCamelCase					,  atol=3e-3    )




 @parameterized.expand(
     [
         # fmt: off
         [13, [-0.2051, -0.1803, -0.2311, -0.2114, -0.3292, -0.3574, -0.2953, -0.3323]],
         [37, [-0.2632, -0.2625, -0.2199, -0.2741, -0.4539, -0.4990, -0.3720, -0.4925]],
         # fmt: on
     ]    )
 @require_torch_gpu
 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:int 							=       self.get_sd_vae_model()
       a						:str 							=       self.get_sd_image(_lowerCamelCase					,  shape=(3, 4, 64, 64)    )

       with torch.no_grad():
             a						:Union[str, Any] 							=       model.decode(_lowerCamelCase    ).sample

       assert list(sample.shape    ) == [3, 3, 512, 512]

       a						:str 							=       sample[-1, -2:, :2, -2:].flatten().cpu()
       a						:List[Any] 							=       torch.tensor(_lowerCamelCase    )

       assert torch_all_close(_lowerCamelCase					,  _lowerCamelCase					,  atol=1e-3    )




 @parameterized.expand(
     [
         # fmt: off
         [27, [-0.0369, 0.0207, -0.0776, -0.0682, -0.1747, -0.1930, -0.1465, -0.2039]],
         [16, [-0.1628, -0.2134, -0.2747, -0.2642, -0.3774, -0.4404, -0.3687, -0.4277]],
         # fmt: on
     ]    )
 @require_torch_gpu
 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:int 							=       self.get_sd_vae_model(fpaa=_lowerCamelCase    )
       a						:List[Any] 							=       self.get_sd_image(_lowerCamelCase					,  shape=(3, 4, 64, 64)					,  fpaa=_lowerCamelCase    )

       with torch.no_grad():
             a						:List[Any] 							=       model.decode(_lowerCamelCase    ).sample

       assert list(sample.shape    ) == [3, 3, 512, 512]

       a						:Any 							=       sample[-1, -2:, :2, -2:].flatten().float().cpu()
       a						:Any 							=       torch.tensor(_lowerCamelCase    )

       assert torch_all_close(_lowerCamelCase					,  _lowerCamelCase					,  atol=5e-3    )




 @parameterized.expand([(13,), (16,), (27,)]    )
 @require_torch_gpu
 @unittest.skipIf(not is_xformers_available()					,  reason='''xformers is not required when using PyTorch 2.0.'''    )
 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       a						:Dict 							=       self.get_sd_vae_model(fpaa=_lowerCamelCase    )
       a						:Dict 							=       self.get_sd_image(_lowerCamelCase					,  shape=(3, 4, 64, 64)					,  fpaa=_lowerCamelCase    )

       with torch.no_grad():
             a						:Union[str, Any] 							=       model.decode(_lowerCamelCase    ).sample

       model.enable_xformers_memory_efficient_attention()
       with torch.no_grad():
             a						:Optional[int] 							=       model.decode(_lowerCamelCase    ).sample

       assert list(sample.shape    ) == [3, 3, 512, 512]

       assert torch_all_close(_lowerCamelCase					,  _lowerCamelCase					,  atol=1e-1    )




 @parameterized.expand([(13,), (16,), (37,)]    )
 @require_torch_gpu
 @unittest.skipIf(not is_xformers_available()					,  reason='''xformers is not required when using PyTorch 2.0.'''    )
 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       a						:Union[str, Any] 							=       self.get_sd_vae_model()
       a						:Union[str, Any] 							=       self.get_sd_image(_lowerCamelCase					,  shape=(3, 4, 64, 64)    )

       with torch.no_grad():
             a						:Dict 							=       model.decode(_lowerCamelCase    ).sample

       model.enable_xformers_memory_efficient_attention()
       with torch.no_grad():
             a						:Tuple 							=       model.decode(_lowerCamelCase    ).sample

       assert list(sample.shape    ) == [3, 3, 512, 512]

       assert torch_all_close(_lowerCamelCase					,  _lowerCamelCase					,  atol=1e-2    )




 @parameterized.expand(
     [
         # fmt: off
         [33, [-0.3001, 0.0918, -2.6984, -3.9720, -3.2099, -5.0353, 1.7338, -0.2065, 3.4267]],
         [47, [-1.5030, -4.3871, -6.0355, -9.1157, -1.6661, -2.7853, 2.1607, -5.0823, 2.5633]],
         # fmt: on
     ]    )
 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:Union[str, Any] 							=       self.get_sd_vae_model()
       a						:Optional[int] 							=       self.get_sd_image(_lowerCamelCase    )
       a						:Dict 							=       self.get_generator(_lowerCamelCase    )

       with torch.no_grad():
             a						:Optional[Any] 							=       model.encode(_lowerCamelCase    ).latent_dist
             a						:int 							=       dist.sample(generator=_lowerCamelCase    )

       assert list(sample.shape    ) == [image.shape[0], 4] + [i // 8 for i in image.shape[2:]]

       a						:Any 							=       sample[0, -1, -3:, -3:].flatten().cpu()
       a						:List[str] 							=       torch.tensor(_lowerCamelCase    )

       a						:int 							=       3e-3 if torch_device != '''mps''' else 1e-2
       assert torch_all_close(_lowerCamelCase					,  _lowerCamelCase					,  atol=_lowerCamelCase    )













import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin






class 		lowercase_					(	lowercase ,  unittest.TestCase   ):






      '''simple docstring'''




      __snake_case       =  ProphetNetTokenizer
      __snake_case       =  False







      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->Optional[Any]:




       """simple docstring"""


       super().setUp()

       a		  =      [
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       a		  =      os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['''vocab_file''']  )
       with open(self.vocab_file ,					'''w''' ,					encoding='''utf-8'''  ) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )







      def       __lowerCAmelCase					(      self      :						List[str] ,					__UpperCAmelCase      :						str  )    ->Dict:




       """simple docstring"""


       a		  =      '''UNwant\u00E9d,running'''
       a		  =      '''unwanted, running'''
       return input_text, output_text







      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->Optional[Any]:




       """simple docstring"""


       a		  =      self.tokenizer_class(self.vocab_file  )

       a		  =      tokenizer.tokenize('''UNwant\u00E9d,running'''  )
       self.assertListEqual(__UpperCAmelCase ,					['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase  ) ,					[9, 6, 7, 12, 10, 11]  )







      def       __lowerCAmelCase					(      self      :						int  )    ->Any:




       """simple docstring"""


       a		  =      BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  ) ,					['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->int:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ) ,					['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ) ,					['''hello''']  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->Optional[int]:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ) ,					['''h\u00E9llo''']  )







      def       __lowerCAmelCase					(      self      :						Dict  )    ->str:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ) ,					['''hello''']  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->Dict:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ) ,					['''hello''']  )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Optional[Any]:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ) ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Tuple:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )







      def       __lowerCAmelCase					(      self      :						int  )    ->Optional[int]:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->int:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					never_split=['''[UNK]''']  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  ) ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->int:




       """simple docstring"""


       a		  =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       a		  =      {}
       for i, token in enumerate(__UpperCAmelCase  ):
        a		  =      i
       a		  =      WordpieceTokenizer(vocab=__UpperCAmelCase ,					unk_token='''[UNK]'''  )

       self.assertListEqual(tokenizer.tokenize(''''''  ) ,					[]  )

       self.assertListEqual(tokenizer.tokenize('''unwanted running'''  ) ,					['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

       self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  ) ,					['''[UNK]''', '''runn''', '''##ing''']  )







      @require_torch
      def       __lowerCAmelCase					(      self      :						int  )    ->int:




       """simple docstring"""


       a		  =      self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

       a		  =      ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
       a		  =      [1_037, 2_146, 20_423, 2_005, 7_680, 7_849, 3_989, 1_012, 102]
       a		  =      tokenizer(__UpperCAmelCase ,					padding=__UpperCAmelCase ,					return_tensors='''pt'''  )
       self.assertIsInstance(__UpperCAmelCase ,					__UpperCAmelCase  )
       a		  =      list(batch.input_ids.numpy()[0]  )
       self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase  )

       self.assertEqual((2, 9) ,					batch.input_ids.shape  )
       self.assertEqual((2, 9) ,					batch.attention_mask.shape  )







      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->List[str]:




       """simple docstring"""


       self.assertTrue(_is_whitespace(''' '''  )  )
       self.assertTrue(_is_whitespace('''\t'''  )  )
       self.assertTrue(_is_whitespace('''\r'''  )  )
       self.assertTrue(_is_whitespace('''\n'''  )  )
       self.assertTrue(_is_whitespace('''\u00A0'''  )  )

       self.assertFalse(_is_whitespace('''A'''  )  )
       self.assertFalse(_is_whitespace('''-'''  )  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->List[str]:




       """simple docstring"""


       self.assertTrue(_is_control('''\u0005'''  )  )

       self.assertFalse(_is_control('''A'''  )  )
       self.assertFalse(_is_control(''' '''  )  )
       self.assertFalse(_is_control('''\t'''  )  )
       self.assertFalse(_is_control('''\r'''  )  )







      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->List[str]:




       """simple docstring"""


       self.assertTrue(_is_punctuation('''-'''  )  )
       self.assertTrue(_is_punctuation('''$'''  )  )
       self.assertTrue(_is_punctuation('''`'''  )  )
       self.assertTrue(_is_punctuation('''.'''  )  )

       self.assertFalse(_is_punctuation('''A'''  )  )
       self.assertFalse(_is_punctuation(''' '''  )  )







      @slow
      def       __lowerCAmelCase					(      self      :						List[str]  )    ->List[str]:




       """simple docstring"""


       a		  =      self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

       a		  =      tokenizer.encode('''sequence builders''' ,					add_special_tokens=__UpperCAmelCase  )
       a		  =      tokenizer.encode('''multi-sequence build''' ,					add_special_tokens=__UpperCAmelCase  )

       a		  =      tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase  )
       a		  =      tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase ,					__UpperCAmelCase  )

       assert encoded_sentence == text + [102]
       assert encoded_pair == text + [102] + text_a + [102]


