"""simple docstring"""


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class   UpperCAmelCase_	(				_UpperCamelCase,    unittest.TestCase   ):
    snake_case_											=      ElectraTokenizer
    snake_case_											=      ElectraTokenizerFast
    snake_case_											=      True
    snake_case_											=      True
    snake_case_											=      filter_non_english
    def 		_lowerCamelCase      (	self			:       Tuple		)						->    List[Any]:
       super().setUp()

       _lowercase							     =       [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       _lowercase							     =       os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES["vocab_file"]		)
       with open(self.vocab_file   ,						"w"   ,						encoding="utf-8"		) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)
    def 		_lowerCamelCase      (	self			:       List[Any]   ,						_lowercase			:       List[str]		)						->    Union[str, Any]:
       _lowercase							     =       "UNwant\u00E9d,running"
       _lowercase							     =       "unwanted, running"
       return input_text, output_text
    def 		_lowerCamelCase      (	self			:       int		)						->    Dict:
       _lowercase							     =       self.tokenizer_class(self.vocab_file		)

       _lowercase							     =       tokenizer.tokenize("UNwant\u00E9d,running"		)
       self.assertListEqual(__lowerCamelCase   ,						["un", "##want", "##ed", ",", "runn", "##ing"]		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase		)   ,						[9, 6, 7, 1_2, 1_0, 1_1]		)
    def 		_lowerCamelCase      (	self			:       List[Any]		)						->    Optional[int]:
       if not self.test_rust_tokenizer:
          return

       _lowercase							     =       self.get_tokenizer()
       _lowercase							     =       self.get_rust_tokenizer()

       _lowercase							     =       "UNwant\u00E9d,running"

       _lowercase							     =       tokenizer.tokenize(__lowerCamelCase		)
       _lowercase							     =       rust_tokenizer.tokenize(__lowerCamelCase		)
       self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)

       _lowercase							     =       tokenizer.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase		)
       _lowercase							     =       rust_tokenizer.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase		)
       self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)

       _lowercase							     =       self.get_rust_tokenizer()
       _lowercase							     =       tokenizer.encode(__lowerCamelCase		)
       _lowercase							     =       rust_tokenizer.encode(__lowerCamelCase		)
       self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)

       # With lower casing
       _lowercase							     =       self.get_tokenizer(do_lower_case=__lowerCamelCase		)
       _lowercase							     =       self.get_rust_tokenizer(do_lower_case=__lowerCamelCase		)

       _lowercase							     =       "UNwant\u00E9d,running"

       _lowercase							     =       tokenizer.tokenize(__lowerCamelCase		)
       _lowercase							     =       rust_tokenizer.tokenize(__lowerCamelCase		)
       self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)

       _lowercase							     =       tokenizer.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase		)
       _lowercase							     =       rust_tokenizer.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase		)
       self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)

       _lowercase							     =       self.get_rust_tokenizer()
       _lowercase							     =       tokenizer.encode(__lowerCamelCase		)
       _lowercase							     =       rust_tokenizer.encode(__lowerCamelCase		)
       self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)
    def 		_lowerCamelCase      (	self			:       Union[str, Any]		)						->    List[str]:
       _lowercase							     =       BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)   ,						["ah", "\u535A", "\u63A8", "zz"]		)
    def 		_lowerCamelCase      (	self			:       str		)						->    Union[str, Any]:
       _lowercase							     =       BasicTokenizer(do_lower_case=__lowerCamelCase		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)   ,						["hello", "!", "how", "are", "you", "?"]		)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)   ,						["hello"]		)
    def 		_lowerCamelCase      (	self			:       Dict		)						->    str:
       _lowercase							     =       BasicTokenizer(do_lower_case=__lowerCamelCase   ,						strip_accents=__lowerCamelCase		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)   ,						["hällo", "!", "how", "are", "you", "?"]		)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)   ,						["h\u00E9llo"]		)
    def 		_lowerCamelCase      (	self			:       Optional[Any]		)						->    int:
       _lowercase							     =       BasicTokenizer(do_lower_case=__lowerCamelCase   ,						strip_accents=__lowerCamelCase		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)   ,						["hallo", "!", "how", "are", "you", "?"]		)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)   ,						["hello"]		)
    def 		_lowerCamelCase      (	self			:       Tuple		)						->    int:
       _lowercase							     =       BasicTokenizer(do_lower_case=__lowerCamelCase		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)   ,						["hallo", "!", "how", "are", "you", "?"]		)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)   ,						["hello"]		)
    def 		_lowerCamelCase      (	self			:       List[str]		)						->    str:
       _lowercase							     =       BasicTokenizer(do_lower_case=__lowerCamelCase		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)   ,						["HeLLo", "!", "how", "Are", "yoU", "?"]		)
    def 		_lowerCamelCase      (	self			:       Optional[Any]		)						->    Optional[int]:
       _lowercase							     =       BasicTokenizer(do_lower_case=__lowerCamelCase   ,						strip_accents=__lowerCamelCase		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)   ,						["HäLLo", "!", "how", "Are", "yoU", "?"]		)
    def 		_lowerCamelCase      (	self			:       int		)						->    Optional[int]:
       _lowercase							     =       BasicTokenizer(do_lower_case=__lowerCamelCase   ,						strip_accents=__lowerCamelCase		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)   ,						["HaLLo", "!", "how", "Are", "yoU", "?"]		)
    def 		_lowerCamelCase      (	self			:       List[Any]		)						->    str:
       _lowercase							     =       BasicTokenizer(do_lower_case=__lowerCamelCase   ,						never_split=["[UNK]"]		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)   ,						["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)
    def 		_lowerCamelCase      (	self			:       List[str]		)						->    Optional[Any]:
       _lowercase							     =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       _lowercase							     =       {}
       for i, token in enumerate(__lowerCamelCase		):
          _lowercase							     =       i
       _lowercase							     =       WordpieceTokenizer(vocab=__lowerCamelCase   ,						unk_token="[UNK]"		)

       self.assertListEqual(tokenizer.tokenize(""		)   ,						[]		)

       self.assertListEqual(tokenizer.tokenize("unwanted running"		)   ,						["un", "##want", "##ed", "runn", "##ing"]		)

       self.assertListEqual(tokenizer.tokenize("unwantedX running"		)   ,						["[UNK]", "runn", "##ing"]		)
    def 		_lowerCamelCase      (	self			:       Optional[int]		)						->    int:
       self.assertTrue(_is_whitespace(" "		)		)
       self.assertTrue(_is_whitespace("\t"		)		)
       self.assertTrue(_is_whitespace("\r"		)		)
       self.assertTrue(_is_whitespace("\n"		)		)
       self.assertTrue(_is_whitespace("\u00A0"		)		)

       self.assertFalse(_is_whitespace("A"		)		)
       self.assertFalse(_is_whitespace("-"		)		)
    def 		_lowerCamelCase      (	self			:       Union[str, Any]		)						->    Optional[Any]:
       self.assertTrue(_is_control("\u0005"		)		)

       self.assertFalse(_is_control("A"		)		)
       self.assertFalse(_is_control(" "		)		)
       self.assertFalse(_is_control("\t"		)		)
       self.assertFalse(_is_control("\r"		)		)
    def 		_lowerCamelCase      (	self			:       int		)						->    Any:
       self.assertTrue(_is_punctuation("-"		)		)
       self.assertTrue(_is_punctuation("$"		)		)
       self.assertTrue(_is_punctuation("`"		)		)
       self.assertTrue(_is_punctuation("."		)		)

       self.assertFalse(_is_punctuation("A"		)		)
       self.assertFalse(_is_punctuation(" "		)		)
    def 		_lowerCamelCase      (	self			:       List[Any]		)						->    Optional[Any]:
       _lowercase							     =       self.get_tokenizer()
       _lowercase							     =       self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(__lowerCamelCase		) for t in ["Test", "\xad", "test"]]   ,						[["[UNK]"], [], ["[UNK]"]]		)

       self.assertListEqual(
           [rust_tokenizer.tokenize(__lowerCamelCase		) for t in ["Test", "\xad", "test"]]   ,						[["[UNK]"], [], ["[UNK]"]]		)
    @slow
    def 		_lowerCamelCase      (	self			:       Union[str, Any]		)						->    Any:
       _lowercase							     =       self.tokenizer_class.from_pretrained("google/electra-base-discriminator"		)

       _lowercase							     =       tokenizer.encode("sequence builders"   ,						add_special_tokens=__lowerCamelCase		)
       _lowercase							     =       tokenizer.encode("multi-sequence build"   ,						add_special_tokens=__lowerCamelCase		)

       _lowercase							     =       tokenizer.build_inputs_with_special_tokens(__lowerCamelCase		)
       _lowercase							     =       tokenizer.build_inputs_with_special_tokens(__lowerCamelCase   ,						__lowerCamelCase		)

       assert encoded_sentence == [1_0_1] + text + [1_0_2]
       assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
    def 		_lowerCamelCase      (	self			:       str		)						->    Union[str, Any]:
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
             _lowercase							     =       self.rust_tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase		)

             _lowercase							     =       f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
             _lowercase							     =       tokenizer_r.encode_plus(
                 __lowerCamelCase   ,						return_attention_mask=__lowerCamelCase   ,						return_token_type_ids=__lowerCamelCase   ,						return_offsets_mapping=__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase   ,						)

             _lowercase							     =       tokenizer_r.do_lower_case if hasattr(__lowerCamelCase   ,						"do_lower_case"		) else False
             _lowercase							     =       (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "Allen"),
                     ((2_1, 2_3), "##NL"),
                     ((2_3, 2_4), "##P"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "allen"),
                     ((2_1, 2_3), "##nl"),
                     ((2_3, 2_4), "##p"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]   ,						tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
             self.assertEqual([e[0] for e in expected_results]   ,						tokens["offset_mapping"]		)




    def 		_lowerCamelCase      (	self			:       Optional[Any]		)						->    Union[str, Any]:
       _lowercase							     =       ["的", "人", "有"]
       _lowercase							     =       "".join(__lowerCamelCase		)
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
             _lowercase							     =       True
             _lowercase							     =       self.tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase		)
             _lowercase							     =       self.rust_tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase		)

             _lowercase							     =       tokenizer_p.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase		)
             _lowercase							     =       tokenizer_r.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase		)

             _lowercase							     =       tokenizer_r.convert_ids_to_tokens(__lowerCamelCase		)
             _lowercase							     =       tokenizer_p.convert_ids_to_tokens(__lowerCamelCase		)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)
             self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)

             _lowercase							     =       False
             _lowercase							     =       self.rust_tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase		)
             _lowercase							     =       self.tokenizer_class.from_pretrained(__lowerCamelCase   ,						**__lowerCamelCase		)

             _lowercase							     =       tokenizer_r.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase		)
             _lowercase							     =       tokenizer_p.encode(__lowerCamelCase   ,						add_special_tokens=__lowerCamelCase		)

             _lowercase							     =       tokenizer_r.convert_ids_to_tokens(__lowerCamelCase		)
             _lowercase							     =       tokenizer_p.convert_ids_to_tokens(__lowerCamelCase		)

             # it is expected that only the first Chinese character is not preceded by "##".
             _lowercase							     =       [
                 f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__lowerCamelCase		)
             ]
             self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)
             self.assertListEqual(__lowerCamelCase   ,						__lowerCamelCase		)





import tempfile
import unittest

import torch

from diffusers import UNetaDConditionModel
from diffusers.training_utils import EMAModel
from diffusers.utils.testing_utils import enable_full_determinism, skip_mps, torch_device


enable_full_determinism()




class        snake_case__				(unittest.TestCase  ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			:				Optional[int]									=   """hf-internal-testing/tiny-stable-diffusion-pipe"""
				SCREAMING_SNAKE_CASE_			:				str									=   1
				SCREAMING_SNAKE_CASE_			:				Dict									=   77
				SCREAMING_SNAKE_CASE_			:				List[str]									=   32
				SCREAMING_SNAKE_CASE_			:				List[str]									=   4
				SCREAMING_SNAKE_CASE_			:				Union[str, Any]									=   64
				SCREAMING_SNAKE_CASE_			:				int									=   torch.manual_seed(0  )





				def 				__UpperCAmelCase				( self							:	List[Any]   , __lowerCamelCase							:	List[Any]=0.9_999				)   ->	Tuple:
							a       =     UNetaDConditionModel.from_pretrained(self.model_id   , subfolder="unet"				)
							a       =     unet.to(__lowerCamelCase				)
							a       =     EMAModel(unet.parameters()   , decay=__lowerCamelCase   , model_cls=__lowerCamelCase   , model_config=unet.config				)
							return unet, ema_unet





				def 				__UpperCAmelCase				( self							:	Tuple				)   ->	Any:
							a       =     torch.randn(
							    self.batch_size   , self.num_in_channels   , self.latent_height   , self.latent_width   , generator=self.generator				).to(__lowerCamelCase				)
							a       =     torch.randint(0   , 10_00   , size=(self.batch_size,)   , generator=self.generator				).to(__lowerCamelCase				)
							a       =     torch.randn(
							    self.batch_size   , self.prompt_length   , self.text_encoder_hidden_dim   , generator=self.generator				).to(__lowerCamelCase				)
							return noisy_latents, timesteps, encoder_hidden_states





				def 				__UpperCAmelCase				( self							:	Optional[int]   , __lowerCamelCase							:	Any				)   ->	Optional[int]:
							a       =     {}
							for k, param in unet.state_dict().items():
										a       =     torch.randn_like(__lowerCamelCase				) + (param * torch.randn_like(__lowerCamelCase				))
										updated_state_dict.update({k: updated_param}				)
							unet.load_state_dict(__lowerCamelCase				)
							return unet





				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	Union[str, Any]:
							a       ,						a       =     self.get_models()
							# Take the first (hypothetical) EMA step.
							ema_unet.step(unet.parameters()				)
							assert ema_unet.optimization_step == 1

							# Take two more.
							for _ in range(2				):
										ema_unet.step(unet.parameters()				)
							assert ema_unet.optimization_step == 3





				def 				__UpperCAmelCase				( self							:	Optional[int]				)   ->	Optional[int]:
							a       ,						a       =     self.get_models()
							# Since the `unet` is not being updated (i.e., backprop'd)
							# there won't be any difference between the `params` of `unet`
							# and `ema_unet` even if we call `ema_unet.step(unet.parameters())`.
							ema_unet.step(unet.parameters()				)
							a       =     list(unet.parameters()				)
							for s_param, param in zip(ema_unet.shadow_params   , __lowerCamelCase				):
										assert torch.allclose(__lowerCamelCase   , __lowerCamelCase				)

							# The above holds true even if we call `ema.step()` multiple times since
							# `unet` params are still not being updated.
							for _ in range(4				):
										ema_unet.step(unet.parameters()				)
							for s_param, param in zip(ema_unet.shadow_params   , __lowerCamelCase				):
										assert torch.allclose(__lowerCamelCase   , __lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Any				)   ->	Dict:
							a       ,						a       =     self.get_models()
							# Here we simulate the parameter updates for `unet`. Since there might
							# be some parameters which are initialized to zero we take extra care to
							# initialize their values to something non-zero before the multiplication.
							a       =     self.simulate_backprop(__lowerCamelCase				)

							# Take the EMA step.
							ema_unet.step(unet_pseudo_updated_step_one.parameters()				)

							# Now the EMA'd parameters won't be equal to the original model parameters.
							a       =     list(unet_pseudo_updated_step_one.parameters()				)
							for s_param, param in zip(ema_unet.shadow_params   , __lowerCamelCase				):
										assert ~torch.allclose(__lowerCamelCase   , __lowerCamelCase				)

							# Ensure this is the case when we take multiple EMA steps.
							for _ in range(4				):
										ema_unet.step(unet.parameters()				)
							for s_param, param in zip(ema_unet.shadow_params   , __lowerCamelCase				):
										assert ~torch.allclose(__lowerCamelCase   , __lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	Any:
							# If we call EMA step after a backpropagation consecutively for two times,
							# the shadow params from those two steps should be different.
							a       ,						a       =     self.get_models()

							# First backprop + EMA
							a       =     self.simulate_backprop(__lowerCamelCase				)
							ema_unet.step(unet_step_one.parameters()				)
							a       =     ema_unet.shadow_params

							# Second backprop + EMA
							a       =     self.simulate_backprop(__lowerCamelCase				)
							ema_unet.step(unet_step_two.parameters()				)
							a       =     ema_unet.shadow_params

							for step_one, step_two in zip(__lowerCamelCase   , __lowerCamelCase				):
										assert ~torch.allclose(__lowerCamelCase   , __lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Any				)   ->	List[Any]:
							# If there's no decay even if there are backprops, EMA steps
							# won't take any effect i.e., the shadow params would remain the
							# same.
							a       ,						a       =     self.get_models(decay=0.0				)
							a       =     self.simulate_backprop(__lowerCamelCase				)
							ema_unet.step(unet_step_one.parameters()				)
							a       =     ema_unet.shadow_params

							a       =     self.simulate_backprop(__lowerCamelCase				)
							ema_unet.step(unet_step_two.parameters()				)
							a       =     ema_unet.shadow_params

							for step_one, step_two in zip(__lowerCamelCase   , __lowerCamelCase				):
										assert torch.allclose(__lowerCamelCase   , __lowerCamelCase				)





				@skip_mps
				def 				__UpperCAmelCase				( self							:	List[str]				)   ->	Dict:
							a       ,						a       =     self.get_models()
							a       ,						a       ,						a       =     self.get_dummy_inputs()

							with tempfile.TemporaryDirectory() as tmpdir:
										ema_unet.save_pretrained(__lowerCamelCase				)
										a       =     UNetaDConditionModel.from_pretrained(__lowerCamelCase   , model_cls=__lowerCamelCase				)
										a       =     loaded_unet.to(unet.device				)

							# Since no EMA step has been performed the outputs should match.
							a       =     unet(__lowerCamelCase   , __lowerCamelCase   , __lowerCamelCase				).sample
							a       =     loaded_unet(__lowerCamelCase   , __lowerCamelCase   , __lowerCamelCase				).sample

							assert torch.allclose(__lowerCamelCase   , __lowerCamelCase   , atol=1e-4				)






