
import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class    _UpperCamelCase(   SCREAMING_SNAKE_CASE		,						unittest.TestCase				):
						__A:						int				   =      ElectraTokenizer
						__A:						str				   =      ElectraTokenizerFast
						__A:						Optional[int]				   =      True
						__A:						Tuple				   =      True
						__A:						Union[str, Any]				   =      filter_non_english

						def 						a__ (					self		:			Any			):
													super().setUp()

													_UpperCAmelCase  :			Dict	       =		[
													    "[UNK]",
													    "[CLS]",
													    "[SEP]",
													    "[PAD]",
													    "[MASK]",
													    "want",
													    "##want",
													    "##ed",
													    "wa",
													    "un",
													    "runn",
													    "##ing",
													    ",",
													    "low",
													    "lowest",
													]
													_UpperCAmelCase  :			List[Any]	       =		os.path.join(self.tmpdirname	,						VOCAB_FILES_NAMES["vocab_file"]			)
													with open(self.vocab_file	,						"w"	,						encoding="utf-8"			) as vocab_writer:
																				vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)

						def 						a__ (					self		:			List[Any]	,						_lowerCamelCase		:			Any			):
													_UpperCAmelCase  :			Any	       =		"UNwant\u00E9d,running"
													_UpperCAmelCase  :			List[Any]	       =		"unwanted, running"
													return input_text, output_text

						def 						a__ (					self		:			Union[str, Any]			):
													_UpperCAmelCase  :			Tuple	       =		self.tokenizer_class(self.vocab_file			)

													_UpperCAmelCase  :			List[str]	       =		tokenizer.tokenize("UNwant\u00E9d,running"			)
													self.assertListEqual(_lowerCamelCase	,						["un", "##want", "##ed", ",", "runn", "##ing"]			)
													self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase			)	,						[9, 6, 7, 12, 10, 11]			)

						def 						a__ (					self		:			List[Any]			):
													if not self.test_rust_tokenizer:
																				return

													_UpperCAmelCase  :			Tuple	       =		self.get_tokenizer()
													_UpperCAmelCase  :			List[Any]	       =		self.get_rust_tokenizer()

													_UpperCAmelCase  :			Dict	       =		"UNwant\u00E9d,running"

													_UpperCAmelCase  :			List[Any]	       =		tokenizer.tokenize(_lowerCamelCase			)
													_UpperCAmelCase  :			List[str]	       =		rust_tokenizer.tokenize(_lowerCamelCase			)
													self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)

													_UpperCAmelCase  :			List[Any]	       =		tokenizer.encode(_lowerCamelCase	,						add_special_tokens=_lowerCamelCase			)
													_UpperCAmelCase  :			int	       =		rust_tokenizer.encode(_lowerCamelCase	,						add_special_tokens=_lowerCamelCase			)
													self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)

													_UpperCAmelCase  :			Dict	       =		self.get_rust_tokenizer()
													_UpperCAmelCase  :			List[Any]	       =		tokenizer.encode(_lowerCamelCase			)
													_UpperCAmelCase  :			Dict	       =		rust_tokenizer.encode(_lowerCamelCase			)
													self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)

													# With lower casing
													_UpperCAmelCase  :			int	       =		self.get_tokenizer(do_lower_case=_lowerCamelCase			)
													_UpperCAmelCase  :			str	       =		self.get_rust_tokenizer(do_lower_case=_lowerCamelCase			)

													_UpperCAmelCase  :			Any	       =		"UNwant\u00E9d,running"

													_UpperCAmelCase  :			str	       =		tokenizer.tokenize(_lowerCamelCase			)
													_UpperCAmelCase  :			List[Any]	       =		rust_tokenizer.tokenize(_lowerCamelCase			)
													self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)

													_UpperCAmelCase  :			int	       =		tokenizer.encode(_lowerCamelCase	,						add_special_tokens=_lowerCamelCase			)
													_UpperCAmelCase  :			List[str]	       =		rust_tokenizer.encode(_lowerCamelCase	,						add_special_tokens=_lowerCamelCase			)
													self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)

													_UpperCAmelCase  :			Tuple	       =		self.get_rust_tokenizer()
													_UpperCAmelCase  :			int	       =		tokenizer.encode(_lowerCamelCase			)
													_UpperCAmelCase  :			List[Any]	       =		rust_tokenizer.encode(_lowerCamelCase			)
													self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)

						def 						a__ (					self		:			Dict			):
													_UpperCAmelCase  :			Any	       =		BasicTokenizer()

													self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)	,						["ah", "\u535A", "\u63A8", "zz"]			)

						def 						a__ (					self		:			int			):
													_UpperCAmelCase  :			Any	       =		BasicTokenizer(do_lower_case=_lowerCamelCase			)

													self.assertListEqual(
													    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)	,						["hello", "!", "how", "are", "you", "?"]			)
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)	,						["hello"]			)

						def 						a__ (					self		:			str			):
													_UpperCAmelCase  :			List[str]	       =		BasicTokenizer(do_lower_case=_lowerCamelCase	,						strip_accents=_lowerCamelCase			)

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)	,						["hällo", "!", "how", "are", "you", "?"]			)
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)	,						["h\u00E9llo"]			)

						def 						a__ (					self		:			Dict			):
													_UpperCAmelCase  :			Dict	       =		BasicTokenizer(do_lower_case=_lowerCamelCase	,						strip_accents=_lowerCamelCase			)

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)	,						["hallo", "!", "how", "are", "you", "?"]			)
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)	,						["hello"]			)

						def 						a__ (					self		:			int			):
													_UpperCAmelCase  :			Optional[Any]	       =		BasicTokenizer(do_lower_case=_lowerCamelCase			)

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)	,						["hallo", "!", "how", "are", "you", "?"]			)
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)	,						["hello"]			)

						def 						a__ (					self		:			str			):
													_UpperCAmelCase  :			Optional[int]	       =		BasicTokenizer(do_lower_case=_lowerCamelCase			)

													self.assertListEqual(
													    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)	,						["HeLLo", "!", "how", "Are", "yoU", "?"]			)

						def 						a__ (					self		:			Optional[int]			):
													_UpperCAmelCase  :			Tuple	       =		BasicTokenizer(do_lower_case=_lowerCamelCase	,						strip_accents=_lowerCamelCase			)

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)	,						["HäLLo", "!", "how", "Are", "yoU", "?"]			)

						def 						a__ (					self		:			List[str]			):
													_UpperCAmelCase  :			Optional[Any]	       =		BasicTokenizer(do_lower_case=_lowerCamelCase	,						strip_accents=_lowerCamelCase			)

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)	,						["HaLLo", "!", "how", "Are", "yoU", "?"]			)

						def 						a__ (					self		:			int			):
													_UpperCAmelCase  :			int	       =		BasicTokenizer(do_lower_case=_lowerCamelCase	,						never_split=["[UNK]"]			)

													self.assertListEqual(
													    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)	,						["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)

						def 						a__ (					self		:			int			):
													_UpperCAmelCase  :			Union[str, Any]	       =		["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

													_UpperCAmelCase  :			Optional[Any]	       =		{}
													for i, token in enumerate(_lowerCamelCase			):
																				_UpperCAmelCase  :			int	       =		i
													_UpperCAmelCase  :			List[str]	       =		WordpieceTokenizer(vocab=_lowerCamelCase	,						unk_token="[UNK]"			)

													self.assertListEqual(tokenizer.tokenize(""			)	,						[]			)

													self.assertListEqual(tokenizer.tokenize("unwanted running"			)	,						["un", "##want", "##ed", "runn", "##ing"]			)

													self.assertListEqual(tokenizer.tokenize("unwantedX running"			)	,						["[UNK]", "runn", "##ing"]			)

						def 						a__ (					self		:			Tuple			):
													self.assertTrue(_is_whitespace(" "			)			)
													self.assertTrue(_is_whitespace("\t"			)			)
													self.assertTrue(_is_whitespace("\r"			)			)
													self.assertTrue(_is_whitespace("\n"			)			)
													self.assertTrue(_is_whitespace("\u00A0"			)			)

													self.assertFalse(_is_whitespace("A"			)			)
													self.assertFalse(_is_whitespace("-"			)			)

						def 						a__ (					self		:			List[Any]			):
													self.assertTrue(_is_control("\u0005"			)			)

													self.assertFalse(_is_control("A"			)			)
													self.assertFalse(_is_control(" "			)			)
													self.assertFalse(_is_control("\t"			)			)
													self.assertFalse(_is_control("\r"			)			)

						def 						a__ (					self		:			Tuple			):
													self.assertTrue(_is_punctuation("-"			)			)
													self.assertTrue(_is_punctuation("$"			)			)
													self.assertTrue(_is_punctuation("`"			)			)
													self.assertTrue(_is_punctuation("."			)			)

													self.assertFalse(_is_punctuation("A"			)			)
													self.assertFalse(_is_punctuation(" "			)			)

						def 						a__ (					self		:			int			):
													_UpperCAmelCase  :			List[str]	       =		self.get_tokenizer()
													_UpperCAmelCase  :			Tuple	       =		self.get_rust_tokenizer()

													# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
													self.assertListEqual([tokenizer.tokenize(_lowerCamelCase			) for t in ["Test", "\xad", "test"]]	,						[["[UNK]"], [], ["[UNK]"]]			)

													self.assertListEqual(
													    [rust_tokenizer.tokenize(_lowerCamelCase			) for t in ["Test", "\xad", "test"]]	,						[["[UNK]"], [], ["[UNK]"]]			)

						@slow
						def 						a__ (					self		:			List[str]			):
													_UpperCAmelCase  :			List[Any]	       =		self.tokenizer_class.from_pretrained("google/electra-base-discriminator"			)

													_UpperCAmelCase  :			Any	       =		tokenizer.encode("sequence builders"	,						add_special_tokens=_lowerCamelCase			)
													_UpperCAmelCase  :			int	       =		tokenizer.encode("multi-sequence build"	,						add_special_tokens=_lowerCamelCase			)

													_UpperCAmelCase  :			Dict	       =		tokenizer.build_inputs_with_special_tokens(_lowerCamelCase			)
													_UpperCAmelCase  :			List[Any]	       =		tokenizer.build_inputs_with_special_tokens(_lowerCamelCase	,						_lowerCamelCase			)

													assert encoded_sentence == [1_01] + text + [1_02]
													assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]

						def 						a__ (					self		:			Dict			):
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
																											_UpperCAmelCase  :			Dict	       =		self.rust_tokenizer_class.from_pretrained(_lowerCamelCase	,						**_lowerCamelCase			)

																											_UpperCAmelCase  :			Optional[int]	       =		f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																											_UpperCAmelCase  :			Any	       =		tokenizer_r.encode_plus(
																											    _lowerCamelCase	,						return_attention_mask=_lowerCamelCase	,						return_token_type_ids=_lowerCamelCase	,						return_offsets_mapping=_lowerCamelCase	,						add_special_tokens=_lowerCamelCase	,						)

																											_UpperCAmelCase  :			Dict	       =		tokenizer_r.do_lower_case if hasattr(_lowerCamelCase	,						"do_lower_case"			) else False
																											_UpperCAmelCase  :			Dict	       =		(
																											    [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), "A"),
																											        ((1, 2), ","),
																											        ((3, 5), "na"),
																											        ((5, 6), "##ï"),
																											        ((6, 8), "##ve"),
																											        ((9, 15), tokenizer_r.mask_token),
																											        ((16, 21), "Allen"),
																											        ((21, 23), "##NL"),
																											        ((23, 24), "##P"),
																											        ((25, 33), "sentence"),
																											        ((33, 34), "."),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											    if not do_lower_case
																											    else [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), "a"),
																											        ((1, 2), ","),
																											        ((3, 8), "naive"),
																											        ((9, 15), tokenizer_r.mask_token),
																											        ((16, 21), "allen"),
																											        ((21, 23), "##nl"),
																											        ((23, 24), "##p"),
																											        ((25, 33), "sentence"),
																											        ((33, 34), "."),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											)

																											self.assertEqual(
																											    [e[1] for e in expected_results]	,						tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
																											self.assertEqual([e[0] for e in expected_results]	,						tokens["offset_mapping"]			)






						def 						a__ (					self		:			Dict			):
													_UpperCAmelCase  :			Union[str, Any]	       =		["的", "人", "有"]
													_UpperCAmelCase  :			Union[str, Any]	       =		"".join(_lowerCamelCase			)
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
																											_UpperCAmelCase  :			Optional[Any]	       =		True
																											_UpperCAmelCase  :			Optional[int]	       =		self.tokenizer_class.from_pretrained(_lowerCamelCase	,						**_lowerCamelCase			)
																											_UpperCAmelCase  :			Dict	       =		self.rust_tokenizer_class.from_pretrained(_lowerCamelCase	,						**_lowerCamelCase			)

																											_UpperCAmelCase  :			Union[str, Any]	       =		tokenizer_p.encode(_lowerCamelCase	,						add_special_tokens=_lowerCamelCase			)
																											_UpperCAmelCase  :			int	       =		tokenizer_r.encode(_lowerCamelCase	,						add_special_tokens=_lowerCamelCase			)

																											_UpperCAmelCase  :			Dict	       =		tokenizer_r.convert_ids_to_tokens(_lowerCamelCase			)
																											_UpperCAmelCase  :			List[str]	       =		tokenizer_p.convert_ids_to_tokens(_lowerCamelCase			)

																											# it is expected that each Chinese character is not preceded by "##"
																											self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)
																											self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)

																											_UpperCAmelCase  :			Any	       =		False
																											_UpperCAmelCase  :			Dict	       =		self.rust_tokenizer_class.from_pretrained(_lowerCamelCase	,						**_lowerCamelCase			)
																											_UpperCAmelCase  :			Dict	       =		self.tokenizer_class.from_pretrained(_lowerCamelCase	,						**_lowerCamelCase			)

																											_UpperCAmelCase  :			Tuple	       =		tokenizer_r.encode(_lowerCamelCase	,						add_special_tokens=_lowerCamelCase			)
																											_UpperCAmelCase  :			Dict	       =		tokenizer_p.encode(_lowerCamelCase	,						add_special_tokens=_lowerCamelCase			)

																											_UpperCAmelCase  :			Optional[int]	       =		tokenizer_r.convert_ids_to_tokens(_lowerCamelCase			)
																											_UpperCAmelCase  :			Any	       =		tokenizer_p.convert_ids_to_tokens(_lowerCamelCase			)

																											# it is expected that only the first Chinese character is not preceded by "##".
																											_UpperCAmelCase  :			List[Any]	       =		[
																											    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_lowerCamelCase			)
																											]
																											self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)
																											self.assertListEqual(_lowerCamelCase	,						_lowerCamelCase			)






import torch

from accelerate import Accelerator, DistributedType
class    _UpperCamelCase:

						def __enter__(					self		:			str			):
													if self.enabled:
																				_UpperCAmelCase  :			int	       =		self.model.no_sync()
																				self.model_sync_obj.__enter__()

													return self

						def __exit__(					self		:			str	,						_lowerCamelCase		:			Union[str, Any]	,						_lowerCamelCase		:			Tuple	,						_lowerCamelCase		:			Optional[Any]			):
													if self.enabled:
																				# Average all models on exit
																				self._sync_and_avg_model_params()
																				self.model_sync_obj.__exit__(_lowerCamelCase	,						_lowerCamelCase	,						_lowerCamelCase			)

						def __init__(					self		:			str	,						_lowerCamelCase		:			Accelerator	,						_lowerCamelCase		:			torch.nn.Module	,						_lowerCamelCase		:			int	,						_lowerCamelCase		:			bool = True			):
													if accelerator.distributed_type not in [
													    DistributedType.NO,
													    DistributedType.MULTI_CPU,
													    DistributedType.MULTI_GPU,
													]:
																				raise NotImplementedError("LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)"			)
													_UpperCAmelCase  :			Any	       =		enabled and accelerator.distributed_type != DistributedType.NO
													_UpperCAmelCase  :			str	       =		0
													if self.enabled:
																				_UpperCAmelCase  :			Tuple	       =		accelerator
																				_UpperCAmelCase  :			List[str]	       =		model
																				_UpperCAmelCase  :			Dict	       =		local_sgd_steps

						def 						a__ (					self		:			Optional[int]			):
													self.num_steps += 1
													if not self.enabled:
																				return

													if self.num_steps % self.local_sgd_steps == 0:
																				self._sync_and_avg_model_params()






						def 						a__ (					self		:			Tuple			):
													self.accelerator.wait_for_everyone()
													with self.accelerator.autocast():
																				for param in self.model.parameters():
																											_UpperCAmelCase  :			Optional[Any]	       =		self.accelerator.reduce(param.data	,						reduction="mean"			)




