



import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 						lowerCamelCase__			(		_a					,       unittest.TestCase ):
						_lowerCAmelCase             =			RealmTokenizer
						_lowerCAmelCase             =			RealmTokenizerFast
						_lowerCAmelCase             =			True
						_lowerCAmelCase             =			True
						_lowerCAmelCase             =			filter_non_english


						def 	_lowerCamelCase      (			self    :	Union[str, Any]      ):
													super().setUp()

													a__:							Any									=[
													    "[UNK]",
													    "[CLS]",
													    "[SEP]",
													    "[PAD]",
													    "[MASK]",
													    "want",
													    "##want",
													    "##ed",
													    "wa",
													    "un",
													    "runn",
													    "##ing",
													    ",",
													    "low",
													    "lowest",
													]
													a__:							int									=os.path.join(self.tmpdirname   ,							VOCAB_FILES_NAMES["vocab_file"]      )
													with open(self.vocab_file   ,							"w"   ,							encoding="utf-8"      ) as vocab_writer:
																				vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )


						def 	_lowerCamelCase      (			self    :	List[str]   ,							_a    :	Any      ):
													a__:							Optional[int]									="UNwant\u00E9d,running"
													a__:							Any									="unwanted, running"
													return input_text, output_text


						def 	_lowerCamelCase      (			self    :	Tuple      ):
													a__:							int									=self.tokenizer_class(self.vocab_file      )

													a__:							List[Any]									=tokenizer.tokenize("UNwant\u00E9d,running"      )
													self.assertListEqual(_a   ,							["un", "##want", "##ed", ",", "runn", "##ing"]      )
													self.assertListEqual(tokenizer.convert_tokens_to_ids(_a      )   ,							[9, 6, 7, 1_2, 1_0, 1_1]      )


						def 	_lowerCamelCase      (			self    :	Optional[Any]      ):
													if not self.test_rust_tokenizer:
																				return

													a__:							Any									=self.get_tokenizer()
													a__:							Dict									=self.get_rust_tokenizer()

													a__:							str									="UNwant\u00E9d,running"

													a__:							Union[str, Any]									=tokenizer.tokenize(_a      )
													a__:							Any									=rust_tokenizer.tokenize(_a      )
													self.assertListEqual(_a   ,							_a      )

													a__:							Union[str, Any]									=tokenizer.encode(_a   ,							add_special_tokens=_a      )
													a__:							List[str]									=rust_tokenizer.encode(_a   ,							add_special_tokens=_a      )
													self.assertListEqual(_a   ,							_a      )

													a__:							Tuple									=self.get_rust_tokenizer()
													a__:							Dict									=tokenizer.encode(_a      )
													a__:							List[Any]									=rust_tokenizer.encode(_a      )
													self.assertListEqual(_a   ,							_a      )

													# With lower casing
													a__:							Any									=self.get_tokenizer(do_lower_case=_a      )
													a__:							Any									=self.get_rust_tokenizer(do_lower_case=_a      )

													a__:							List[Any]									="UNwant\u00E9d,running"

													a__:							Optional[int]									=tokenizer.tokenize(_a      )
													a__:							str									=rust_tokenizer.tokenize(_a      )
													self.assertListEqual(_a   ,							_a      )

													a__:							Dict									=tokenizer.encode(_a   ,							add_special_tokens=_a      )
													a__:							Any									=rust_tokenizer.encode(_a   ,							add_special_tokens=_a      )
													self.assertListEqual(_a   ,							_a      )

													a__:							List[str]									=self.get_rust_tokenizer()
													a__:							Optional[int]									=tokenizer.encode(_a      )
													a__:							Optional[int]									=rust_tokenizer.encode(_a      )
													self.assertListEqual(_a   ,							_a      )


						def 	_lowerCamelCase      (			self    :	Optional[int]      ):
													a__:							str									=BasicTokenizer()

													self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"      )   ,							["ah", "\u535A", "\u63A8", "zz"]      )


						def 	_lowerCamelCase      (			self    :	Optional[int]      ):
													a__:							Tuple									=BasicTokenizer(do_lower_case=_a      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )   ,							["hello", "!", "how", "are", "you", "?"]      )
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )   ,							["hello"]      )


						def 	_lowerCamelCase      (			self    :	str      ):
													a__:							Union[str, Any]									=BasicTokenizer(do_lower_case=_a   ,							strip_accents=_a      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["hällo", "!", "how", "are", "you", "?"]      )
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )   ,							["h\u00E9llo"]      )


						def 	_lowerCamelCase      (			self    :	str      ):
													a__:							Dict									=BasicTokenizer(do_lower_case=_a   ,							strip_accents=_a      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["hallo", "!", "how", "are", "you", "?"]      )
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )   ,							["hello"]      )


						def 	_lowerCamelCase      (			self    :	Union[str, Any]      ):
													a__:							Tuple									=BasicTokenizer(do_lower_case=_a      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["hallo", "!", "how", "are", "you", "?"]      )
													self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )   ,							["hello"]      )


						def 	_lowerCamelCase      (			self    :	Tuple      ):
													a__:							Union[str, Any]									=BasicTokenizer(do_lower_case=_a      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )   ,							["HeLLo", "!", "how", "Are", "yoU", "?"]      )


						def 	_lowerCamelCase      (			self    :	Union[str, Any]      ):
													a__:							str									=BasicTokenizer(do_lower_case=_a   ,							strip_accents=_a      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["HäLLo", "!", "how", "Are", "yoU", "?"]      )


						def 	_lowerCamelCase      (			self    :	str      ):
													a__:							List[str]									=BasicTokenizer(do_lower_case=_a   ,							strip_accents=_a      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )   ,							["HaLLo", "!", "how", "Are", "yoU", "?"]      )


						def 	_lowerCamelCase      (			self    :	Optional[int]      ):
													a__:							List[str]									=BasicTokenizer(do_lower_case=_a   ,							never_split=["[UNK]"]      )

													self.assertListEqual(
													    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"      )   ,							["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]      )


						def 	_lowerCamelCase      (			self    :	Dict      ):
													a__:							Union[str, Any]									=["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

													a__:							int									={}
													for i, token in enumerate(_a      ):
																				a__:							int									=i
													a__:							List[str]									=WordpieceTokenizer(vocab=_a   ,							unk_token="[UNK]"      )

													self.assertListEqual(tokenizer.tokenize(""      )   ,							[]      )

													self.assertListEqual(tokenizer.tokenize("unwanted running"      )   ,							["un", "##want", "##ed", "runn", "##ing"]      )

													self.assertListEqual(tokenizer.tokenize("unwantedX running"      )   ,							["[UNK]", "runn", "##ing"]      )


						def 	_lowerCamelCase      (			self    :	Tuple      ):
													self.assertTrue(_is_whitespace(" "      )      )
													self.assertTrue(_is_whitespace("\t"      )      )
													self.assertTrue(_is_whitespace("\r"      )      )
													self.assertTrue(_is_whitespace("\n"      )      )
													self.assertTrue(_is_whitespace("\u00A0"      )      )

													self.assertFalse(_is_whitespace("A"      )      )
													self.assertFalse(_is_whitespace("-"      )      )


						def 	_lowerCamelCase      (			self    :	List[Any]      ):
													self.assertTrue(_is_control("\u0005"      )      )

													self.assertFalse(_is_control("A"      )      )
													self.assertFalse(_is_control(" "      )      )
													self.assertFalse(_is_control("\t"      )      )
													self.assertFalse(_is_control("\r"      )      )


						def 	_lowerCamelCase      (			self    :	List[Any]      ):
													self.assertTrue(_is_punctuation("-"      )      )
													self.assertTrue(_is_punctuation("$"      )      )
													self.assertTrue(_is_punctuation("`"      )      )
													self.assertTrue(_is_punctuation("."      )      )

													self.assertFalse(_is_punctuation("A"      )      )
													self.assertFalse(_is_punctuation(" "      )      )


						def 	_lowerCamelCase      (			self    :	int      ):
													a__:							str									=self.get_tokenizer()

													# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
													self.assertListEqual([tokenizer.tokenize(_a      ) for t in ["Test", "\xad", "test"]]   ,							[["[UNK]"], [], ["[UNK]"]]      )

													if self.test_rust_tokenizer:
																				a__:							Tuple									=self.get_rust_tokenizer()
																				self.assertListEqual(
																				    [rust_tokenizer.tokenize(_a      ) for t in ["Test", "\xad", "test"]]   ,							[["[UNK]"], [], ["[UNK]"]]      )


						@slow
						def 	_lowerCamelCase      (			self    :	Optional[int]      ):
													a__:							Optional[int]									=self.tokenizer_class.from_pretrained("bert-base-uncased"      )

													a__:							Optional[int]									=tokenizer.encode("sequence builders"   ,							add_special_tokens=_a      )
													a__:							str									=tokenizer.encode("multi-sequence build"   ,							add_special_tokens=_a      )

													a__:							Dict									=tokenizer.build_inputs_with_special_tokens(_a      )
													a__:							List[str]									=tokenizer.build_inputs_with_special_tokens(_a   ,							_a      )

													assert encoded_sentence == [1_0_1] + text + [1_0_2]
													assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]


						def 	_lowerCamelCase      (			self    :	Dict      ):
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
																											a__:							int									=self.rust_tokenizer_class.from_pretrained(_a   ,							**_a      )

																											a__:							List[Any]									=F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																											a__:							Dict									=tokenizer_r.encode_plus(
																											    _a   ,							return_attention_mask=_a   ,							return_token_type_ids=_a   ,							return_offsets_mapping=_a   ,							add_special_tokens=_a   ,							)

																											a__:							Union[str, Any]									=tokenizer_r.do_lower_case if hasattr(_a   ,							"do_lower_case"      ) else False
																											a__:							Optional[int]									=(
																											    [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), "A"),
																											        ((1, 2), ","),
																											        ((3, 5), "na"),
																											        ((5, 6), "##ï"),
																											        ((6, 8), "##ve"),
																											        ((9, 1_5), tokenizer_r.mask_token),
																											        ((1_6, 2_1), "Allen"),
																											        ((2_1, 2_3), "##NL"),
																											        ((2_3, 2_4), "##P"),
																											        ((2_5, 3_3), "sentence"),
																											        ((3_3, 3_4), "."),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											    if not do_lower_case
																											    else [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), "a"),
																											        ((1, 2), ","),
																											        ((3, 8), "naive"),
																											        ((9, 1_5), tokenizer_r.mask_token),
																											        ((1_6, 2_1), "allen"),
																											        ((2_1, 2_3), "##nl"),
																											        ((2_3, 2_4), "##p"),
																											        ((2_5, 3_3), "sentence"),
																											        ((3_3, 3_4), "."),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											)

																											self.assertEqual(
																											    [e[1] for e in expected_results]   ,							tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]      )      )
																											self.assertEqual([e[0] for e in expected_results]   ,							tokens["offset_mapping"]      )







						@slow
						def 	_lowerCamelCase      (			self    :	int      ):
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
																											a__:							int									=self.rust_tokenizer_class.from_pretrained(_a   ,							**_a      )
																											a__:							int									=self.tokenizer_class.from_pretrained(_a   ,							**_a      )
																											a__:							Dict									=[["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

																											a__:							List[Any]									=tokenizer_r.batch_encode_candidates(_a   ,							max_length=1_0   ,							return_tensors="np"      )
																											a__:							List[str]									=tokenizer_p.batch_encode_candidates(_a   ,							max_length=1_0   ,							return_tensors="np"      )

																											a__:							List[str]									=(2, 2, 1_0)

																											self.assertEqual(encoded_sentence_r["input_ids"].shape   ,							_a      )
																											self.assertEqual(encoded_sentence_r["attention_mask"].shape   ,							_a      )
																											self.assertEqual(encoded_sentence_r["token_type_ids"].shape   ,							_a      )

																											self.assertEqual(encoded_sentence_p["input_ids"].shape   ,							_a      )
																											self.assertEqual(encoded_sentence_p["attention_mask"].shape   ,							_a      )
																											self.assertEqual(encoded_sentence_p["token_type_ids"].shape   ,							_a      )










def 						_lowerCAmelCase    (					__lowerCamelCase :  list  ):



 """simple docstring"""





 if len(__lowerCamelCase  ) <= 1:
  return data
 else:
  return [
      *quick_sort([e for e in data[1:] if e <= data[0]]  ),
      data[0],
      *quick_sort([e for e in data[1:] if e > data[0]]  ),
  ]


if __name__ == "__main__":
      import doctest

      doctest.testmod()






