


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class      snake_case__	(   _A    ,							unittest.TestCase	):
				__lowerCAmelCase		=   RealmTokenizer
				__lowerCAmelCase		=   RealmTokenizerFast
				__lowerCAmelCase		=   True
				__lowerCAmelCase		=   True
				__lowerCAmelCase		=   filter_non_english
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										super().setUp()

										__magic_name__					:				Optional[Any]				=   [
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										__magic_name__					:				List[Any]				=   os.path.join(self.tmpdirname  ,   VOCAB_FILES_NAMES['vocab_file']   )
										with open(self.vocab_file  ,   'w'  ,   encoding='utf-8'   ) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]   )   )
				def 			UpperCamelCase_    (	self  ,   _UpperCAmelCase   ):




										"""simple docstring"""




										__magic_name__					:				int				=   'UNwant\u00E9d,running'
										__magic_name__					:				str				=   'unwanted, running'
										return input_text, output_text
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Optional[int]				=   self.tokenizer_class(self.vocab_file   )

										__magic_name__					:				Any				=   tokenizer.tokenize('UNwant\u00E9d,running'   )
										self.assertListEqual(_UpperCAmelCase  ,   ['un', '##want', '##ed', ',', 'runn', '##ing']   )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase   )  ,   [9, 6, 7, 12, 10, 11]   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										if not self.test_rust_tokenizer:
																return

										__magic_name__					:				int				=   self.get_tokenizer()
										__magic_name__					:				Optional[int]				=   self.get_rust_tokenizer()

										__magic_name__					:				Union[str, Any]				=   'UNwant\u00E9d,running'

										__magic_name__					:				Optional[int]				=   tokenizer.tokenize(_UpperCAmelCase   )
										__magic_name__					:				List[Any]				=   rust_tokenizer.tokenize(_UpperCAmelCase   )
										self.assertListEqual(_UpperCAmelCase  ,   _UpperCAmelCase   )

										__magic_name__					:				Optional[int]				=   tokenizer.encode(_UpperCAmelCase  ,   add_special_tokens=_UpperCAmelCase   )
										__magic_name__					:				Dict				=   rust_tokenizer.encode(_UpperCAmelCase  ,   add_special_tokens=_UpperCAmelCase   )
										self.assertListEqual(_UpperCAmelCase  ,   _UpperCAmelCase   )

										__magic_name__					:				str				=   self.get_rust_tokenizer()
										__magic_name__					:				Tuple				=   tokenizer.encode(_UpperCAmelCase   )
										__magic_name__					:				Tuple				=   rust_tokenizer.encode(_UpperCAmelCase   )
										self.assertListEqual(_UpperCAmelCase  ,   _UpperCAmelCase   )

										# With lower casing
										__magic_name__					:				List[str]				=   self.get_tokenizer(do_lower_case=_UpperCAmelCase   )
										__magic_name__					:				Tuple				=   self.get_rust_tokenizer(do_lower_case=_UpperCAmelCase   )

										__magic_name__					:				Any				=   'UNwant\u00E9d,running'

										__magic_name__					:				str				=   tokenizer.tokenize(_UpperCAmelCase   )
										__magic_name__					:				str				=   rust_tokenizer.tokenize(_UpperCAmelCase   )
										self.assertListEqual(_UpperCAmelCase  ,   _UpperCAmelCase   )

										__magic_name__					:				Optional[Any]				=   tokenizer.encode(_UpperCAmelCase  ,   add_special_tokens=_UpperCAmelCase   )
										__magic_name__					:				int				=   rust_tokenizer.encode(_UpperCAmelCase  ,   add_special_tokens=_UpperCAmelCase   )
										self.assertListEqual(_UpperCAmelCase  ,   _UpperCAmelCase   )

										__magic_name__					:				Union[str, Any]				=   self.get_rust_tokenizer()
										__magic_name__					:				List[str]				=   tokenizer.encode(_UpperCAmelCase   )
										__magic_name__					:				List[str]				=   rust_tokenizer.encode(_UpperCAmelCase   )
										self.assertListEqual(_UpperCAmelCase  ,   _UpperCAmelCase   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				int				=   BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'   )  ,   ['ah', '\u535A', '\u63A8', 'zz']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				List[Any]				=   BasicTokenizer(do_lower_case=_UpperCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '   )  ,   ['hello', '!', 'how', 'are', 'you', '?']   )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )  ,   ['hello']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Union[str, Any]				=   BasicTokenizer(do_lower_case=_UpperCAmelCase  ,   strip_accents=_UpperCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )  ,   ['hällo', '!', 'how', 'are', 'you', '?']   )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )  ,   ['h\u00E9llo']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Any				=   BasicTokenizer(do_lower_case=_UpperCAmelCase  ,   strip_accents=_UpperCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )  ,   ['hallo', '!', 'how', 'are', 'you', '?']   )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )  ,   ['hello']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Optional[Any]				=   BasicTokenizer(do_lower_case=_UpperCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )  ,   ['hallo', '!', 'how', 'are', 'you', '?']   )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )  ,   ['hello']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Any				=   BasicTokenizer(do_lower_case=_UpperCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '   )  ,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				List[str]				=   BasicTokenizer(do_lower_case=_UpperCAmelCase  ,   strip_accents=_UpperCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )  ,   ['HäLLo', '!', 'how', 'Are', 'yoU', '?']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				List[Any]				=   BasicTokenizer(do_lower_case=_UpperCAmelCase  ,   strip_accents=_UpperCAmelCase   )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )  ,   ['HaLLo', '!', 'how', 'Are', 'yoU', '?']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Union[str, Any]				=   BasicTokenizer(do_lower_case=_UpperCAmelCase  ,   never_split=['[UNK]']   )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'   )  ,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Dict				=   ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										__magic_name__					:				Dict				=   {}
										for i, token in enumerate(_UpperCAmelCase   ):
																__magic_name__					:				str				=   i
										__magic_name__					:				Tuple				=   WordpieceTokenizer(vocab=_UpperCAmelCase  ,   unk_token='[UNK]'   )

										self.assertListEqual(tokenizer.tokenize(''   )  ,   []   )

										self.assertListEqual(tokenizer.tokenize('unwanted running'   )  ,   ['un', '##want', '##ed', 'runn', '##ing']   )

										self.assertListEqual(tokenizer.tokenize('unwantedX running'   )  ,   ['[UNK]', 'runn', '##ing']   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										self.assertTrue(_is_whitespace(' '   )   )
										self.assertTrue(_is_whitespace('\t'   )   )
										self.assertTrue(_is_whitespace('\r'   )   )
										self.assertTrue(_is_whitespace('\n'   )   )
										self.assertTrue(_is_whitespace('\u00A0'   )   )

										self.assertFalse(_is_whitespace('A'   )   )
										self.assertFalse(_is_whitespace('-'   )   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										self.assertTrue(_is_control('\u0005'   )   )

										self.assertFalse(_is_control('A'   )   )
										self.assertFalse(_is_control(' '   )   )
										self.assertFalse(_is_control('\t'   )   )
										self.assertFalse(_is_control('\r'   )   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										self.assertTrue(_is_punctuation('-'   )   )
										self.assertTrue(_is_punctuation('$'   )   )
										self.assertTrue(_is_punctuation('`'   )   )
										self.assertTrue(_is_punctuation('.'   )   )

										self.assertFalse(_is_punctuation('A'   )   )
										self.assertFalse(_is_punctuation(' '   )   )
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				List[str]				=   self.get_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(_UpperCAmelCase   ) for t in ['Test', '\xad', 'test']]  ,   [['[UNK]'], [], ['[UNK]']]   )

										if self.test_rust_tokenizer:
																__magic_name__					:				str				=   self.get_rust_tokenizer()
																self.assertListEqual(
																    [rust_tokenizer.tokenize(_UpperCAmelCase   ) for t in ['Test', '\xad', 'test']]  ,   [['[UNK]'], [], ['[UNK]']]   )
				@slow
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				List[Any]				=   self.tokenizer_class.from_pretrained('bert-base-uncased'   )

										__magic_name__					:				Any				=   tokenizer.encode('sequence builders'  ,   add_special_tokens=_UpperCAmelCase   )
										__magic_name__					:				int				=   tokenizer.encode('multi-sequence build'  ,   add_special_tokens=_UpperCAmelCase   )

										__magic_name__					:				str				=   tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase   )
										__magic_name__					:				List[str]				=   tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase  ,   _UpperCAmelCase   )

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
																						__magic_name__					:				Tuple				=   self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase  ,   **_UpperCAmelCase   )

																						__magic_name__					:				Any				=   F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																						__magic_name__					:				Dict				=   tokenizer_r.encode_plus(
																						    _UpperCAmelCase  ,   return_attention_mask=_UpperCAmelCase  ,   return_token_type_ids=_UpperCAmelCase  ,   return_offsets_mapping=_UpperCAmelCase  ,   add_special_tokens=_UpperCAmelCase  ,   )

																						__magic_name__					:				Optional[int]				=   tokenizer_r.do_lower_case if hasattr(_UpperCAmelCase  ,   'do_lower_case'   ) else False
																						__magic_name__					:				str				=   (
																						    [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), 'A'),
																						        ((1, 2), ','),
																						        ((3, 5), 'na'),
																						        ((5, 6), '##ï'),
																						        ((6, 8), '##ve'),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), 'Allen'),
																						        ((21, 23), '##NL'),
																						        ((23, 24), '##P'),
																						        ((25, 33), 'sentence'),
																						        ((33, 34), '.'),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						    if not do_lower_case
																						    else [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), 'a'),
																						        ((1, 2), ','),
																						        ((3, 8), 'naive'),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), 'allen'),
																						        ((21, 23), '##nl'),
																						        ((23, 24), '##p'),
																						        ((25, 33), 'sentence'),
																						        ((33, 34), '.'),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						)

																						self.assertEqual(
																						    [e[1] for e in expected_results]  ,   tokenizer_r.convert_ids_to_tokens(tokens['input_ids']   )   )
																						self.assertEqual([e[0] for e in expected_results]  ,   tokens['offset_mapping']   )





				@slow
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
																						__magic_name__					:				Optional[Any]				=   self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase  ,   **_UpperCAmelCase   )
																						__magic_name__					:				List[str]				=   self.tokenizer_class.from_pretrained(_UpperCAmelCase  ,   **_UpperCAmelCase   )
																						__magic_name__					:				List[str]				=   [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

																						__magic_name__					:				Any				=   tokenizer_r.batch_encode_candidates(_UpperCAmelCase  ,   max_length=10  ,   return_tensors='np'   )
																						__magic_name__					:				Optional[Any]				=   tokenizer_p.batch_encode_candidates(_UpperCAmelCase  ,   max_length=10  ,   return_tensors='np'   )

																						__magic_name__					:				List[str]				=   (2, 2, 10)

																						self.assertEqual(encoded_sentence_r['input_ids'].shape  ,   _UpperCAmelCase   )
																						self.assertEqual(encoded_sentence_r['attention_mask'].shape  ,   _UpperCAmelCase   )
																						self.assertEqual(encoded_sentence_r['token_type_ids'].shape  ,   _UpperCAmelCase   )

																						self.assertEqual(encoded_sentence_p['input_ids'].shape  ,   _UpperCAmelCase   )
																						self.assertEqual(encoded_sentence_p['attention_mask'].shape  ,   _UpperCAmelCase   )
																						self.assertEqual(encoded_sentence_p['token_type_ids'].shape  ,   _UpperCAmelCase   )






from __future__ import annotations

import random
import string



class      snake_case__	:
				def __init__(	self  ,   _UpperCAmelCase = None   ):




										"""simple docstring"""




										__magic_name__					:				Union[str, Any]				=   passcode or self.__passcode_creator()
										__magic_name__					:				Any				=   self.__make_key_list()
										__magic_name__					:				Tuple				=   self.__make_shift_key()
				def __str__(	self   ):




										"""simple docstring"""




										return "".join(self.__passcode   )
				def 			UpperCamelCase_    (	self  ,   _UpperCAmelCase   ):




										"""simple docstring"""




										for i in range(1  ,   len(_UpperCAmelCase   )  ,   2   ):
																iterlist[i] *= -1
										return iterlist
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Any				=   string.ascii_letters + string.digits
										__magic_name__					:				Optional[Any]				=   [random.choice(_UpperCAmelCase   ) for _ in range(random.randint(10  ,   20   )   )]
										return password
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Any				=   (
										    string.ascii_letters + string.digits + string.punctuation + ' \t\n'
										)

										__magic_name__					:				List[str]				=   []

										# creates points known as breakpoints to break the key_list_options at those
										# points and pivot each substring
										__magic_name__					:				List[Any]				=   sorted(set(self.__passcode   )   )
										__magic_name__					:				list[str]				=   []

										# algorithm for creating a new shuffled list, keys_l, out of key_list_options
										for i in key_list_options:
																temp_list.extend(_UpperCAmelCase   )

																# checking breakpoints at which to pivot temporary sublist and add it into
																# keys_l
																if i in breakpoints or i == key_list_options[-1]:
																						keys_l.extend(temp_list[::-1]   )
																						temp_list.clear()

        # returning a shuffled keys_l to prevent brute force guessing of shift key
										return keys_l
				def 			UpperCamelCase_    (	self   ):




										"""simple docstring"""




										__magic_name__					:				Dict				=   sum(self.__neg_pos([ord(_UpperCAmelCase   ) for x in self.__passcode]   )   )
										return num if num > 0 else len(self.__passcode   )
				def 			UpperCamelCase_    (	self  ,   _UpperCAmelCase   ):




										"""simple docstring"""




										__magic_name__					:				List[Any]				=   ''

										# decoding shift like Caesar cipher algorithm implementing negative shift or
										# reverse shift or left shift
										for i in encoded_message:
																__magic_name__					:				int				=   self.__key_list.index(_UpperCAmelCase   )
																decoded_message += self.__key_list[
																    (position - self.__shift_key) % -len(self.__key_list   )
																]

										return decoded_message





				def 			UpperCamelCase_    (	self  ,   _UpperCAmelCase   ):




										"""simple docstring"""




										__magic_name__					:				List[Any]				=   ''

										# encoding shift like Caesar cipher algorithm implementing positive shift or
										# forward shift or right shift
										for i in plaintext:
																__magic_name__					:				Dict				=   self.__key_list.index(_UpperCAmelCase   )
																encoded_message += self.__key_list[
																    (position + self.__shift_key) % len(self.__key_list   )
																]

										return encoded_message






def 			__SCREAMING_SNAKE_CASE							(					lowercase__	:str = "Hello, this is a modified Caesar cipher"					)							->	str:
						__magic_name__					:				List[str]				=   ShuffledShiftCipher()
						return cipa.decrypt(cipa.encrypt(lowercase__					)					)


if __name__ == "__main__":
			import doctest

			doctest.testmod()


