






'''simple docstring'''

import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 				__magic_name__  (						UpperCAmelCase_,    unittest.TestCase					):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_       :   Optional[int]   =	RealmTokenizer
	SCREAMING_SNAKE_CASE_       :   Any   =	RealmTokenizerFast
	SCREAMING_SNAKE_CASE_       :   int   =	True
	SCREAMING_SNAKE_CASE_       :   Optional[int]   =	True
	SCREAMING_SNAKE_CASE_       :   Any   =	filter_non_english
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							super().setUp()

							lowercase_   : Any   	=      [
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    '[PAD]',
							    '[MASK]',
							    'want',
							    '##want',
							    '##ed',
							    'wa',
							    'un',
							    'runn',
							    '##ing',
							    ',',
							    'low',
							    'lowest',
							]
							lowercase_   : Union[str, Any]   	=      os.path.join(self.tmpdirname	,   VOCAB_FILES_NAMES['vocab_file']							)
							with open(self.vocab_file	,   'w'	,   encoding='utf-8'							) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]							)							)
	def 							lowerCamelCase__		(				self	,   _lowercase							)    ->	Optional[int]:
							lowercase_   : Dict   	=      'UNwant\u00E9d,running'
							lowercase_   : Optional[int]   	=      'unwanted, running'
							return input_text, output_text
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							lowercase_   : Optional[Any]   	=      self.tokenizer_class(self.vocab_file							)

							lowercase_   : str   	=      tokenizer.tokenize('UNwant\u00E9d,running'							)
							self.assertListEqual(_lowercase	,   ['un', '##want', '##ed', ',', 'runn', '##ing']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase							)	,   [9, 6, 7, 12, 10, 11]							)
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							if not self.test_rust_tokenizer:
													return

							lowercase_   : Any   	=      self.get_tokenizer()
							lowercase_   : Union[str, Any]   	=      self.get_rust_tokenizer()

							lowercase_   : List[Any]   	=      'UNwant\u00E9d,running'

							lowercase_   : Tuple   	=      tokenizer.tokenize(_lowercase							)
							lowercase_   : List[Any]   	=      rust_tokenizer.tokenize(_lowercase							)
							self.assertListEqual(_lowercase	,   _lowercase							)

							lowercase_   : str   	=      tokenizer.encode(_lowercase	,   add_special_tokens=_lowercase							)
							lowercase_   : Dict   	=      rust_tokenizer.encode(_lowercase	,   add_special_tokens=_lowercase							)
							self.assertListEqual(_lowercase	,   _lowercase							)

							lowercase_   : Union[str, Any]   	=      self.get_rust_tokenizer()
							lowercase_   : Dict   	=      tokenizer.encode(_lowercase							)
							lowercase_   : Optional[int]   	=      rust_tokenizer.encode(_lowercase							)
							self.assertListEqual(_lowercase	,   _lowercase							)

							# With lower casing
							lowercase_   : List[Any]   	=      self.get_tokenizer(do_lower_case=_lowercase							)
							lowercase_   : List[str]   	=      self.get_rust_tokenizer(do_lower_case=_lowercase							)

							lowercase_   : str   	=      'UNwant\u00E9d,running'

							lowercase_   : int   	=      tokenizer.tokenize(_lowercase							)
							lowercase_   : str   	=      rust_tokenizer.tokenize(_lowercase							)
							self.assertListEqual(_lowercase	,   _lowercase							)

							lowercase_   : Optional[int]   	=      tokenizer.encode(_lowercase	,   add_special_tokens=_lowercase							)
							lowercase_   : Optional[int]   	=      rust_tokenizer.encode(_lowercase	,   add_special_tokens=_lowercase							)
							self.assertListEqual(_lowercase	,   _lowercase							)

							lowercase_   : List[str]   	=      self.get_rust_tokenizer()
							lowercase_   : str   	=      tokenizer.encode(_lowercase							)
							lowercase_   : Any   	=      rust_tokenizer.encode(_lowercase							)
							self.assertListEqual(_lowercase	,   _lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Optional[int]   	=      BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'							)	,   ['ah', '\u535A', '\u63A8', 'zz']							)
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							lowercase_   : Optional[Any]   	=      BasicTokenizer(do_lower_case=_lowercase							)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)	,   ['hello', '!', 'how', 'are', 'you', '?']							)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)	,   ['hello']							)
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							lowercase_   : List[Any]   	=      BasicTokenizer(do_lower_case=_lowercase	,   strip_accents=_lowercase							)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)	,   ['hällo', '!', 'how', 'are', 'you', '?']							)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)	,   ['h\u00E9llo']							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Dict   	=      BasicTokenizer(do_lower_case=_lowercase	,   strip_accents=_lowercase							)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)	,   ['hallo', '!', 'how', 'are', 'you', '?']							)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)	,   ['hello']							)
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : Dict   	=      BasicTokenizer(do_lower_case=_lowercase							)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)	,   ['hallo', '!', 'how', 'are', 'you', '?']							)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)	,   ['hello']							)
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							lowercase_   : Dict   	=      BasicTokenizer(do_lower_case=_lowercase							)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)	,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?']							)
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							lowercase_   : int   	=      BasicTokenizer(do_lower_case=_lowercase	,   strip_accents=_lowercase							)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)	,   ['HäLLo', '!', 'how', 'Are', 'yoU', '?']							)
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Tuple   	=      BasicTokenizer(do_lower_case=_lowercase	,   strip_accents=_lowercase							)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)	,   ['HaLLo', '!', 'how', 'Are', 'yoU', '?']							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : List[Any]   	=      BasicTokenizer(do_lower_case=_lowercase	,   never_split=['[UNK]']							)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'							)	,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']							)
	def 							lowerCamelCase__		(				self							)    ->	Optional[Any]:
							lowercase_   : Optional[Any]   	=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

							lowercase_   : List[Any]   	=      {}
							for i, token in enumerate(_lowercase							):
													lowercase_   : Union[str, Any]   	=      i
							lowercase_   : Optional[Any]   	=      WordpieceTokenizer(vocab=_lowercase	,   unk_token='[UNK]'							)

							self.assertListEqual(tokenizer.tokenize(''							)	,   []							)

							self.assertListEqual(tokenizer.tokenize('unwanted running'							)	,   ['un', '##want', '##ed', 'runn', '##ing']							)

							self.assertListEqual(tokenizer.tokenize('unwantedX running'							)	,   ['[UNK]', 'runn', '##ing']							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							self.assertTrue(_is_whitespace(' '							)							)
							self.assertTrue(_is_whitespace('\t'							)							)
							self.assertTrue(_is_whitespace('\r'							)							)
							self.assertTrue(_is_whitespace('\n'							)							)
							self.assertTrue(_is_whitespace('\u00A0'							)							)

							self.assertFalse(_is_whitespace('A'							)							)
							self.assertFalse(_is_whitespace('-'							)							)
	def 							lowerCamelCase__		(				self							)    ->	Optional[Any]:
							self.assertTrue(_is_control('\u0005'							)							)

							self.assertFalse(_is_control('A'							)							)
							self.assertFalse(_is_control(' '							)							)
							self.assertFalse(_is_control('\t'							)							)
							self.assertFalse(_is_control('\r'							)							)
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							self.assertTrue(_is_punctuation('-'							)							)
							self.assertTrue(_is_punctuation('$'							)							)
							self.assertTrue(_is_punctuation('`'							)							)
							self.assertTrue(_is_punctuation('.'							)							)

							self.assertFalse(_is_punctuation('A'							)							)
							self.assertFalse(_is_punctuation(' '							)							)
	def 							lowerCamelCase__		(				self							)    ->	Optional[int]:
							lowercase_   : Dict   	=      self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(_lowercase							) for t in ['Test', '\xad', 'test']]	,   [['[UNK]'], [], ['[UNK]']]							)

							if self.test_rust_tokenizer:
													lowercase_   : Dict   	=      self.get_rust_tokenizer()
													self.assertListEqual(
													    [rust_tokenizer.tokenize(_lowercase							) for t in ['Test', '\xad', 'test']]	,   [['[UNK]'], [], ['[UNK]']]							)
	@slow
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							lowercase_   : Optional[int]   	=      self.tokenizer_class.from_pretrained('bert-base-uncased'							)

							lowercase_   : List[str]   	=      tokenizer.encode('sequence builders'	,   add_special_tokens=_lowercase							)
							lowercase_   : Tuple   	=      tokenizer.encode('multi-sequence build'	,   add_special_tokens=_lowercase							)

							lowercase_   : List[str]   	=      tokenizer.build_inputs_with_special_tokens(_lowercase							)
							lowercase_   : Tuple   	=      tokenizer.build_inputs_with_special_tokens(_lowercase	,   _lowercase							)

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]
	def 							lowerCamelCase__		(				self							)    ->	Optional[Any]:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																			lowercase_   : Tuple   	=      self.rust_tokenizer_class.from_pretrained(_lowercase	,   **_lowercase							)

																			lowercase_   : str   	=      f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																			lowercase_   : Any   	=      tokenizer_r.encode_plus(
																			    _lowercase	,   return_attention_mask=_lowercase	,   return_token_type_ids=_lowercase	,   return_offsets_mapping=_lowercase	,   add_special_tokens=_lowercase	,   )

																			lowercase_   : str   	=      tokenizer_r.do_lower_case if hasattr(_lowercase	,   'do_lower_case'							) else False
																			lowercase_   : Tuple   	=      (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'A'),
																			        ((1, 2), ','),
																			        ((3, 5), 'na'),
																			        ((5, 6), '##ï'),
																			        ((6, 8), '##ve'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'Allen'),
																			        ((21, 23), '##NL'),
																			        ((23, 24), '##P'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), 'a'),
																			        ((1, 2), ','),
																			        ((3, 8), 'naive'),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), 'allen'),
																			        ((21, 23), '##nl'),
																			        ((23, 24), '##p'),
																			        ((25, 33), 'sentence'),
																			        ((33, 34), '.'),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]	,   tokenizer_r.convert_ids_to_tokens(tokens['input_ids']							)							)
																			self.assertEqual([e[0] for e in expected_results]	,   tokens['offset_mapping']							)







	@slow
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																			lowercase_   : int   	=      self.rust_tokenizer_class.from_pretrained(_lowercase	,   **_lowercase							)
																			lowercase_   : Optional[Any]   	=      self.tokenizer_class.from_pretrained(_lowercase	,   **_lowercase							)
																			lowercase_   : Optional[Any]   	=      [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

																			lowercase_   : Tuple   	=      tokenizer_r.batch_encode_candidates(_lowercase	,   max_length=10	,   return_tensors='np'							)
																			lowercase_   : List[Any]   	=      tokenizer_p.batch_encode_candidates(_lowercase	,   max_length=10	,   return_tensors='np'							)

																			lowercase_   : Any   	=      (2, 2, 10)

																			self.assertEqual(encoded_sentence_r['input_ids'].shape	,   _lowercase							)
																			self.assertEqual(encoded_sentence_r['attention_mask'].shape	,   _lowercase							)
																			self.assertEqual(encoded_sentence_r['token_type_ids'].shape	,   _lowercase							)

																			self.assertEqual(encoded_sentence_p['input_ids'].shape	,   _lowercase							)
																			self.assertEqual(encoded_sentence_p['attention_mask'].shape	,   _lowercase							)
																			self.assertEqual(encoded_sentence_p['token_type_ids'].shape	,   _lowercase							)








'''simple docstring'''

import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
						from tokenizers import Tokenizer
						from tokenizers.models import WordPiece






class 				__magic_name__  (						unittest.TestCase					):







	"""simple docstring"""



	def 							lowerCamelCase__		(				self	,   _lowercase							)    ->	Optional[int]:
							lowercase_   : Optional[int]   	=      list(tokenizer_class.max_model_input_sizes.keys()							)
							for model_name in sa_models[:1]:
													lowercase_   : List[str]   	=      tokenizer_class.from_pretrained(_lowercase							)
													self.assertIsNotNone(_lowercase							)
													self.assertIsInstance(_lowercase	,   _lowercase							)
													self.assertIsInstance(_lowercase	,   _lowercase							)

													for special_tok in tokenizer.all_special_tokens:
																			self.assertIsInstance(_lowercase	,   _lowercase							)
																			lowercase_   : Optional[int]   	=      tokenizer.convert_tokens_to_ids(_lowercase							)
																			self.assertIsInstance(_lowercase	,   _lowercase							)
	def 							lowerCamelCase__		(				self	,   _lowercase	,   _lowercase = None							)    ->	str:
							lowercase_   : int   	=      pickle.dumps(_lowercase							)
							self.assertIsNotNone(_lowercase							)

							lowercase_   : Any   	=      pickle.loads(_lowercase							)

							# Ensure is_fast is correctly restored
							self.assertEqual(be_restored.is_fast	,   be_original.is_fast							)

							# Ensure encodings are potentially correctly restored
							if be_original.is_fast:
													self.assertIsNotNone(be_restored.encodings							)
							else:
													self.assertIsNone(be_restored.encodings							)

							# Ensure the keys are the same
							for original_v, restored_v in zip(be_original.values()	,   be_restored.values()							):
													if equal_op:
																			self.assertTrue(equal_op(_lowercase	,   _lowercase							)							)
													else:
																			self.assertEqual(_lowercase	,   _lowercase							)
	@slow
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							self.check_tokenizer_from_pretrained(_lowercase							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							self.assertEqual(TensorType('tf'							)	,   TensorType.TENSORFLOW							)
							self.assertEqual(TensorType('pt'							)	,   TensorType.PYTORCH							)
							self.assertEqual(TensorType('np'							)	,   TensorType.NUMPY							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Dict:
							import numpy as np

							lowercase_   : Any   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : str   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							# Python no tensor
							with self.subTest('BatchEncoding (Python, return_tensors=None)'							):
													self.assert_dump_and_restore(tokenizer_p('Small example to encode'							)							)

							with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.NUMPY							)	,   np.array_equal							)

							with self.subTest('BatchEncoding (Rust, return_tensors=None)'							):
													self.assert_dump_and_restore(tokenizer_r('Small example to encode'							)							)

							with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.NUMPY							)	,   np.array_equal							)
	@require_tf
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Optional[int]:
							import tensorflow as tf

							def tf_array_equals(_lowercase	,   _lowercase							):
													return tf.reduce_all(tf.equal(_lowercase	,   _lowercase							)							)

							lowercase_   : Optional[Any]   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : str   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.TENSORFLOW							)	,   _lowercase							)

							with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.TENSORFLOW							)	,   _lowercase							)
	@require_torch
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							import torch

							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : Optional[int]   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'							):
													self.assert_dump_and_restore(
													    tokenizer_p('Small example to encode'	,   return_tensors=TensorType.PYTORCH							)	,   torch.equal							)

							with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'							):
													self.assert_dump_and_restore(
													    tokenizer_r('Small example to encode'	,   return_tensors=TensorType.PYTORCH							)	,   torch.equal							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)
							lowercase_   : Tuple   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)

							with self.subTest('Python Tokenizer'							):
													self.assertFalse(tokenizer_p('Small example to_encode'							).is_fast							)

							with self.subTest('Rust Tokenizer'							):
													self.assertTrue(tokenizer_r('Small example to_encode'							).is_fast							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							lowercase_   : Optional[Any]   	=      BertTokenizerFast.from_pretrained('bert-base-cased'							)
							lowercase_   : Dict   	=      tokenizer_r(['Test', '\xad', 'test']	,   is_split_into_words=_lowercase							)

							self.assertEqual(encoded.word_to_tokens(0							)	,   TokenSpan(start=1	,   end=2							)							)
							self.assertEqual(encoded.word_to_tokens(1							)	,   _lowercase							)
							self.assertEqual(encoded.word_to_tokens(2							)	,   TokenSpan(start=2	,   end=3							)							)
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Any   	=      batch.convert_to_tensors(tensor_type='np'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='np'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : Union[str, Any]   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Optional[int]   	=      batch.convert_to_tensors(tensor_type='np'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_torch
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='pt'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='pt'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : int   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Union[str, Any]   	=      batch.convert_to_tensors(tensor_type='pt'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_tf
	def 							lowerCamelCase__		(				self							)    ->	Tuple:
							lowercase_   : Dict   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : List[Any]   	=      batch.convert_to_tensors(tensor_type='tf'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Tuple   	=      batch.convert_to_tensors(tensor_type='tf'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : int   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='tf'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	@require_flax
	def 							lowerCamelCase__		(				self							)    ->	Union[str, Any]:
							lowercase_   : int   	=      BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}							)
							lowercase_   : Optional[Any]   	=      batch.convert_to_tensors(tensor_type='jax'							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (2, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (2,)							)
							# test converting the converted
							with CaptureStderr() as cs:
													lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='jax'							)
							self.assertFalse(len(cs.err							)	,   msg=f"should have no warning, but got {cs.err}"							)

							lowercase_   : List[Any]   	=      BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}							)
							lowercase_   : Dict   	=      batch.convert_to_tensors(tensor_type='jax'	,   prepend_batch_axis=_lowercase							)
							self.assertEqual(tensor_batch['inputs'].shape	,   (1, 3)							)
							self.assertEqual(tensor_batch['labels'].shape	,   (1,)							)
	def 							lowerCamelCase__		(				self							)    ->	List[str]:
							lowercase_   : Any   	=      [{'input_ids': np.array([0, 1, 2]							)}, {'input_ids': np.array([0, 1, 2, 3]							)}]
							lowercase_   : str   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : Any   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   np.ndarray							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : List[str]   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='np'							)
							self.assertTrue(isinstance(batch['input_ids']	,   np.ndarray							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_torch
	def 							lowerCamelCase__		(				self							)    ->	List[Any]:
							import torch

							lowercase_   : Union[str, Any]   	=      [{'input_ids': torch.tensor([0, 1, 2]							)}, {'input_ids': torch.tensor([0, 1, 2, 3]							)}]
							lowercase_   : Optional[int]   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : str   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   torch.Tensor							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : Tuple   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='pt'							)
							self.assertTrue(isinstance(batch['input_ids']	,   torch.Tensor							)							)
							self.assertEqual(batch['input_ids'].tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_tf
	def 							lowerCamelCase__		(				self							)    ->	Any:
							import tensorflow as tf

							lowercase_   : Any   	=      [{'input_ids': tf.constant([0, 1, 2]							)}, {'input_ids': tf.constant([0, 1, 2, 3]							)}]
							lowercase_   : Dict   	=      BertTokenizer.from_pretrained('bert-base-cased'							)

							lowercase_   : Any   	=      tokenizer.pad(_lowercase	,   padding=_lowercase							)
							self.assertTrue(isinstance(batch['input_ids']	,   tf.Tensor							)							)
							self.assertEqual(batch['input_ids'].numpy().tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
							lowercase_   : Dict   	=      tokenizer.pad(_lowercase	,   padding=_lowercase	,   return_tensors='tf'							)
							self.assertTrue(isinstance(batch['input_ids']	,   tf.Tensor							)							)
							self.assertEqual(batch['input_ids'].numpy().tolist()	,   [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]							)
	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	str:
							lowercase_   : Optional[int]   	=      Tokenizer(WordPiece(unk_token='[UNK]'							)							)
							PreTrainedTokenizerFast(tokenizer_object=_lowercase							)







	@require_tokenizers
	def 							lowerCamelCase__		(				self							)    ->	int:
							lowercase_   : Union[str, Any]   	=      Tokenizer(WordPiece(unk_token='[UNK]'							)							)
							with tempfile.TemporaryDirectory() as tmpdirname:
													bert_tokenizer.save(os.path.join(_lowercase	,   'tokenizer.json'							)							)
													PreTrainedTokenizerFast(tokenizer_file=os.path.join(_lowercase	,   'tokenizer.json'							)							)
