



"""simple docstring"""







import json
import os
import unittest

from transformers import BartTokenizer, BartTokenizerFast, BatchEncoding
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors





@require_tokenizers
class   UpperCamelCase	(     snake_case					,   unittest.TestCase       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       str						       =  BartTokenizer
	SCREAMING_SNAKE_CASE_							:       Optional[int]						       =  BartTokenizerFast
	SCREAMING_SNAKE_CASE_							:       Dict						       =  True
	SCREAMING_SNAKE_CASE_							:       List[str]						       =  filter_roberta_detectors






	def    lowerCamelCase__					(     self   ):
			super().setUp()
			_lowercase				: Tuple     =   [
			    """l""",
			    """o""",
			    """w""",
			    """e""",
			    """r""",
			    """s""",
			    """t""",
			    """i""",
			    """d""",
			    """n""",
			    """\u0120""",
			    """\u0120l""",
			    """\u0120n""",
			    """\u0120lo""",
			    """\u0120low""",
			    """er""",
			    """\u0120lowest""",
			    """\u0120newer""",
			    """\u0120wider""",
			    """<unk>""",
			]
			_lowercase				: Tuple     =   dict(zip(UpperCAmelCase_				,range(len(UpperCAmelCase_   )   )   )   )
			_lowercase				: Tuple     =   ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
			_lowercase				: Any     =   {"""unk_token""": """<unk>"""}

			_lowercase				: int     =   os.path.join(self.tmpdirname				,VOCAB_FILES_NAMES["""vocab_file"""]   )
			_lowercase				: Tuple     =   os.path.join(self.tmpdirname				,VOCAB_FILES_NAMES["""merges_file"""]   )
			with open(self.vocab_file				,"""w"""				,encoding="""utf-8"""   ) as fp:
					fp.write(json.dumps(UpperCAmelCase_   ) + """\n"""   )
			with open(self.merges_file				,"""w"""				,encoding="""utf-8"""   ) as fp:
					fp.write("""\n""".join(UpperCAmelCase_   )   )






	def    lowerCamelCase__					(     self				,**UpperCAmelCase_   ):
			kwargs.update(self.special_tokens_map   )
			return self.tokenizer_class.from_pretrained(self.tmpdirname				,**UpperCAmelCase_   )






	def    lowerCamelCase__					(     self				,**UpperCAmelCase_   ):
			kwargs.update(self.special_tokens_map   )
			return self.rust_tokenizer_class.from_pretrained(self.tmpdirname				,**UpperCAmelCase_   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_   ):
			return "lower newer", "lower newer"






	@cached_property
	def    lowerCamelCase__					(     self   ):
			return BartTokenizer.from_pretrained("""facebook/bart-large"""   )






	@cached_property
	def    lowerCamelCase__					(     self   ):
			return BartTokenizerFast.from_pretrained("""facebook/bart-large"""   )






	@require_torch
	def    lowerCamelCase__					(     self   ):
			_lowercase				: int     =   ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
			_lowercase				: Any     =   [0, 2_50, 2_51, 1_78_18, 13, 3_91_86, 19_38, 4, 2]

			for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
					_lowercase				: Optional[Any]     =   tokenizer(UpperCAmelCase_				,max_length=len(UpperCAmelCase_   )				,padding=UpperCAmelCase_				,return_tensors="""pt"""   )
					self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )

					self.assertEqual((2, 9)				,batch.input_ids.shape   )
					self.assertEqual((2, 9)				,batch.attention_mask.shape   )
					_lowercase				: str     =   batch.input_ids.tolist()[0]
					self.assertListEqual(UpperCAmelCase_				,UpperCAmelCase_   )
					# Test that special tokens are reset






	@require_torch
	def    lowerCamelCase__					(     self   ):
			_lowercase				: str     =   ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
			for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
					_lowercase				: Union[str, Any]     =   tokenizer(UpperCAmelCase_				,padding=UpperCAmelCase_				,return_tensors="""pt"""   )
					# check if input_ids are returned and no labels
					self.assertIn("""input_ids"""				,UpperCAmelCase_   )
					self.assertIn("""attention_mask"""				,UpperCAmelCase_   )
					self.assertNotIn("""labels"""				,UpperCAmelCase_   )
					self.assertNotIn("""decoder_attention_mask"""				,UpperCAmelCase_   )






	@require_torch
	def    lowerCamelCase__					(     self   ):
			_lowercase				: Any     =   [
			    """Summary of the text.""",
			    """Another summary.""",
			]
			for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
					_lowercase				: Optional[int]     =   tokenizer(text_target=UpperCAmelCase_				,max_length=32				,padding="""max_length"""				,return_tensors="""pt"""   )
					self.assertEqual(32				,targets["""input_ids"""].shape[1]   )






	@require_torch
	def    lowerCamelCase__					(     self   ):
			for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
					_lowercase				: Tuple     =   tokenizer(
					    ["""I am a small frog""" * 10_24, """I am a small frog"""]				,padding=UpperCAmelCase_				,truncation=UpperCAmelCase_				,return_tensors="""pt"""   )
					self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )
					self.assertEqual(batch.input_ids.shape				,(2, 10_24)   )






	@require_torch
	def    lowerCamelCase__					(     self   ):
			_lowercase				: Tuple     =   ["""A long paragraph for summarization."""]
			_lowercase				: Any     =   [
			    """Summary of the text.""",
			]
			for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
					_lowercase				: Dict     =   tokenizer(UpperCAmelCase_				,return_tensors="""pt"""   )
					_lowercase				: List[str]     =   tokenizer(text_target=UpperCAmelCase_				,return_tensors="""pt"""   )
					_lowercase				: Optional[int]     =   inputs["""input_ids"""]
					_lowercase				: Optional[int]     =   targets["""input_ids"""]
					self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()   )
					self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()   )
					self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()   )
					self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()   )






	def    lowerCamelCase__					(     self   ):
			pass






	def    lowerCamelCase__					(     self   ):
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
							_lowercase				: Optional[Any]     =   self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,**UpperCAmelCase_   )
							_lowercase				: List[str]     =   self.tokenizer_class.from_pretrained(UpperCAmelCase_				,**UpperCAmelCase_   )
							_lowercase				: List[Any]     =   """A, <mask> AllenNLP sentence."""
							_lowercase				: Union[str, Any]     =   tokenizer_r.encode_plus(UpperCAmelCase_				,add_special_tokens=UpperCAmelCase_				,return_token_type_ids=UpperCAmelCase_   )
							_lowercase				: List[Any]     =   tokenizer_p.encode_plus(UpperCAmelCase_				,add_special_tokens=UpperCAmelCase_				,return_token_type_ids=UpperCAmelCase_   )

							# token_type_ids should put 0 everywhere
							self.assertEqual(sum(tokens_r["""token_type_ids"""]   )				,sum(tokens_p["""token_type_ids"""]   )   )

							# attention_mask should put 1 everywhere, so sum over length should be 1
							self.assertEqual(
							    sum(tokens_r["""attention_mask"""]   ) / len(tokens_r["""attention_mask"""]   )				,sum(tokens_p["""attention_mask"""]   ) / len(tokens_p["""attention_mask"""]   )				,)

							_lowercase				: Tuple     =   tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]   )
							_lowercase				: List[str]     =   tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]   )

							# Rust correctly handles the space before the mask while python doesnt
							self.assertSequenceEqual(tokens_p["""input_ids"""]				,[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]   )
							self.assertSequenceEqual(tokens_r["""input_ids"""]				,[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]   )

							self.assertSequenceEqual(
							    UpperCAmelCase_				,["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]   )
							self.assertSequenceEqual(
							    UpperCAmelCase_				,["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]   )








"""simple docstring"""







import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
		from tokenizers import Tokenizer
		from tokenizers.models import WordPiece





class   UpperCamelCase	(     unittest.TestCase       ):




	"""simple docstring"""






	def    lowerCamelCase__					(     self				,UpperCAmelCase_   ):
			_lowercase				: Optional[int]     =   list(tokenizer_class.max_model_input_sizes.keys()   )
			for model_name in sa_models[:1]:
					_lowercase				: Any     =   tokenizer_class.from_pretrained(UpperCAmelCase_   )
					self.assertIsNotNone(UpperCAmelCase_   )
					self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )
					self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )

					for special_tok in tokenizer.all_special_tokens:
							self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )
							_lowercase				: Any     =   tokenizer.convert_tokens_to_ids(UpperCAmelCase_   )
							self.assertIsInstance(UpperCAmelCase_				,UpperCAmelCase_   )






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_ = None   ):
			_lowercase				: Any     =   pickle.dumps(UpperCAmelCase_   )
			self.assertIsNotNone(UpperCAmelCase_   )

			_lowercase				: Optional[int]     =   pickle.loads(UpperCAmelCase_   )

			# Ensure is_fast is correctly restored
			self.assertEqual(be_restored.is_fast				,be_original.is_fast   )

			# Ensure encodings are potentially correctly restored
			if be_original.is_fast:
					self.assertIsNotNone(be_restored.encodings   )
			else:
					self.assertIsNone(be_restored.encodings   )

			# Ensure the keys are the same
			for original_v, restored_v in zip(be_original.values()				,be_restored.values()   ):
					if equal_op:
							self.assertTrue(equal_op(UpperCAmelCase_				,UpperCAmelCase_   )   )
					else:
							self.assertEqual(UpperCAmelCase_				,UpperCAmelCase_   )






	@slow
	def    lowerCamelCase__					(     self   ):
			self.check_tokenizer_from_pretrained(UpperCAmelCase_   )






	def    lowerCamelCase__					(     self   ):
			self.assertEqual(TensorType("""tf"""   )				,TensorType.TENSORFLOW   )
			self.assertEqual(TensorType("""pt"""   )				,TensorType.PYTORCH   )
			self.assertEqual(TensorType("""np"""   )				,TensorType.NUMPY   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			import numpy as np

			_lowercase				: Optional[Any]     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
			_lowercase				: Dict     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

			# Python no tensor
			with self.subTest("""BatchEncoding (Python, return_tensors=None)"""   ):
					self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""   )   )

			with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_p("""Small example to encode"""				,return_tensors=TensorType.NUMPY   )				,np.array_equal   )

			with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""   ):
					self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""   )   )

			with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_r("""Small example to encode"""				,return_tensors=TensorType.NUMPY   )				,np.array_equal   )






	@require_tf
	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			import tensorflow as tf

			def tf_array_equals(UpperCAmelCase_				,UpperCAmelCase_   ):
					return tf.reduce_all(tf.equal(UpperCAmelCase_				,UpperCAmelCase_   )   )

			_lowercase				: Optional[int]     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
			_lowercase				: Any     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

			with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_p("""Small example to encode"""				,return_tensors=TensorType.TENSORFLOW   )				,UpperCAmelCase_   )

			with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_r("""Small example to encode"""				,return_tensors=TensorType.TENSORFLOW   )				,UpperCAmelCase_   )






	@require_torch
	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			import torch

			_lowercase				: Union[str, Any]     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
			_lowercase				: int     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

			with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_p("""Small example to encode"""				,return_tensors=TensorType.PYTORCH   )				,torch.equal   )

			with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""   ):
					self.assert_dump_and_restore(
					    tokenizer_r("""Small example to encode"""				,return_tensors=TensorType.PYTORCH   )				,torch.equal   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			_lowercase				: int     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )
			_lowercase				: Tuple     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )

			with self.subTest("""Python Tokenizer"""   ):
					self.assertFalse(tokenizer_p("""Small example to_encode"""   ).is_fast   )

			with self.subTest("""Rust Tokenizer"""   ):
					self.assertTrue(tokenizer_r("""Small example to_encode"""   ).is_fast   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			_lowercase				: Optional[Any]     =   BertTokenizerFast.from_pretrained("""bert-base-cased"""   )
			_lowercase				: Optional[int]     =   tokenizer_r(["""Test""", """\xad""", """test"""]				,is_split_into_words=UpperCAmelCase_   )

			self.assertEqual(encoded.word_to_tokens(0   )				,TokenSpan(start=1				,end=2   )   )
			self.assertEqual(encoded.word_to_tokens(1   )				,UpperCAmelCase_   )
			self.assertEqual(encoded.word_to_tokens(2   )				,TokenSpan(start=2				,end=3   )   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Tuple     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
			_lowercase				: Any     =   batch.convert_to_tensors(tensor_type="""np"""   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(2, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(2,)   )
			# test converting the converted
			with CaptureStderr() as cs:
					_lowercase				: Tuple     =   batch.convert_to_tensors(tensor_type="""np"""   )
			self.assertFalse(len(cs.err   )				,msg=f"""should have no warning, but got {cs.err}"""   )

			_lowercase				: List[str]     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
			_lowercase				: Any     =   batch.convert_to_tensors(tensor_type="""np"""				,prepend_batch_axis=UpperCAmelCase_   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(1, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(1,)   )






	@require_torch
	def    lowerCamelCase__					(     self   ):
			_lowercase				: str     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
			_lowercase				: Any     =   batch.convert_to_tensors(tensor_type="""pt"""   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(2, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(2,)   )
			# test converting the converted
			with CaptureStderr() as cs:
					_lowercase				: str     =   batch.convert_to_tensors(tensor_type="""pt"""   )
			self.assertFalse(len(cs.err   )				,msg=f"""should have no warning, but got {cs.err}"""   )

			_lowercase				: Dict     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
			_lowercase				: List[str]     =   batch.convert_to_tensors(tensor_type="""pt"""				,prepend_batch_axis=UpperCAmelCase_   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(1, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(1,)   )






	@require_tf
	def    lowerCamelCase__					(     self   ):
			_lowercase				: int     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
			_lowercase				: List[str]     =   batch.convert_to_tensors(tensor_type="""tf"""   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(2, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(2,)   )
			# test converting the converted
			with CaptureStderr() as cs:
					_lowercase				: Tuple     =   batch.convert_to_tensors(tensor_type="""tf"""   )
			self.assertFalse(len(cs.err   )				,msg=f"""should have no warning, but got {cs.err}"""   )

			_lowercase				: Tuple     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
			_lowercase				: List[str]     =   batch.convert_to_tensors(tensor_type="""tf"""				,prepend_batch_axis=UpperCAmelCase_   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(1, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(1,)   )






	@require_flax
	def    lowerCamelCase__					(     self   ):
			_lowercase				: Any     =   BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}   )
			_lowercase				: Tuple     =   batch.convert_to_tensors(tensor_type="""jax"""   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(2, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(2,)   )
			# test converting the converted
			with CaptureStderr() as cs:
					_lowercase				: str     =   batch.convert_to_tensors(tensor_type="""jax"""   )
			self.assertFalse(len(cs.err   )				,msg=f"""should have no warning, but got {cs.err}"""   )

			_lowercase				: Union[str, Any]     =   BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}   )
			_lowercase				: List[Any]     =   batch.convert_to_tensors(tensor_type="""jax"""				,prepend_batch_axis=UpperCAmelCase_   )
			self.assertEqual(tensor_batch["""inputs"""].shape				,(1, 3)   )
			self.assertEqual(tensor_batch["""labels"""].shape				,(1,)   )






	def    lowerCamelCase__					(     self   ):
			_lowercase				: Optional[int]     =   [{"""input_ids""": np.array([0, 1, 2]   )}, {"""input_ids""": np.array([0, 1, 2, 3]   )}]
			_lowercase				: Optional[int]     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )

			_lowercase				: List[str]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,np.ndarray   )   )
			self.assertEqual(batch["""input_ids"""].tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
			_lowercase				: Optional[int]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_				,return_tensors="""np"""   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,np.ndarray   )   )
			self.assertEqual(batch["""input_ids"""].tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )






	@require_torch
	def    lowerCamelCase__					(     self   ):
			import torch

			_lowercase				: List[Any]     =   [{"""input_ids""": torch.tensor([0, 1, 2]   )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]   )}]
			_lowercase				: Tuple     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )

			_lowercase				: List[Any]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,torch.Tensor   )   )
			self.assertEqual(batch["""input_ids"""].tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
			_lowercase				: List[Any]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_				,return_tensors="""pt"""   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,torch.Tensor   )   )
			self.assertEqual(batch["""input_ids"""].tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )






	@require_tf
	def    lowerCamelCase__					(     self   ):
			import tensorflow as tf

			_lowercase				: List[Any]     =   [{"""input_ids""": tf.constant([0, 1, 2]   )}, {"""input_ids""": tf.constant([0, 1, 2, 3]   )}]
			_lowercase				: Dict     =   BertTokenizer.from_pretrained("""bert-base-cased"""   )

			_lowercase				: Optional[int]     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,tf.Tensor   )   )
			self.assertEqual(batch["""input_ids"""].numpy().tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
			_lowercase				: str     =   tokenizer.pad(UpperCAmelCase_				,padding=UpperCAmelCase_				,return_tensors="""tf"""   )
			self.assertTrue(isinstance(batch["""input_ids"""]				,tf.Tensor   )   )
			self.assertEqual(batch["""input_ids"""].numpy().tolist()				,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			_lowercase				: Tuple     =   Tokenizer(WordPiece(unk_token="""[UNK]"""   )   )
			PreTrainedTokenizerFast(tokenizer_object=UpperCAmelCase_   )






	@require_tokenizers
	def    lowerCamelCase__					(     self   ):
			_lowercase				: List[Any]     =   Tokenizer(WordPiece(unk_token="""[UNK]"""   )   )
			with tempfile.TemporaryDirectory() as tmpdirname:
					bert_tokenizer.save(os.path.join(UpperCAmelCase_				,"""tokenizer.json"""   )   )
					PreTrainedTokenizerFast(tokenizer_file=os.path.join(UpperCAmelCase_				,"""tokenizer.json"""   )   )



