



import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
			from tokenizers import Tokenizer
			from tokenizers.models import WordPiece
class       _lowerCAmelCase (			unittest.TestCase      ):






			"""simple docstring"""





			def 		snake_case__      (					self						:							int      ,  lowercase						:							Optional[int] )    ->       Optional[Any]:
									"""simple docstring"""







									__lowercase											=					list(tokenizer_class.max_model_input_sizes.keys() )
									for model_name in sa_models[:1]:
															__lowercase											=					tokenizer_class.from_pretrained(lowercase )
															self.assertIsNotNone(lowercase )
															self.assertIsInstance(lowercase      ,  lowercase )
															self.assertIsInstance(lowercase      ,  lowercase )

															for special_tok in tokenizer.all_special_tokens:
																					self.assertIsInstance(lowercase      ,  lowercase )
																					__lowercase											=					tokenizer.convert_tokens_to_ids(lowercase )
																					self.assertIsInstance(lowercase      ,  lowercase )





			def 		snake_case__      (					self						:							Optional[int]      ,  lowercase						:							BatchEncoding      ,  lowercase						:							Optional[Callable] = None )    ->       Union[str, Any]:
									"""simple docstring"""







									__lowercase											=					pickle.dumps(lowercase )
									self.assertIsNotNone(lowercase )

									__lowercase											=					pickle.loads(lowercase )

									# Ensure is_fast is correctly restored
									self.assertEqual(be_restored.is_fast      ,  be_original.is_fast )

									# Ensure encodings are potentially correctly restored
									if be_original.is_fast:
															self.assertIsNotNone(be_restored.encodings )
									else:
															self.assertIsNone(be_restored.encodings )

									# Ensure the keys are the same
									for original_v, restored_v in zip(be_original.values()      ,  be_restored.values() ):
															if equal_op:
																					self.assertTrue(equal_op(lowercase      ,  lowercase ) )
															else:
																					self.assertEqual(lowercase      ,  lowercase )





			@slow
			def 		snake_case__      (					self						:							Dict )    ->       Optional[int]:
									"""simple docstring"""







									self.check_tokenizer_from_pretrained(lowercase )





			def 		snake_case__      (					self						:							Union[str, Any] )    ->       Any:
									"""simple docstring"""







									self.assertEqual(TensorType("""tf""" )      ,  TensorType.TENSORFLOW )
									self.assertEqual(TensorType("""pt""" )      ,  TensorType.PYTORCH )
									self.assertEqual(TensorType("""np""" )      ,  TensorType.NUMPY )





			@require_tokenizers
			def 		snake_case__      (					self						:							str )    ->       int:
									"""simple docstring"""







									import numpy as np

									__lowercase											=					BertTokenizer.from_pretrained("""bert-base-cased""" )
									__lowercase											=					BertTokenizerFast.from_pretrained("""bert-base-cased""" )

									# Python no tensor
									with self.subTest("""BatchEncoding (Python, return_tensors=None)""" ):
															self.assert_dump_and_restore(tokenizer_p("""Small example to encode""" ) )

									with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)""" ):
															self.assert_dump_and_restore(
															    tokenizer_p("""Small example to encode"""      ,  return_tensors=TensorType.NUMPY )      ,  np.array_equal )

									with self.subTest("""BatchEncoding (Rust, return_tensors=None)""" ):
															self.assert_dump_and_restore(tokenizer_r("""Small example to encode""" ) )

									with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)""" ):
															self.assert_dump_and_restore(
															    tokenizer_r("""Small example to encode"""      ,  return_tensors=TensorType.NUMPY )      ,  np.array_equal )





			@require_tf
			@require_tokenizers
			def 		snake_case__      (					self						:							Optional[int] )    ->       Dict:
									"""simple docstring"""







									import tensorflow as tf

									def tf_array_equals(lowercase						:							Union[str, Any]      ,  lowercase						:							Union[str, Any] ):
															return tf.reduce_all(tf.equal(lowercase      ,  lowercase ) )

									__lowercase											=					BertTokenizer.from_pretrained("""bert-base-cased""" )
									__lowercase											=					BertTokenizerFast.from_pretrained("""bert-base-cased""" )

									with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)""" ):
															self.assert_dump_and_restore(
															    tokenizer_p("""Small example to encode"""      ,  return_tensors=TensorType.TENSORFLOW )      ,  lowercase )

									with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)""" ):
															self.assert_dump_and_restore(
															    tokenizer_r("""Small example to encode"""      ,  return_tensors=TensorType.TENSORFLOW )      ,  lowercase )





			@require_torch
			@require_tokenizers
			def 		snake_case__      (					self						:							Optional[int] )    ->       Dict:
									"""simple docstring"""







									import torch

									__lowercase											=					BertTokenizer.from_pretrained("""bert-base-cased""" )
									__lowercase											=					BertTokenizerFast.from_pretrained("""bert-base-cased""" )

									with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)""" ):
															self.assert_dump_and_restore(
															    tokenizer_p("""Small example to encode"""      ,  return_tensors=TensorType.PYTORCH )      ,  torch.equal )

									with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)""" ):
															self.assert_dump_and_restore(
															    tokenizer_r("""Small example to encode"""      ,  return_tensors=TensorType.PYTORCH )      ,  torch.equal )





			@require_tokenizers
			def 		snake_case__      (					self						:							Union[str, Any] )    ->       Tuple:
									"""simple docstring"""







									__lowercase											=					BertTokenizer.from_pretrained("""bert-base-cased""" )
									__lowercase											=					BertTokenizerFast.from_pretrained("""bert-base-cased""" )

									with self.subTest("""Python Tokenizer""" ):
															self.assertFalse(tokenizer_p("""Small example to_encode""" ).is_fast )

									with self.subTest("""Rust Tokenizer""" ):
															self.assertTrue(tokenizer_r("""Small example to_encode""" ).is_fast )





			@require_tokenizers
			def 		snake_case__      (					self						:							Any )    ->       Any:
									"""simple docstring"""







									__lowercase											=					BertTokenizerFast.from_pretrained("""bert-base-cased""" )
									__lowercase											=					tokenizer_r(["""Test""", """\xad""", """test"""]      ,  is_split_into_words=lowercase )

									self.assertEqual(encoded.word_to_tokens(0 )      ,  TokenSpan(start=1      ,  end=2 ) )
									self.assertEqual(encoded.word_to_tokens(1 )      ,  lowercase )
									self.assertEqual(encoded.word_to_tokens(2 )      ,  TokenSpan(start=2      ,  end=3 ) )





			def 		snake_case__      (					self						:							Dict )    ->       str:
									"""simple docstring"""







									__lowercase											=					BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]} )
									__lowercase											=					batch.convert_to_tensors(tensor_type="""np""" )
									self.assertEqual(tensor_batch["""inputs"""].shape      ,  (2, 3) )
									self.assertEqual(tensor_batch["""labels"""].shape      ,  (2,) )
									# test converting the converted
									with CaptureStderr() as cs:
															__lowercase											=					batch.convert_to_tensors(tensor_type="""np""" )
									self.assertFalse(len(cs.err )      ,  msg=F"should have no warning, but got {cs.err}" )

									__lowercase											=					BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0} )
									__lowercase											=					batch.convert_to_tensors(tensor_type="""np"""      ,  prepend_batch_axis=lowercase )
									self.assertEqual(tensor_batch["""inputs"""].shape      ,  (1, 3) )
									self.assertEqual(tensor_batch["""labels"""].shape      ,  (1,) )





			@require_torch
			def 		snake_case__      (					self						:							Any )    ->       Dict:
									"""simple docstring"""







									__lowercase											=					BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]} )
									__lowercase											=					batch.convert_to_tensors(tensor_type="""pt""" )
									self.assertEqual(tensor_batch["""inputs"""].shape      ,  (2, 3) )
									self.assertEqual(tensor_batch["""labels"""].shape      ,  (2,) )
									# test converting the converted
									with CaptureStderr() as cs:
															__lowercase											=					batch.convert_to_tensors(tensor_type="""pt""" )
									self.assertFalse(len(cs.err )      ,  msg=F"should have no warning, but got {cs.err}" )

									__lowercase											=					BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0} )
									__lowercase											=					batch.convert_to_tensors(tensor_type="""pt"""      ,  prepend_batch_axis=lowercase )
									self.assertEqual(tensor_batch["""inputs"""].shape      ,  (1, 3) )
									self.assertEqual(tensor_batch["""labels"""].shape      ,  (1,) )





			@require_tf
			def 		snake_case__      (					self						:							List[str] )    ->       Dict:
									"""simple docstring"""







									__lowercase											=					BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]} )
									__lowercase											=					batch.convert_to_tensors(tensor_type="""tf""" )
									self.assertEqual(tensor_batch["""inputs"""].shape      ,  (2, 3) )
									self.assertEqual(tensor_batch["""labels"""].shape      ,  (2,) )
									# test converting the converted
									with CaptureStderr() as cs:
															__lowercase											=					batch.convert_to_tensors(tensor_type="""tf""" )
									self.assertFalse(len(cs.err )      ,  msg=F"should have no warning, but got {cs.err}" )

									__lowercase											=					BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0} )
									__lowercase											=					batch.convert_to_tensors(tensor_type="""tf"""      ,  prepend_batch_axis=lowercase )
									self.assertEqual(tensor_batch["""inputs"""].shape      ,  (1, 3) )
									self.assertEqual(tensor_batch["""labels"""].shape      ,  (1,) )





			@require_flax
			def 		snake_case__      (					self						:							str )    ->       List[str]:
									"""simple docstring"""







									__lowercase											=					BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]} )
									__lowercase											=					batch.convert_to_tensors(tensor_type="""jax""" )
									self.assertEqual(tensor_batch["""inputs"""].shape      ,  (2, 3) )
									self.assertEqual(tensor_batch["""labels"""].shape      ,  (2,) )
									# test converting the converted
									with CaptureStderr() as cs:
															__lowercase											=					batch.convert_to_tensors(tensor_type="""jax""" )
									self.assertFalse(len(cs.err )      ,  msg=F"should have no warning, but got {cs.err}" )

									__lowercase											=					BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0} )
									__lowercase											=					batch.convert_to_tensors(tensor_type="""jax"""      ,  prepend_batch_axis=lowercase )
									self.assertEqual(tensor_batch["""inputs"""].shape      ,  (1, 3) )
									self.assertEqual(tensor_batch["""labels"""].shape      ,  (1,) )





			def 		snake_case__      (					self						:							List[Any] )    ->       str:
									"""simple docstring"""







									__lowercase											=					[{"""input_ids""": np.array([0, 1, 2] )}, {"""input_ids""": np.array([0, 1, 2, 3] )}]
									__lowercase											=					BertTokenizer.from_pretrained("""bert-base-cased""" )

									__lowercase											=					tokenizer.pad(lowercase      ,  padding=lowercase )
									self.assertTrue(isinstance(batch["""input_ids"""]      ,  np.ndarray ) )
									self.assertEqual(batch["""input_ids"""].tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
									__lowercase											=					tokenizer.pad(lowercase      ,  padding=lowercase      ,  return_tensors="""np""" )
									self.assertTrue(isinstance(batch["""input_ids"""]      ,  np.ndarray ) )
									self.assertEqual(batch["""input_ids"""].tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )





			@require_torch
			def 		snake_case__      (					self						:							Union[str, Any] )    ->       Any:
									"""simple docstring"""







									import torch

									__lowercase											=					[{"""input_ids""": torch.tensor([0, 1, 2] )}, {"""input_ids""": torch.tensor([0, 1, 2, 3] )}]
									__lowercase											=					BertTokenizer.from_pretrained("""bert-base-cased""" )

									__lowercase											=					tokenizer.pad(lowercase      ,  padding=lowercase )
									self.assertTrue(isinstance(batch["""input_ids"""]      ,  torch.Tensor ) )
									self.assertEqual(batch["""input_ids"""].tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
									__lowercase											=					tokenizer.pad(lowercase      ,  padding=lowercase      ,  return_tensors="""pt""" )
									self.assertTrue(isinstance(batch["""input_ids"""]      ,  torch.Tensor ) )
									self.assertEqual(batch["""input_ids"""].tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )





			@require_tf
			def 		snake_case__      (					self						:							str )    ->       Dict:
									"""simple docstring"""







									import tensorflow as tf

									__lowercase											=					[{"""input_ids""": tf.constant([0, 1, 2] )}, {"""input_ids""": tf.constant([0, 1, 2, 3] )}]
									__lowercase											=					BertTokenizer.from_pretrained("""bert-base-cased""" )

									__lowercase											=					tokenizer.pad(lowercase      ,  padding=lowercase )
									self.assertTrue(isinstance(batch["""input_ids"""]      ,  tf.Tensor ) )
									self.assertEqual(batch["""input_ids"""].numpy().tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )
									__lowercase											=					tokenizer.pad(lowercase      ,  padding=lowercase      ,  return_tensors="""tf""" )
									self.assertTrue(isinstance(batch["""input_ids"""]      ,  tf.Tensor ) )
									self.assertEqual(batch["""input_ids"""].numpy().tolist()      ,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]] )





			@require_tokenizers
			def 		snake_case__      (					self						:							Optional[Any] )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					Tokenizer(WordPiece(unk_token="""[UNK]""" ) )
									PreTrainedTokenizerFast(tokenizer_object=lowercase )







			@require_tokenizers
			def 		snake_case__      (					self						:							Tuple )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					Tokenizer(WordPiece(unk_token="""[UNK]""" ) )
									with tempfile.TemporaryDirectory() as tmpdirname:
															bert_tokenizer.save(os.path.join(lowercase      ,  """tokenizer.json""" ) )
															PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowercase      ,  """tokenizer.json""" ) )










from .stack import Stack






def 			UpperCAmelCase__     (     lowercase__ )			->   bool:
						__lowercase											=					Stack()
						__lowercase											=					{"""(""": """)""", """[""": """]""", """{""": """}"""}
						for bracket in parentheses:
												if bracket in bracket_pairs:
																		stack.push(lowercase__ )
												elif bracket in (")", "]", "}"):
																		if stack.is_empty() or bracket_pairs[stack.pop()] != bracket:
																								return False
						return stack.is_empty()


if __name__ == "__main__":
			from doctest import testmod

			testmod()

			UpperCamelCase__			=    ["((()))", "((())", "(()))"]
			print("Balanced parentheses demonstration:\n")
			for example in examples:
						UpperCamelCase__			=    "" if balanced_parentheses(example) else "not "
						print(F"""{example} is {not_str}balanced""")





