




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
 from tokenizers import Tokenizer
 from tokenizers.models import WordPiece

class 			_lowerCamelCase    (      unittest.TestCase ):

      """simple docstring"""

      def    snake_case						(  self       :			Tuple			,							snake_case       :			Optional[int]					):
          __UpperCamelCase			=    list(tokenizer_class.max_model_input_sizes.keys()					)
          for model_name in sa_models[:1]:
              __UpperCamelCase			=    tokenizer_class.from_pretrained(lowerCamelCase__					)
              self.assertIsNotNone(lowerCamelCase__					)
              self.assertIsInstance(lowerCamelCase__			,							lowerCamelCase__					)
              self.assertIsInstance(lowerCamelCase__			,							lowerCamelCase__					)

              for special_tok in tokenizer.all_special_tokens:
                  self.assertIsInstance(lowerCamelCase__			,							lowerCamelCase__					)
                  __UpperCamelCase			=    tokenizer.convert_tokens_to_ids(lowerCamelCase__					)
                  self.assertIsInstance(lowerCamelCase__			,							lowerCamelCase__					)







      def    snake_case						(  self       :			Optional[Any]			,							snake_case       :			BatchEncoding			,							snake_case       :			Optional[Callable] = None					):
          __UpperCamelCase			=    pickle.dumps(lowerCamelCase__					)
          self.assertIsNotNone(lowerCamelCase__					)

          __UpperCamelCase			=    pickle.loads(lowerCamelCase__					)

          # Ensure is_fast is correctly restored
          self.assertEqual(be_restored.is_fast			,							be_original.is_fast					)

          # Ensure encodings are potentially correctly restored
          if be_original.is_fast:
              self.assertIsNotNone(be_restored.encodings					)
          else:
              self.assertIsNone(be_restored.encodings					)

          # Ensure the keys are the same
          for original_v, restored_v in zip(be_original.values()			,							be_restored.values()					):
              if equal_op:
                  self.assertTrue(equal_op(lowerCamelCase__			,							lowerCamelCase__					)					)
              else:
                  self.assertEqual(lowerCamelCase__			,							lowerCamelCase__					)







      @slow
      def    snake_case						(  self       :			Dict					):
          self.check_tokenizer_from_pretrained(lowerCamelCase__					)







      def    snake_case						(  self       :			Optional[Any]					):
          self.assertEqual(TensorType('''tf'''					)			,							TensorType.TENSORFLOW					)
          self.assertEqual(TensorType('''pt'''					)			,							TensorType.PYTORCH					)
          self.assertEqual(TensorType('''np'''					)			,							TensorType.NUMPY					)







      @require_tokenizers
      def    snake_case						(  self       :			int					):
          import numpy as np

          __UpperCamelCase			=    BertTokenizer.from_pretrained('''bert-base-cased'''					)
          __UpperCamelCase			=    BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

          # Python no tensor
          with self.subTest('''BatchEncoding (Python, return_tensors=None)'''					):
              self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''					)					)

          with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''					):
              self.assert_dump_and_restore(
                  tokenizer_p('''Small example to encode'''			,							return_tensors=TensorType.NUMPY					)			,							np.array_equal					)

          with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''					):
              self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''					)					)

          with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''					):
              self.assert_dump_and_restore(
                  tokenizer_r('''Small example to encode'''			,							return_tensors=TensorType.NUMPY					)			,							np.array_equal					)







      @require_tf
      @require_tokenizers
      def    snake_case						(  self       :			Tuple					):
          import tensorflow as tf

          def tf_array_equals(snake_case       :			Tuple			,							snake_case       :			List[Any]					):
              return tf.reduce_all(tf.equal(lowerCamelCase__			,							lowerCamelCase__					)					)

          __UpperCamelCase			=    BertTokenizer.from_pretrained('''bert-base-cased'''					)
          __UpperCamelCase			=    BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

          with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''					):
              self.assert_dump_and_restore(
                  tokenizer_p('''Small example to encode'''			,							return_tensors=TensorType.TENSORFLOW					)			,							lowerCamelCase__					)

          with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''					):
              self.assert_dump_and_restore(
                  tokenizer_r('''Small example to encode'''			,							return_tensors=TensorType.TENSORFLOW					)			,							lowerCamelCase__					)







      @require_torch
      @require_tokenizers
      def    snake_case						(  self       :			Optional[int]					):
          import torch

          __UpperCamelCase			=    BertTokenizer.from_pretrained('''bert-base-cased'''					)
          __UpperCamelCase			=    BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

          with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''					):
              self.assert_dump_and_restore(
                  tokenizer_p('''Small example to encode'''			,							return_tensors=TensorType.PYTORCH					)			,							torch.equal					)

          with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''					):
              self.assert_dump_and_restore(
                  tokenizer_r('''Small example to encode'''			,							return_tensors=TensorType.PYTORCH					)			,							torch.equal					)







      @require_tokenizers
      def    snake_case						(  self       :			Optional[int]					):
          __UpperCamelCase			=    BertTokenizer.from_pretrained('''bert-base-cased'''					)
          __UpperCamelCase			=    BertTokenizerFast.from_pretrained('''bert-base-cased'''					)

          with self.subTest('''Python Tokenizer'''					):
              self.assertFalse(tokenizer_p('''Small example to_encode'''					).is_fast					)

          with self.subTest('''Rust Tokenizer'''					):
              self.assertTrue(tokenizer_r('''Small example to_encode'''					).is_fast					)







      @require_tokenizers
      def    snake_case						(  self       :			List[str]					):
          __UpperCamelCase			=    BertTokenizerFast.from_pretrained('''bert-base-cased'''					)
          __UpperCamelCase			=    tokenizer_r(['''Test''', '''\xad''', '''test''']			,							is_split_into_words=lowerCamelCase__					)

          self.assertEqual(encoded.word_to_tokens(0					)			,							TokenSpan(start=1			,							end=2					)					)
          self.assertEqual(encoded.word_to_tokens(1					)			,							lowerCamelCase__					)
          self.assertEqual(encoded.word_to_tokens(2					)			,							TokenSpan(start=2			,							end=3					)					)







      def    snake_case						(  self       :			Optional[int]					):
          __UpperCamelCase			=    BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
          __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''np'''					)
          self.assertEqual(tensor_batch['''inputs'''].shape			,							(2, 3)					)
          self.assertEqual(tensor_batch['''labels'''].shape			,							(2,)					)
          # test converting the converted
          with CaptureStderr() as cs:
              __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''np'''					)
          self.assertFalse(len(cs.err					)			,							msg=F"should have no warning, but got {cs.err}"					)

          __UpperCamelCase			=    BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
          __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''np'''			,							prepend_batch_axis=lowerCamelCase__					)
          self.assertEqual(tensor_batch['''inputs'''].shape			,							(1, 3)					)
          self.assertEqual(tensor_batch['''labels'''].shape			,							(1,)					)







      @require_torch
      def    snake_case						(  self       :			Tuple					):
          __UpperCamelCase			=    BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
          __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''pt'''					)
          self.assertEqual(tensor_batch['''inputs'''].shape			,							(2, 3)					)
          self.assertEqual(tensor_batch['''labels'''].shape			,							(2,)					)
          # test converting the converted
          with CaptureStderr() as cs:
              __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''pt'''					)
          self.assertFalse(len(cs.err					)			,							msg=F"should have no warning, but got {cs.err}"					)

          __UpperCamelCase			=    BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
          __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''pt'''			,							prepend_batch_axis=lowerCamelCase__					)
          self.assertEqual(tensor_batch['''inputs'''].shape			,							(1, 3)					)
          self.assertEqual(tensor_batch['''labels'''].shape			,							(1,)					)







      @require_tf
      def    snake_case						(  self       :			List[Any]					):
          __UpperCamelCase			=    BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
          __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''tf'''					)
          self.assertEqual(tensor_batch['''inputs'''].shape			,							(2, 3)					)
          self.assertEqual(tensor_batch['''labels'''].shape			,							(2,)					)
          # test converting the converted
          with CaptureStderr() as cs:
              __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''tf'''					)
          self.assertFalse(len(cs.err					)			,							msg=F"should have no warning, but got {cs.err}"					)

          __UpperCamelCase			=    BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
          __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''tf'''			,							prepend_batch_axis=lowerCamelCase__					)
          self.assertEqual(tensor_batch['''inputs'''].shape			,							(1, 3)					)
          self.assertEqual(tensor_batch['''labels'''].shape			,							(1,)					)







      @require_flax
      def    snake_case						(  self       :			Dict					):
          __UpperCamelCase			=    BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}					)
          __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''jax'''					)
          self.assertEqual(tensor_batch['''inputs'''].shape			,							(2, 3)					)
          self.assertEqual(tensor_batch['''labels'''].shape			,							(2,)					)
          # test converting the converted
          with CaptureStderr() as cs:
              __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''jax'''					)
          self.assertFalse(len(cs.err					)			,							msg=F"should have no warning, but got {cs.err}"					)

          __UpperCamelCase			=    BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}					)
          __UpperCamelCase			=    batch.convert_to_tensors(tensor_type='''jax'''			,							prepend_batch_axis=lowerCamelCase__					)
          self.assertEqual(tensor_batch['''inputs'''].shape			,							(1, 3)					)
          self.assertEqual(tensor_batch['''labels'''].shape			,							(1,)					)







      def    snake_case						(  self       :			Optional[Any]					):
          __UpperCamelCase			=    [{'''input_ids''': np.array([0, 1, 2]					)}, {'''input_ids''': np.array([0, 1, 2, 3]					)}]
          __UpperCamelCase			=    BertTokenizer.from_pretrained('''bert-base-cased'''					)

          __UpperCamelCase			=    tokenizer.pad(lowerCamelCase__			,							padding=lowerCamelCase__					)
          self.assertTrue(isinstance(batch['''input_ids''']			,							np.ndarray					)					)
          self.assertEqual(batch['''input_ids'''].tolist()			,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
          __UpperCamelCase			=    tokenizer.pad(lowerCamelCase__			,							padding=lowerCamelCase__			,							return_tensors='''np'''					)
          self.assertTrue(isinstance(batch['''input_ids''']			,							np.ndarray					)					)
          self.assertEqual(batch['''input_ids'''].tolist()			,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)







      @require_torch
      def    snake_case						(  self       :			Any					):
          import torch

          __UpperCamelCase			=    [{'''input_ids''': torch.tensor([0, 1, 2]					)}, {'''input_ids''': torch.tensor([0, 1, 2, 3]					)}]
          __UpperCamelCase			=    BertTokenizer.from_pretrained('''bert-base-cased'''					)

          __UpperCamelCase			=    tokenizer.pad(lowerCamelCase__			,							padding=lowerCamelCase__					)
          self.assertTrue(isinstance(batch['''input_ids''']			,							torch.Tensor					)					)
          self.assertEqual(batch['''input_ids'''].tolist()			,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
          __UpperCamelCase			=    tokenizer.pad(lowerCamelCase__			,							padding=lowerCamelCase__			,							return_tensors='''pt'''					)
          self.assertTrue(isinstance(batch['''input_ids''']			,							torch.Tensor					)					)
          self.assertEqual(batch['''input_ids'''].tolist()			,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)







      @require_tf
      def    snake_case						(  self       :			Tuple					):
          import tensorflow as tf

          __UpperCamelCase			=    [{'''input_ids''': tf.constant([0, 1, 2]					)}, {'''input_ids''': tf.constant([0, 1, 2, 3]					)}]
          __UpperCamelCase			=    BertTokenizer.from_pretrained('''bert-base-cased'''					)

          __UpperCamelCase			=    tokenizer.pad(lowerCamelCase__			,							padding=lowerCamelCase__					)
          self.assertTrue(isinstance(batch['''input_ids''']			,							tf.Tensor					)					)
          self.assertEqual(batch['''input_ids'''].numpy().tolist()			,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
          __UpperCamelCase			=    tokenizer.pad(lowerCamelCase__			,							padding=lowerCamelCase__			,							return_tensors='''tf'''					)
          self.assertTrue(isinstance(batch['''input_ids''']			,							tf.Tensor					)					)
          self.assertEqual(batch['''input_ids'''].numpy().tolist()			,							[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)







      @require_tokenizers
      def    snake_case						(  self       :			Union[str, Any]					):
          __UpperCamelCase			=    Tokenizer(WordPiece(unk_token='''[UNK]'''					)					)
          PreTrainedTokenizerFast(tokenizer_object=lowerCamelCase__					)







      @require_tokenizers
      def    snake_case						(  self       :			Any					):
          __UpperCamelCase			=    Tokenizer(WordPiece(unk_token='''[UNK]'''					)					)
          with tempfile.TemporaryDirectory() as tmpdirname:
              bert_tokenizer.save(os.path.join(lowerCamelCase__			,							'''tokenizer.json'''					)					)
              PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowerCamelCase__			,							'''tokenizer.json'''					)					)







"""simple docstring"""






def     a_		(   lowerCamelCase			):
				if num < 0:
								return -1
				return 1 if num else 0



def     a_		(   ):
				assert signum(5			) == 1
				assert signum(-5			) == -1
				assert signum(0			) == 0


if __name__ == "__main__":
						print(signum(12))
						print(signum(-12))
						print(signum(0))



